Merge pull request #98 from LD-Reborn/95-add-parallel-embeddings-pre-fetching-setting

Added entity index embeddings prefetching, fixed zero-searchdomain fr…

src/Server/Helper/SearchdomainHelper.cs

@@ -58,28 +58,42 @@ public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelp
             return null;
         }
 
-        // toBeCached: model -> [datapoint.text * n]
+        // Prefetch embeddings
         Dictionary<string, List<string>> toBeCached = [];
+        Dictionary<string, List<string>> toBeCachedParallel = [];
         foreach (JSONEntity jSONEntity in jsonEntities)
         {
+            Dictionary<string, List<string>> targetDictionary = toBeCached;
+            if (searchdomainManager.GetSearchdomain(jSONEntity.Searchdomain).settings.ParallelEmbeddingsPrefetch)
+            {
+                targetDictionary = toBeCachedParallel;
+            }
             foreach (JSONDatapoint datapoint in jSONEntity.Datapoints)
             {
                 foreach (string model in datapoint.Model)
                 {
-                    if (!toBeCached.ContainsKey(model))
+                    if (!targetDictionary.ContainsKey(model))
                     {
-                        toBeCached[model] = [];
+                        targetDictionary[model] = [];
                     }
-                    toBeCached[model].Add(datapoint.Text);
+                    targetDictionary[model].Add(datapoint.Text);
                 }
             }
         }
+        
         foreach (var toBeCachedKV in toBeCached)
         {
             string model = toBeCachedKV.Key;
             List<string> uniqueStrings = [.. toBeCachedKV.Value.Distinct()];
             Datapoint.GetEmbeddings([.. uniqueStrings], [model], aIProvider, embeddingCache);
         }
+        Parallel.ForEach(toBeCachedParallel, toBeCachedParallelKV =>
+        {
+            string model = toBeCachedParallelKV.Key;
+            List<string> uniqueStrings = [.. toBeCachedParallelKV.Value.Distinct()];
+            Datapoint.GetEmbeddings([.. uniqueStrings], [model], aIProvider, embeddingCache);
+        });
+        // Index/parse the entities
         ConcurrentQueue<Entity> retVal = [];
         ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = 16 }; // <-- This is needed! Otherwise if we try to index 100+ entities at once, it spawns 100 threads, exploding the SQL pool
         Parallel.ForEach(jsonEntities, parallelOptions, jSONEntity =>

src/Server/Resources/SharedResources.de.resx

@@ -315,4 +315,13 @@
   <data name="queryCacheEntryCountLoadedInfo" xml:space="preserve">
     <value>Anzahl der Einträge, die insgesamt in den Query-Cache der geladenen Searchdomains passen.</value>
   </data>
+  <data name="Query cache size" xml:space="preserve">
+    <value>Query Cache Größe</value>
+  </data>
+  <data name="Embeddings parallel prefetching" xml:space="preserve">
+    <value>Embeddings parallel prefetchen</value>
+  </data>
+  <data name="parallelEmbeddingsPrefetchInfo" xml:space="preserve">
+    <value>Wenn diese Einstellung aktiv ist, wird das Abrufen von Embeddings beim Indizieren von Entities parallelisiert. Deaktiviere diese Einstellung, falls Model-unloading ein Problem ist.</value>
+  </data>
 </root>

src/Server/Resources/SharedResources.en.resx

@@ -315,4 +315,13 @@
   <data name="queryCacheEntryCountLoadedInfo" xml:space="preserve">
     <value>Number of query cache entries that can be stored in the query cache of all loaded searchdomains.</value>
   </data>
+  <data name="Query cache size" xml:space="preserve">
+    <value>Query Cache size</value>
+  </data>
+  <data name="Embeddings parallel prefetching" xml:space="preserve">
+    <value>Embeddings parallel prefetching</value>
+  </data>
+  <data name="parallelEmbeddingsPrefetchInfo" xml:space="preserve">
+    <value>With this setting activated the embeddings retrieval will be parallelized when indexing entities. Disable this setting if model unloading is an issue.</value>
+  </data>
 </root>

src/Server/Views/Home/Index.cshtml

@@ -152,13 +152,6 @@
     var searchdomains = null;
 
     document.addEventListener('DOMContentLoaded', async () => {
-        // Initialize all tooltips
-        var tooltipTriggerList = [].slice.call(document.querySelectorAll('[data-bs-toggle="tooltip"]'))
-        var tooltipList = tooltipTriggerList.map(function (tooltipTriggerEl) {
-            return new bootstrap.Tooltip(tooltipTriggerEl)
-        });
-
-
         let searchdomainCount = document.getElementById("searchdomainCount");
         showThrobber(searchdomainCount);
         let searchdomainEntityCount = document.getElementById("searchdomainEntityCount");

src/Server/Views/Home/Searchdomains.cshtml

@@ -66,10 +66,17 @@
               <label class="form-check-label" for="searchdomainConfigQueryCacheSize">@T["Query cache size"]:</label>
               <input type="number" class="form-control" id="searchdomainConfigQueryCacheSize" />
             </div>
-            <div class="col-md-6">
+            <div class="col-md-6 mt-3">
               <input type="checkbox" class="form-check-input" id="searchdomainConfigCacheReconciliation" />
               <label class="form-check-label" for="searchdomainConfigCacheReconciliation">@T["Cache reconciliation"]</label>
             </div>
+            <div class="col-md-6 mt-3">
+              <input type="checkbox" class="form-check-input" id="searchdomainConfigParallelEmbeddingsPrefetch" />
+              <label class="form-check-label" for="searchdomainConfigParallelEmbeddingsPrefetch">@T["Embeddings parallel prefetching"]</label>
+              <i class="bi bi-info-circle-fill text-info" 
+                data-bs-toggle="tooltip" 
+                title="@T["parallelEmbeddingsPrefetchInfo"]"></i>
+            </div>
           </div>
           <div class="row align-items-center mb-3">
             <div class="col-md-2 mt-md-0">
@@ -362,10 +369,14 @@
             <label class="form-check-label mb-2" for="createSearchdomainQueryCacheSize">@T["Query cache size"]:</label>
             <input type="number" class="form-control" id="createSearchdomainQueryCacheSize" />
           </div>
-          <div class="col-md-7">
+          <div class="col-md-7 mt-3">
             <input type="checkbox" class="form-check-input" id="createSearchdomainWithCacheReconciliation" />
             <label class="form-check-label" for="createSearchdomainWithCacheReconciliation">@T["Enable cache reconciliation"]</label>
           </div>
+          <div class="col-md-6 mt-3">
+            <input type="checkbox" class="form-check-input" id="createSearchdomainConfigParallelEmbeddingsPrefetch" />
+            <label class="form-check-label" for="createSearchdomainConfigParallelEmbeddingsPrefetch">@T["Embeddings parallel prefetching"]</label>
+          </div>
         </div>
       </div>
 
@@ -672,7 +683,10 @@
         queriesFilter.addEventListener('input', () => {
             populateQueriesTable(queriesFilter.value);
         });
-        selectDomain(0);
+        try
+        {
+            selectDomain(0);
+        } catch (error) {}
 
         document
             .getElementById('searchdomainRename')
@@ -711,7 +725,8 @@
                 const domainKey = getSelectedDomainKey();
                 const cacheReconciliation = document.getElementById('searchdomainConfigCacheReconciliation').checked;
                 const queryCacheSize = document.getElementById('searchdomainConfigQueryCacheSize').value;
-                updateSearchdomainConfig(domainKey, { CacheReconciliation: cacheReconciliation, QueryCacheSize: queryCacheSize});
+                const parallelEmbeddingsPrefetch = document.getElementById('searchdomainConfigParallelEmbeddingsPrefetch').checked;
+                updateSearchdomainConfig(domainKey, { CacheReconciliation: cacheReconciliation, QueryCacheSize: queryCacheSize, ParallelEmbeddingsPrefetch: parallelEmbeddingsPrefetch});
             });
 
         document
@@ -794,8 +809,8 @@
                 const name = document.getElementById('createSearchdomainName').value;
                 const queryCacheSize = document.getElementById('createSearchdomainQueryCacheSize').value;
                 const cacheReconciliation = document.getElementById('createSearchdomainWithCacheReconciliation').checked;
-                const settings = { CacheReconciliation: cacheReconciliation, QueryCacheSize: queryCacheSize };
+                const parallelEmbeddingsPrefetch = document.getElementById('createSearchdomainConfigParallelEmbeddingsPrefetch').checked;
-                // Implement create logic here
+                const settings = { CacheReconciliation: cacheReconciliation, QueryCacheSize: queryCacheSize, ParallelEmbeddingsPrefetch: parallelEmbeddingsPrefetch };
                 fetch(`/Searchdomain?searchdomain=${encodeURIComponent(name)}`, {
                     method: 'POST',
                     headers: {
@@ -1072,9 +1087,11 @@
         let searchdomainConfigPromise = getSearchdomainConfig(getSelectedDomainKey());
         let configElementCachereconciliation = document.getElementById('searchdomainConfigCacheReconciliation');
         let configElementCacheSize = document.getElementById('searchdomainConfigQueryCacheSize');
+        let configElementParallelEmbeddingsPrefetch = document.getElementById('searchdomainConfigParallelEmbeddingsPrefetch');
 
         showThrobber(document.querySelector('#searchdomainConfigQueryCacheSize'), true);
         showThrobber(document.querySelector('#searchdomainConfigCacheReconciliation'), true);
+        showThrobber(document.querySelector('#searchdomainConfigParallelEmbeddingsPrefetch'), true);
         let cacheUtilizationPromise = getSearchdomainCacheUtilization(getSelectedDomainKey());
         let databaseUtilizationPromise = getSearchdomainDatabaseUtilization(getSelectedDomainKey());
 
@@ -1117,11 +1134,14 @@
 
         searchdomainConfigPromise.then(searchdomainConfig => {
             hideThrobber(document.querySelector('#searchdomainConfigCacheReconciliation'), true);
+            hideThrobber(document.querySelector('#searchdomainConfigParallelEmbeddingsPrefetch'), true);
+
             if (searchdomainConfig != null && searchdomainConfig.Settings != null)
             {
                 configElementCacheSize.value = searchdomainConfig.Settings.QueryCacheSize;
                 configElementCachereconciliation.checked = searchdomainConfig.Settings.CacheReconciliation;
                 configElementCachereconciliation.disabled = false;
+                configElementParallelEmbeddingsPrefetch.checked = searchdomainConfig.Settings.ParallelEmbeddingsPrefetch;
             } else {
                 configElementCachereconciliation.disabled = true;
                 showToast("@T["Unable to fetch searchdomain config"]", "danger");

src/Server/Views/Shared/_Layout.cshtml

@@ -12,6 +12,7 @@
     <meta name="description" content="Embeddingsearch server" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
     <title>@ViewData["Title"] - embeddingsearch</title>
+    <link rel="preload" href="~/fonts/bootstrap-icons.woff2" as="font" type="font/woff2" crossorigin="anonymous"/>
     @if (!Context.Request.Query.ContainsKey("renderRaw") && !Context.Request.Query.ContainsKey("noCriticalCSS"))
     {
         <link rel="preload" href="~/lib/bootstrap/dist/css/bootstrap.min.css" as="style"/>

src/Server/wwwroot/js/site.js

@@ -49,3 +49,13 @@ function showToast(message, type) {
     bsToast.show();
     toast.addEventListener('hidden.bs.toast', () => toast.remove());
 }
+
+document.addEventListener('DOMContentLoaded', async () => {
+    // Initialize all tooltips
+    var tooltipTriggerList = [].slice.call(document.querySelectorAll('[data-bs-toggle="tooltip"]'))
+    var tooltipList = tooltipTriggerList.map(function (tooltipTriggerEl) {
+        let retVal = new bootstrap.Tooltip(tooltipTriggerEl);
+        tooltipTriggerEl.role = "tooltip";
+        return retVal;
+    });
+});

src/Shared/Models/SearchdomainModels.cs

@@ -95,12 +95,14 @@ public struct DateTimedSearchResult(DateTime dateTime, List<ResultItem> results)
     }
 }
 
-public struct SearchdomainSettings(bool cacheReconciliation = false, int queryCacheSize = 1_000_000)
+public struct SearchdomainSettings(bool cacheReconciliation = false, int queryCacheSize = 1_000_000, bool parallelEmbeddingsPrefetch = false)
 {
     [JsonPropertyName("CacheReconciliation")]
     public bool CacheReconciliation { get; set; } = cacheReconciliation;
     [JsonPropertyName("QueryCacheSize")]
     public int QueryCacheSize { get; set; } = queryCacheSize;
+    [JsonPropertyName("ParallelEmbeddingsPrefetch")]
+    public bool ParallelEmbeddingsPrefetch { get; set; } = parallelEmbeddingsPrefetch;
 }
 
 public static class MemorySizes