lucretia/embeddingsearch
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added query cache size limiting, added custom enumerable LRUCache, renamed search to query in various places, fixed client GetEmbeddingsCacheSize endpoint

Browse Source
This commit is contained in:
LD-Reborn
2026-01-03 17:57:18 +01:00
parent 063c81e8dc
commit 027a9244ad
7 changed files with 270 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
src/Client/Client.cs
View File

@@ -121,13 +121,13 @@ public class Client
}), new StringContent(settings, Encoding.UTF8, "application/json"));
}
public async Task<SearchdomainSearchesResults> SearchdomainGetQueriesAsync(string searchdomain)
public async Task<SearchdomainQueriesResults> SearchdomainGetQueriesAsync(string searchdomain)
{
Dictionary<string, string> parameters = new()
{
{"searchdomain", searchdomain}
};
return await FetchUrlAndProcessJson<SearchdomainSearchesResults>(HttpMethod.Get, GetUrl($"{baseUri}/Searchdomain", "Queries", parameters));
return await FetchUrlAndProcessJson<SearchdomainQueriesResults>(HttpMethod.Get, GetUrl($"{baseUri}/Searchdomain", "Queries", parameters));
}
public async Task<EntityQueryResults> SearchdomainQueryAsync(string query)
@@ -190,13 +190,13 @@ public class Client
return await FetchUrlAndProcessJson<SearchdomainUpdateResults>(HttpMethod.Put, GetUrl($"{baseUri}/Searchdomain", "Settings", parameters), content);
}
public async Task<SearchdomainSearchCacheSizeResults> SearchdomainGetQueryCacheSizeAsync(string searchdomain)
public async Task<SearchdomainQueryCacheSizeResults> SearchdomainGetQueryCacheSizeAsync(string searchdomain)
{
Dictionary<string, string> parameters = new()
{
{"searchdomain", searchdomain}
};
return await FetchUrlAndProcessJson<SearchdomainSearchCacheSizeResults>(HttpMethod.Get, GetUrl($"{baseUri}/Searchdomain/QueryCache", "Size", parameters));
return await FetchUrlAndProcessJson<SearchdomainQueryCacheSizeResults>(HttpMethod.Get, GetUrl($"{baseUri}/Searchdomain/QueryCache", "Size", parameters));
}
public async Task<SearchdomainInvalidateCacheResults> SearchdomainClearQueryCache(string searchdomain)
@@ -222,9 +222,9 @@ public class Client
return await FetchUrlAndProcessJson<ServerGetModelsResult>(HttpMethod.Get, GetUrl($"{baseUri}/Server", "Models", []));
}
public async Task<ServerGetEmbeddingCacheSizeResult> ServerGetEmbeddingCacheSizeAsync()
public async Task<ServerGetStatsResult> ServerGetStatsAsync()
{
return await FetchUrlAndProcessJson<ServerGetEmbeddingCacheSizeResult>(HttpMethod.Get, GetUrl($"{baseUri}/Server/EmbeddingCache", "Size", []));
return await FetchUrlAndProcessJson<ServerGetStatsResult>(HttpMethod.Get, GetUrl($"{baseUri}/Server/Stats", "Size", []));
}
private async Task<T> FetchUrlAndProcessJson<T>(HttpMethod httpMethod, string url, HttpContent? content = null)

20
src/Server/Controllers/SearchdomainController.cs
View File

@@ -5,6 +5,7 @@ using Microsoft.AspNetCore.Http.HttpResults;
using Microsoft.AspNetCore.Mvc;
using Server.Exceptions;
using Server.Helper;
using Shared;
using Shared.Models;
namespace Server.Controllers;
@@ -134,13 +135,13 @@ public class SearchdomainController : ControllerBase
/// </summary>
/// <param name="searchdomain">Name of the searchdomain</param>
[HttpGet("Queries")]
public ActionResult<SearchdomainSearchesResults> GetQueries([Required]string searchdomain)
public ActionResult<SearchdomainQueriesResults> GetQueries([Required]string searchdomain)
{
(Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomain, _logger);
if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new SearchdomainUpdateResults(){Success = false, Message = message});
Dictionary<string, DateTimedSearchResult> searchCache = searchdomain_.searchCache;
Dictionary<string, DateTimedSearchResult> searchCache = searchdomain_.queryCache.AsDictionary();
return Ok(new SearchdomainSearchesResults() { Searches = searchCache, Success = true });
return Ok(new SearchdomainQueriesResults() { Searches = searchCache, Success = true });
}
/// <summary>
@@ -175,7 +176,7 @@ public class SearchdomainController : ControllerBase
{
(Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomain, _logger);
if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new SearchdomainUpdateResults(){Success = false, Message = message});
Dictionary<string, DateTimedSearchResult> searchCache = searchdomain_.searchCache;
EnumerableLruCache<string, DateTimedSearchResult> searchCache = searchdomain_.queryCache;
bool containsKey = searchCache.ContainsKey(query);
if (containsKey)
{
@@ -196,7 +197,7 @@ public class SearchdomainController : ControllerBase
{
(Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomain, _logger);
if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new SearchdomainUpdateResults(){Success = false, Message = message});
Dictionary<string, DateTimedSearchResult> searchCache = searchdomain_.searchCache;
EnumerableLruCache<string, DateTimedSearchResult> searchCache = searchdomain_.queryCache;
bool containsKey = searchCache.ContainsKey(query);
if (containsKey)
{
@@ -237,6 +238,7 @@ public class SearchdomainController : ControllerBase
};
searchdomain_.helper.ExecuteSQLNonQuery("UPDATE searchdomain set settings = @settings WHERE id = @id", parameters);
searchdomain_.settings = request;
searchdomain_.queryCache.Capacity = request.QueryCacheSize;
return Ok(new SearchdomainUpdateResults(){Success = true});
}
@@ -245,15 +247,17 @@ public class SearchdomainController : ControllerBase
/// </summary>
/// <param name="searchdomain">Name of the searchdomain</param>
[HttpGet("QueryCache/Size")]
public ActionResult<SearchdomainSearchCacheSizeResults> GetSearchCacheSize([Required]string searchdomain)
public ActionResult<SearchdomainQueryCacheSizeResults> GetQueryCacheSize([Required]string searchdomain)
{
if (!SearchdomainHelper.IsSearchdomainLoaded(_domainManager, searchdomain))
{
return Ok(new SearchdomainSearchCacheSizeResults() { QueryCacheSizeBytes = 0, Success = true });
return Ok(new SearchdomainQueryCacheSizeResults() { SizeBytes = 0, ElementCount = 0, ElementMaxCount = 0, Success = true });
}
(Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomain, _logger);
if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new SearchdomainUpdateResults(){Success = false, Message = message});
return Ok(new SearchdomainSearchCacheSizeResults() { QueryCacheSizeBytes = searchdomain_.GetSearchCacheSize(), Success = true });
int elementCount = searchdomain_.queryCache.Count();
int ElementMaxCount = searchdomain_.settings.QueryCacheSize;
return Ok(new SearchdomainQueryCacheSizeResults() { SizeBytes = searchdomain_.GetSearchCacheSize(), ElementCount = elementCount, ElementMaxCount = ElementMaxCount, Success = true });
}
/// <summary>

17
src/Server/Searchdomain.cs
View File

@@ -4,6 +4,7 @@ using System.Text.Json;
using ElmahCore.Mvc.Logger;
using MySql.Data.MySqlClient;
using Server.Helper;
using Shared;
using Shared.Models;
using AdaptiveExpressions;
@@ -17,7 +18,7 @@ public class Searchdomain
public string searchdomain;
public int id;
public SearchdomainSettings settings;
public Dictionary<string, DateTimedSearchResult> searchCache; // Key: query, Value: Search results for that query (with timestamp)
public EnumerableLruCache<string, DateTimedSearchResult> queryCache; // Key: query, Value: Search results for that query (with timestamp)
public List<Entity> entityCache;
public List<string> modelsInUse;
public LRUCache<string, Dictionary<string, float[]>> embeddingCache;
@@ -33,12 +34,12 @@ public class Searchdomain
this.aIProvider = aIProvider;
this.embeddingCache = embeddingCache;
this._logger = logger;
searchCache = [];
entityCache = [];
connection = new MySqlConnection(connectionString);
connection.Open();
helper = new SQLHelper(connection, connectionString);
settings = GetSettings();
queryCache = new(settings.QueryCacheSize);
modelsInUse = []; // To make the compiler shut up - it is set in UpdateSearchDomain() don't worry // yeah, about that...
if (!runEmpty)
{
@@ -163,7 +164,7 @@ public class Searchdomain
public List<(float, string)> Search(string query, int? topN = null)
{
if (searchCache.TryGetValue(query, out DateTimedSearchResult cachedResult))
if (queryCache.TryGetValue(query, out DateTimedSearchResult cachedResult))
{
cachedResult.AccessDateTimes.Add(DateTime.Now);
return [.. cachedResult.Results.Select(r => (r.Score, r.Name))];
@@ -187,7 +188,7 @@ public class Searchdomain
[.. sortedResults.Select(r =>
new ResultItem(r.Item1, r.Item2 ))]
);
searchCache[query] = new DateTimedSearchResult(DateTime.Now, searchResult);
queryCache.Set(query, new DateTimedSearchResult(DateTime.Now, searchResult));
return results;
}
@@ -292,7 +293,7 @@ public class Searchdomain
{
if (settings.CacheReconciliation)
{
foreach (KeyValuePair<string, DateTimedSearchResult> element in searchCache)
foreach (var element in queryCache)
{
string query = element.Key;
DateTimedSearchResult searchResult = element.Value;
@@ -322,7 +323,7 @@ public class Searchdomain
{
if (settings.CacheReconciliation)
{
foreach (KeyValuePair<string, DateTimedSearchResult> element in searchCache)
foreach (KeyValuePair<string, DateTimedSearchResult> element in queryCache)
{
string query = element.Key;
DateTimedSearchResult searchResult = element.Value;
@@ -337,13 +338,13 @@ public class Searchdomain
public void InvalidateSearchCache()
{
searchCache = [];
queryCache = new(settings.QueryCacheSize);
}
public long GetSearchCacheSize()
{
long sizeInBytes = 0;
foreach (var entry in searchCache)
foreach (var entry in queryCache)
{
sizeInBytes += sizeof(int); // string length prefix
sizeInBytes += entry.Key.Length * sizeof(char); // string characters

30
src/Server/Views/Home/Searchdomains.cshtml
View File

@@ -62,11 +62,17 @@
<!-- Settings -->
<div class="row align-items-center mb-3">
<h3>@T["Settings"]</h3>
<div class="col-md-3">
<label class="form-check-label" for="searchdomainConfigQueryCacheSize">@T["Query cache size"]:</label>
<input type="number" class="form-control" id="searchdomainConfigQueryCacheSize" />
</div>
<div class="col-md-6">
<input type="checkbox" class="form-check-input" id="searchdomainConfigCacheReconciliation" />
<label class="form-check-label" for="searchdomainConfigCacheReconciliation">@T["Cache reconciliation"]</label>
</div>
<div class="col-md-2 mt-3 mt-md-0">
</div>
<div class="row align-items-center mb-3">
<div class="col-md-2 mt-md-0">
<button class="btn btn-warning w-100" id="searchdomainConfigUpdate">@T["Update"]</button>
</div>
</div>
@@ -347,11 +353,21 @@
</div>
<div class="modal-body">
<div class="row align-items-center mb-3">
<div class="col-md-12">
<label for="createSearchdomainName" class="form-label">@T["Searchdomain name"]</label>
<input type="text" class="form-control mb-3" id="createSearchdomainName" placeholder="@T["Searchdomain name"]" />
</div>
<div class="col-md-5">
<label class="form-check-label mb-2" for="createSearchdomainQueryCacheSize">@T["Query cache size"]:</label>
<input type="number" class="form-control" id="createSearchdomainQueryCacheSize" />
</div>
<div class="col-md-7">
<input type="checkbox" class="form-check-input" id="createSearchdomainWithCacheReconciliation" />
<label class="form-check-label" for="createSearchdomainWithCacheReconciliation">@T["Enable cache reconciliation"]</label>
</div>
</div>
</div>
<div class="modal-footer">
<button type="button" id="searchdomainConfirmCreate" class="btn btn-primary" data-bs-dismiss="modal">
@@ -694,7 +710,8 @@
.addEventListener('click', () => {
const domainKey = getSelectedDomainKey();
const cacheReconciliation = document.getElementById('searchdomainConfigCacheReconciliation').checked;
updateSearchdomainConfig(domainKey, { CacheReconciliation: cacheReconciliation});
const queryCacheSize = document.getElementById('searchdomainConfigQueryCacheSize').value;
updateSearchdomainConfig(domainKey, { CacheReconciliation: cacheReconciliation, QueryCacheSize: queryCacheSize});
});
document
@@ -775,8 +792,9 @@
document.getElementById('createSearchdomainModal')
);
const name = document.getElementById('createSearchdomainName').value;
const queryCacheSize = document.getElementById('createSearchdomainQueryCacheSize').value;
const cacheReconciliation = document.getElementById('createSearchdomainWithCacheReconciliation').checked;
const settings = { CacheReconciliation: cacheReconciliation };
const settings = { CacheReconciliation: cacheReconciliation, QueryCacheSize: queryCacheSize };
// Implement create logic here
fetch(`/Searchdomain?searchdomain=${encodeURIComponent(name)}`, {
method: 'POST',
@@ -1053,6 +1071,7 @@
let searchdomainConfigPromise = getSearchdomainConfig(getSelectedDomainKey());
let configElementCachereconciliation = document.getElementById('searchdomainConfigCacheReconciliation');
let configElementCacheSize = document.getElementById('searchdomainConfigQueryCacheSize');
let cacheUtilizationPromise = getSearchdomainCacheUtilization(getSelectedDomainKey());
let databaseUtilizationPromise = getSearchdomainDatabaseUtilization(getSelectedDomainKey());
@@ -1097,6 +1116,7 @@
searchdomainConfigPromise.then(searchdomainConfig => {
if (searchdomainConfig != null && searchdomainConfig.Settings != null)
{
configElementCacheSize.value = searchdomainConfig.Settings.QueryCacheSize;
configElementCachereconciliation.checked = searchdomainConfig.Settings.CacheReconciliation;
configElementCachereconciliation.disabled = false;
} else {
@@ -1106,10 +1126,10 @@
}
});
cacheUtilizationPromise.then(cacheUtilization => {
if (cacheUtilization != null && cacheUtilization.QueryCacheSizeBytes != null)
if (cacheUtilization != null && cacheUtilization.SizeBytes != null)
{
document.querySelector('#cacheUtilization').innerText =
`${NumberOfBytesAsHumanReadable(cacheUtilization.QueryCacheSizeBytes)}`;
`${NumberOfBytesAsHumanReadable(cacheUtilization.SizeBytes)}`;
} else {
showToast("@T["Unable to fetch searchdomain cache utilization"]", "danger");
console.error('Failed to fetch searchdomain cache utilization');

202
src/Shared/LRUCache.cs Normal file
View File

@@ -0,0 +1,202 @@
namespace Shared;
public class EnumerableLruCache<TKey, TValue> where TKey : notnull
{
private readonly Dictionary<TKey, TValue> _cache;
private readonly LinkedList<TKey> _keys = new();
private int _capacity;
private ReaderWriterLockSlim _readerWriterLock;
public int Capacity
{
get
{
_readerWriterLock.EnterReadLock();
try
{
return _capacity;
}
finally
{
_readerWriterLock.ExitReadLock();
}
}
set
{
_readerWriterLock.EnterWriteLock();
try
{
_capacity = value;
// Trim cache if new capacity is smaller than current size
while (_keys.Count > _capacity)
{
TKey last = _keys.Last!.Value;
_keys.RemoveLast();
_cache.Remove(last);
}
}
finally
{
_readerWriterLock.ExitWriteLock();
}
}
}
public EnumerableLruCache(int capacity)
{
_readerWriterLock = new();
_capacity = capacity;
_cache = [];
}
public TValue this[TKey key]
{
get
{
_readerWriterLock.EnterReadLock();
try
{
return _cache[key];
}
finally
{
_readerWriterLock.ExitReadLock();
}
}
set
{
_readerWriterLock.EnterWriteLock();
try
{
if (_cache.TryGetValue(key, out _))
{
_keys.Remove(key);
}
_keys.AddFirst(key);
_cache[key] = value;
if (_keys.Count > _capacity)
{
TKey last = _keys.Last!.Value;
_keys.RemoveLast();
_cache.Remove(last);
}
}
finally
{
_readerWriterLock.ExitWriteLock();
}
}
}
public bool ContainsKey(TKey key)
{
_readerWriterLock.EnterReadLock();
try
{
return _cache.ContainsKey(key);
} finally
{
_readerWriterLock.ExitReadLock();
}
}
public int Count()
{
_readerWriterLock.EnterReadLock();
try
{
return _cache.Count;
} finally
{
_readerWriterLock.ExitReadLock();
}
}
public void Set(TKey key, TValue value)
{
_readerWriterLock.EnterWriteLock();
try
{
if (_cache.TryGetValue(key, out _))
{
_keys.Remove(key);
}
_keys.AddFirst(key);
_cache[key] = value;
if (_keys.Count > _capacity)
{
TKey? last = _keys.Last();
_keys.RemoveLast();
_cache.Remove(last);
}
} finally
{
_readerWriterLock.ExitWriteLock();
}
}
public void Remove(TKey key)
{
_readerWriterLock.EnterWriteLock();
try
{
_keys.Remove(key);
_cache.Remove(key);
} finally
{
_readerWriterLock.ExitWriteLock();
}
}
public bool TryGetValue(TKey key, out TValue? value)
{
_readerWriterLock.EnterReadLock();
try
{
return _cache.TryGetValue(key, out value);
} finally
{
_readerWriterLock.ExitReadLock();
}
}
public Dictionary<TKey, TValue> AsDictionary()
{
_readerWriterLock.EnterReadLock();
try
{
return new Dictionary<TKey, TValue>(_cache);
}
finally
{
_readerWriterLock.ExitReadLock();
}
}
public IEnumerable<KeyValuePair<TKey, TValue>> Items()
{
_readerWriterLock.EnterReadLock();
try
{
foreach (var key in _keys)
{
if (_cache.TryGetValue(key, out var value))
{
yield return new KeyValuePair<TKey, TValue>(key, value);
}
}
} finally
{
_readerWriterLock.ExitReadLock();
}
}
public IEnumerator<KeyValuePair<TKey, TValue>> GetEnumerator()
{
return Items().GetEnumerator();
}
}

4
src/Shared/Models/SearchdomainModels.cs
View File

@@ -95,10 +95,12 @@ public struct DateTimedSearchResult(DateTime dateTime, List<ResultItem> results)
}
}
public struct SearchdomainSettings(bool cacheReconciliation = false)
public struct SearchdomainSettings(bool cacheReconciliation = false, int queryCacheSize = 1_000_000)
{
[JsonPropertyName("CacheReconciliation")]
public bool CacheReconciliation { get; set; } = cacheReconciliation;
[JsonPropertyName("QueryCacheSize")]
public int QueryCacheSize { get; set; } = queryCacheSize;
}
public static class MemorySizes

13
src/Shared/Models/SearchdomainResults.cs
View File

@@ -1,4 +1,5 @@
using System.Text.Json.Serialization;
using Shared;
namespace Shared.Models;
@@ -25,7 +26,7 @@ public class SearchdomainDeleteResults : SuccesMessageBaseModel
public required int DeletedEntities { get; set; }
}
public class SearchdomainSearchesResults : SuccesMessageBaseModel
public class SearchdomainQueriesResults : SuccesMessageBaseModel
{
[JsonPropertyName("Searches")]
public required Dictionary<string, DateTimedSearchResult> Searches { get; set; }
@@ -41,10 +42,14 @@ public class SearchdomainSettingsResults : SuccesMessageBaseModel
public required SearchdomainSettings? Settings { get; set; }
}
public class SearchdomainSearchCacheSizeResults : SuccesMessageBaseModel
public class SearchdomainQueryCacheSizeResults : SuccesMessageBaseModel
{
[JsonPropertyName("QueryCacheSizeBytes")]
public required long? QueryCacheSizeBytes { get; set; }
[JsonPropertyName("ElementCount")]
public required int? ElementCount { get; set; }
[JsonPropertyName("ElementMaxCount")]
public required int? ElementMaxCount { get; set; }
[JsonPropertyName("SizeBytes")]
public required long? SizeBytes { get; set; }
}
public class SearchdomainInvalidateCacheResults : SuccesMessageBaseModel {}