diff --git a/src/Client/Client.cs b/src/Client/Client.cs index 290681d..a00537e 100644 --- a/src/Client/Client.cs +++ b/src/Client/Client.cs @@ -47,15 +47,27 @@ public class Client return await FetchUrlAndProcessJson(HttpMethod.Get, url); } - public async Task EntityIndexAsync(List jsonEntity) + public async Task EntityIndexAsync(List jsonEntity, string? sessionId = null, bool? sessionComplete = null) { - return await EntityIndexAsync(JsonSerializer.Serialize(jsonEntity)); + return await EntityIndexAsync(JsonSerializer.Serialize(jsonEntity), sessionId, sessionComplete); } - public async Task EntityIndexAsync(string jsonEntity) + public async Task EntityIndexAsync(string jsonEntity, string? sessionId = null, bool? sessionComplete = null) { var content = new StringContent(jsonEntity, Encoding.UTF8, "application/json"); - return await FetchUrlAndProcessJson(HttpMethod.Put, GetUrl($"{baseUri}", "Entities", []), content); + Dictionary parameters = []; + if (sessionId is not null) parameters.Add("sessionId", sessionId); + if (sessionComplete is not null) parameters.Add("sessionComplete", ((bool)sessionComplete).ToString()); + + return await FetchUrlAndProcessJson( + HttpMethod.Put, + GetUrl( + $"{baseUri}", + $"Entities", + parameters + ), + content + ); } public async Task EntityDeleteAsync(string entityName) diff --git a/src/Indexer/Scripts/example.py b/src/Indexer/Scripts/example.py index 05130c8..687ae7f 100644 --- a/src/Indexer/Scripts/example.py +++ b/src/Indexer/Scripts/example.py @@ -65,6 +65,7 @@ def index_files(toolset: Toolset): jsonEntities.append(jsonEntity) jsonstring = json.dumps(jsonEntities) timer_start = time.time() + # Index all entities in one go. If you need to split it into chunks, use the session attributes! See example_chunked.py result:EntityIndexResult = toolset.Client.EntityIndexAsync(jsonstring).Result timer_end = time.time() toolset.Logger.LogInformation(f"Update was successful: {result.Success} - and was done in {timer_end - timer_start} seconds.") \ No newline at end of file diff --git a/src/Indexer/Scripts/example_chunked.py b/src/Indexer/Scripts/example_chunked.py new file mode 100644 index 0000000..8b08f08 --- /dev/null +++ b/src/Indexer/Scripts/example_chunked.py @@ -0,0 +1,85 @@ +import math +import os +from tools import * +import json +from dataclasses import asdict +import time +import uuid + +example_content = "./Scripts/example_content" +probmethod = "HVEWAvg" +similarityMethod = "Cosine" +example_searchdomain = "example_" + probmethod +example_counter = 0 +models = ["ollama:bge-m3", "ollama:mxbai-embed-large"] +probmethod_datapoint = probmethod +probmethod_entity = probmethod +# Example for a dictionary based weighted average: +# probmethod_datapoint = "DictionaryWeightedAverage:{\"ollama:bge-m3\": 4, \"ollama:mxbai-embed-large\": 1}" +# probmethod_entity = "DictionaryWeightedAverage:{\"title\": 2, \"filename\": 0.1, \"text\": 0.25}" + +def init(toolset: Toolset): + global example_counter + toolset.Logger.LogInformation("{toolset.Name} - init", toolset.Name) + toolset.Logger.LogInformation("This is the init function from the python example script") + toolset.Logger.LogInformation(f"example_counter: {example_counter}") + searchdomainlist:SearchdomainListResults = toolset.Client.SearchdomainListAsync().Result + if example_searchdomain not in searchdomainlist.Searchdomains: + toolset.Client.SearchdomainCreateAsync(example_searchdomain).Result + searchdomainlist = toolset.Client.SearchdomainListAsync().Result + output = "Currently these searchdomains exist:\n" + for searchdomain in searchdomainlist.Searchdomains: + output += f" - {searchdomain}\n" + toolset.Logger.LogInformation(output) + +def update(toolset: Toolset): + global example_counter + toolset.Logger.LogInformation("{toolset.Name} - update", toolset.Name) + toolset.Logger.LogInformation("This is the update function from the python example script") + callbackInfos:ICallbackInfos = toolset.CallbackInfos + if (str(callbackInfos) == "Indexer.Models.RunOnceCallbackInfos"): + toolset.Logger.LogInformation("It was triggered by a runonce call") + elif (str(callbackInfos) == "Indexer.Models.IntervalCallbackInfos"): + toolset.Logger.LogInformation("It was triggered by an interval call") + elif (str(callbackInfos) == "Indexer.Models.ScheduleCallbackInfos"): + toolset.Logger.LogInformation("It was triggered by a schedule call") + elif (str(callbackInfos) == "Indexer.Models.FileUpdateCallbackInfos"): + toolset.Logger.LogInformation("It was triggered by a fileupdate call") + else: + toolset.Logger.LogInformation("It was triggered, but the origin of the call could not be determined") + example_counter += 1 + toolset.Logger.LogInformation(f"example_counter: {example_counter}") + index_files(toolset) + +def index_files(toolset: Toolset): + jsonEntities:list = [] + for filename in os.listdir(example_content): + qualified_filepath = example_content + "/" + filename + with open(qualified_filepath, "r", encoding='utf-8', errors="replace") as file: + title = file.readline() + text = file.read() + datapoints:list = [ + JSONDatapoint("filename", qualified_filepath, probmethod_datapoint, similarityMethod, models), + JSONDatapoint("title", title, probmethod_datapoint, similarityMethod, models), + JSONDatapoint("text", text, probmethod_datapoint, similarityMethod, models) + ] + jsonEntity:dict = asdict(JSONEntity(qualified_filepath, probmethod_entity, example_searchdomain, {}, datapoints)) + jsonEntities.append(jsonEntity) + timer_start = time.time() + chunkSize = 5 + chunkList = chunk_list(jsonEntities, chunkSize) + chunkCount = math.ceil(len(jsonEntities) / chunkSize) + sessionId = uuid.uuid4().hex + print(f"indexing {len(jsonEntities)} entities") + for i, entities in enumerate(chunkList): + isLast = i == chunkCount + print(f'Processing chunk {i} / {len(jsonEntities) / chunkSize}') + jsonstring = json.dumps(entities) + result:EntityIndexResult = toolset.Client.EntityIndexAsync(jsonstring, sessionId, isLast).Result + timer_end = time.time() + toolset.Logger.LogInformation(f"Update was successful: {result.Success} - and was done in {timer_end - timer_start} seconds.") + + +def chunk_list(lst, chunk_size): + for i in range(0, len(lst), chunk_size): + yield lst[i: i + chunk_size] \ No newline at end of file diff --git a/src/Indexer/Scripts/tools.py b/src/Indexer/Scripts/tools.py index aa6005e..983b28c 100644 --- a/src/Indexer/Scripts/tools.py +++ b/src/Indexer/Scripts/tools.py @@ -107,6 +107,8 @@ class Client: # pass async def EntityIndexAsync(jsonEntity:str) -> EntityIndexResult: pass + async def EntityIndexAsync(jsonEntity:str, sessionId:str, sessionComplete:bool) -> EntityIndexResult: + pass async def EntityIndexAsync(searchdomain:str, jsonEntity:str) -> EntityIndexResult: pass async def EntityListAsync(returnEmbeddings:bool = False) -> EntityListResults: diff --git a/src/Server/AIProvider.cs b/src/Server/AIProvider.cs index 5194a4d..d6bf09d 100644 --- a/src/Server/AIProvider.cs +++ b/src/Server/AIProvider.cs @@ -13,7 +13,7 @@ public class AIProvider { private readonly ILogger _logger; private readonly EmbeddingSearchOptions _configuration; - public Dictionary aIProvidersConfiguration; + public Dictionary AiProvidersConfiguration; public AIProvider(ILogger logger, IOptions configuration) { @@ -27,7 +27,7 @@ public class AIProvider } else { - aIProvidersConfiguration = retrievedAiProvidersConfiguration; + AiProvidersConfiguration = retrievedAiProvidersConfiguration; } } @@ -41,7 +41,7 @@ public class AIProvider Uri uri = new(modelUri); string provider = uri.Scheme; string model = uri.AbsolutePath; - AiProvider? aIProvider = aIProvidersConfiguration + AiProvider? aIProvider = AiProvidersConfiguration .FirstOrDefault(x => string.Equals(x.Key.ToLower(), provider.ToLower())) .Value; if (aIProvider is null) @@ -134,7 +134,7 @@ public class AIProvider public string[] GetModels() { - var aIProviders = aIProvidersConfiguration; + var aIProviders = AiProvidersConfiguration; List results = []; foreach (KeyValuePair aIProviderKV in aIProviders) { diff --git a/src/Server/Controllers/EntityController.cs b/src/Server/Controllers/EntityController.cs index d103dd1..55d1d8d 100644 --- a/src/Server/Controllers/EntityController.cs +++ b/src/Server/Controllers/EntityController.cs @@ -14,6 +14,9 @@ public class EntityController : ControllerBase private SearchdomainManager _domainManager; private readonly SearchdomainHelper _searchdomainHelper; private readonly DatabaseHelper _databaseHelper; + private readonly Dictionary _sessions = []; + private readonly object _sessionLock = new(); + private const int SessionTimeoutMinutes = 60; // TODO: remove magic number; add an optional configuration option public EntityController(ILogger logger, IConfiguration config, SearchdomainManager domainManager, SearchdomainHelper searchdomainHelper, DatabaseHelper databaseHelper) { @@ -46,34 +49,34 @@ public class EntityController : ControllerBase (Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomain, _logger); if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new SearchdomainUpdateResults(){Success = false, Message = message}); EntityListResults entityListResults = new() {Results = [], Success = true}; - foreach ((string _, Entity entity) in searchdomain_.entityCache) + foreach ((string _, Entity entity) in searchdomain_.EntityCache) { List attributeResults = []; - foreach (KeyValuePair attribute in entity.attributes) + foreach (KeyValuePair attribute in entity.Attributes) { attributeResults.Add(new AttributeResult() {Name = attribute.Key, Value = attribute.Value}); } List datapointResults = []; - foreach (Datapoint datapoint in entity.datapoints) + foreach (Datapoint datapoint in entity.Datapoints) { if (returnModels) { List embeddingResults = []; - foreach ((string, float[]) embedding in datapoint.embeddings) + foreach ((string, float[]) embedding in datapoint.Embeddings) { embeddingResults.Add(new EmbeddingResult() {Model = embedding.Item1, Embeddings = returnEmbeddings ? embedding.Item2 : []}); } - datapointResults.Add(new DatapointResult() {Name = datapoint.name, ProbMethod = datapoint.probMethod.name, SimilarityMethod = datapoint.similarityMethod.name, Embeddings = embeddingResults}); + datapointResults.Add(new DatapointResult() {Name = datapoint.Name, ProbMethod = datapoint.ProbMethod.Name, SimilarityMethod = datapoint.SimilarityMethod.Name, Embeddings = embeddingResults}); } else { - datapointResults.Add(new DatapointResult() {Name = datapoint.name, ProbMethod = datapoint.probMethod.name, SimilarityMethod = datapoint.similarityMethod.name, Embeddings = null}); + datapointResults.Add(new DatapointResult() {Name = datapoint.Name, ProbMethod = datapoint.ProbMethod.Name, SimilarityMethod = datapoint.SimilarityMethod.Name, Embeddings = null}); } } EntityListResult entityListResult = new() { - Name = entity.name, - ProbMethod = entity.probMethodName, + Name = entity.Name, + ProbMethod = entity.ProbMethodName, Attributes = attributeResults, Datapoints = datapointResults }; @@ -86,31 +89,59 @@ public class EntityController : ControllerBase /// Index entities /// /// - /// Behavior: Creates new entities, but overwrites existing entities that have the same name + /// Behavior: Updates the index using the provided entities. Creates new entities, overwrites existing entities with the same name, and deletes entities that are not part of the index anymore. + /// + /// Can be executed in a single request or in multiple chunks using a (self-defined) session UUID string. + /// + /// For session-based chunk uploads: + /// - Provide sessionId to accumulate entities across multiple requests + /// - Set sessionComplete=true on the final request to finalize and delete entities that are not in the accumulated list + /// - Without sessionId: Missing entities will be deleted from the searchdomain. + /// - Sessions expire after 60 minutes of inactivity (or as otherwise configured in the appsettings) /// /// Entities to index + /// Optional session ID for batch uploads across multiple requests + /// If true, finalizes the session and deletes entities not in the accumulated list [HttpPut("/Entities")] - public async Task> Index([FromBody] List? jsonEntities) + public async Task> Index( + [FromBody] List? jsonEntities, + string? sessionId = null, + bool sessionComplete = false) { try { + if (sessionId is null || string.IsNullOrWhiteSpace(sessionId)) + { + sessionId = Guid.NewGuid().ToString(); // Create a short-lived session + sessionComplete = true; // If no sessionId was set, there is no trackable session. The pseudo-session ends here. + } + // Periodic cleanup of expired sessions + CleanupExpiredEntityIndexSessions(); + EntityIndexSessionData session = GetOrCreateEntityIndexSession(sessionId); + + if (jsonEntities is null && !sessionComplete) + { + return BadRequest(new EntityIndexResult() { Success = false, Message = "jsonEntities can only be null for a complete session" }); + } else if (jsonEntities is null && sessionComplete) + { + await EntityIndexSessionDeleteUnindexedEntities(session); + return Ok(new EntityIndexResult() { Success = true }); + } + + // Standard entity indexing (upsert behavior) List? entities = await _searchdomainHelper.EntitiesFromJSON( _domainManager, _logger, JsonSerializer.Serialize(jsonEntities)); if (entities is not null && jsonEntities is not null) { - List invalidatedSearchdomains = []; - foreach (var jsonEntity in jsonEntities) + session.AccumulatedEntities.AddRange(entities); + + if (sessionComplete) { - string jsonEntityName = jsonEntity.Name; - string jsonEntitySearchdomainName = jsonEntity.Searchdomain; - if (entities.Select(x => x.name == jsonEntityName).Any() - && !invalidatedSearchdomains.Contains(jsonEntitySearchdomainName)) - { - invalidatedSearchdomains.Add(jsonEntitySearchdomainName); - } + await EntityIndexSessionDeleteUnindexedEntities(session); } + return Ok(new EntityIndexResult() { Success = true }); } else @@ -129,6 +160,44 @@ public class EntityController : ControllerBase } + private async Task EntityIndexSessionDeleteUnindexedEntities(EntityIndexSessionData session) + { + var entityGroupsBySearchdomain = session.AccumulatedEntities.GroupBy(e => e.Searchdomain); + + foreach (var entityGroup in entityGroupsBySearchdomain) + { + string searchdomainName = entityGroup.Key; + var entityNamesInRequest = entityGroup.Select(e => e.Name).ToHashSet(); + + (Searchdomain? searchdomain_, int? httpStatusCode, string? message) = + SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomainName, _logger); + + if (searchdomain_ is not null && httpStatusCode is null) // If getting searchdomain was successful + { + var entitiesToDelete = searchdomain_.EntityCache + .Where(kvp => !entityNamesInRequest.Contains(kvp.Value.Name)) + .Select(kvp => kvp.Value) + .ToList(); + + foreach (var entity in entitiesToDelete) + { + searchdomain_.ReconciliateOrInvalidateCacheForDeletedEntity(entity); + await _databaseHelper.RemoveEntity( + [], + _domainManager.Helper, + entity.Name, + searchdomainName); + searchdomain_.EntityCache.TryRemove(entity.Name, out _); + _logger.LogInformation("Deleted entity {entityName} from {searchdomain}", entity.Name, searchdomainName); + } + } + else + { + _logger.LogWarning("Unable to delete entities for searchdomain {searchdomain}", searchdomainName); + } + } + } + /// /// Deletes an entity /// @@ -140,7 +209,7 @@ public class EntityController : ControllerBase (Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomain, _logger); if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new SearchdomainUpdateResults(){Success = false, Message = message}); - Entity? entity_ = SearchdomainHelper.CacheGetEntity(searchdomain_.entityCache, entityName); + Entity? entity_ = SearchdomainHelper.CacheGetEntity(searchdomain_.EntityCache, entityName); if (entity_ is null) { _logger.LogError("Unable to delete the entity {entityName} in {searchdomain} - it was not found under the specified name", [entityName, searchdomain]); @@ -152,10 +221,50 @@ public class EntityController : ControllerBase return Ok(new EntityDeleteResults() {Success = false, Message = "Entity not found"}); } searchdomain_.ReconciliateOrInvalidateCacheForDeletedEntity(entity_); - await _databaseHelper.RemoveEntity([], _domainManager.helper, entityName, searchdomain); + await _databaseHelper.RemoveEntity([], _domainManager.Helper, entityName, searchdomain); - bool success = searchdomain_.entityCache.TryRemove(entityName, out Entity? _); + bool success = searchdomain_.EntityCache.TryRemove(entityName, out Entity? _); return Ok(new EntityDeleteResults() {Success = success}); } + + + private void CleanupExpiredEntityIndexSessions() + { + lock (_sessionLock) + { + var expiredSessions = _sessions + .Where(kvp => (DateTime.UtcNow - kvp.Value.LastInteractionAt).TotalMinutes > SessionTimeoutMinutes) + .Select(kvp => kvp.Key) + .ToList(); + + foreach (var sessionId in expiredSessions) + { + _sessions.Remove(sessionId); + _logger.LogWarning("Removed expired, non-closed session {sessionId}", sessionId); + } + } + } + + private EntityIndexSessionData GetOrCreateEntityIndexSession(string sessionId) + { + lock (_sessionLock) + { + if (!_sessions.TryGetValue(sessionId, out var session)) + { + session = new EntityIndexSessionData(); + _sessions[sessionId] = session; + } else + { + session.LastInteractionAt = DateTime.UtcNow; + } + return session; + } + } } + +public class EntityIndexSessionData +{ + public List AccumulatedEntities { get; set; } = []; + public DateTime LastInteractionAt { get; set; } = DateTime.UtcNow; +} \ No newline at end of file diff --git a/src/Server/Controllers/SearchdomainController.cs b/src/Server/Controllers/SearchdomainController.cs index eecef76..436e32d 100644 --- a/src/Server/Controllers/SearchdomainController.cs +++ b/src/Server/Controllers/SearchdomainController.cs @@ -118,18 +118,18 @@ public class SearchdomainController : ControllerBase Dictionary parameters = new() { {"name", newName}, - {"id", searchdomain_.id} + {"id", searchdomain_.Id} }; - await searchdomain_.helper.ExecuteSQLNonQuery("UPDATE searchdomain set name = @name WHERE id = @id", parameters); + await searchdomain_.Helper.ExecuteSQLNonQuery("UPDATE searchdomain set name = @name WHERE id = @id", parameters); } else { Dictionary parameters = new() { {"name", newName}, {"settings", settings}, - {"id", searchdomain_.id} + {"id", searchdomain_.Id} }; - await searchdomain_.helper.ExecuteSQLNonQuery("UPDATE searchdomain set name = @name, settings = @settings WHERE id = @id", parameters); + await searchdomain_.Helper.ExecuteSQLNonQuery("UPDATE searchdomain set name = @name, settings = @settings WHERE id = @id", parameters); } return Ok(new SearchdomainUpdateResults(){Success = true}); } @@ -143,7 +143,7 @@ public class SearchdomainController : ControllerBase { (Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomain, _logger); if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new SearchdomainUpdateResults(){Success = false, Message = message}); - Dictionary searchCache = searchdomain_.queryCache.AsDictionary(); + Dictionary searchCache = searchdomain_.QueryCache.AsDictionary(); return Ok(new SearchdomainQueriesResults() { Searches = searchCache, Success = true }); } @@ -165,7 +165,7 @@ public class SearchdomainController : ControllerBase { Name = r.Item2, Value = r.Item1, - Attributes = returnAttributes ? (searchdomain_.entityCache[r.Item2]?.attributes ?? null) : null + Attributes = returnAttributes ? (searchdomain_.EntityCache[r.Item2]?.Attributes ?? null) : null })]; return Ok(new EntityQueryResults(){Results = queryResults, Success = true }); } @@ -180,7 +180,7 @@ public class SearchdomainController : ControllerBase { (Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomain, _logger); if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new SearchdomainUpdateResults(){Success = false, Message = message}); - EnumerableLruCache searchCache = searchdomain_.queryCache; + EnumerableLruCache searchCache = searchdomain_.QueryCache; bool containsKey = searchCache.ContainsKey(query); if (containsKey) { @@ -201,7 +201,7 @@ public class SearchdomainController : ControllerBase { (Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomain, _logger); if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new SearchdomainUpdateResults(){Success = false, Message = message}); - EnumerableLruCache searchCache = searchdomain_.queryCache; + EnumerableLruCache searchCache = searchdomain_.QueryCache; bool containsKey = searchCache.ContainsKey(query); if (containsKey) { @@ -222,7 +222,7 @@ public class SearchdomainController : ControllerBase { (Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomain, _logger); if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new SearchdomainUpdateResults(){Success = false, Message = message}); - SearchdomainSettings settings = searchdomain_.settings; + SearchdomainSettings settings = searchdomain_.Settings; return Ok(new SearchdomainSettingsResults() { Settings = settings, Success = true }); } @@ -239,11 +239,11 @@ public class SearchdomainController : ControllerBase Dictionary parameters = new() { {"settings", JsonSerializer.Serialize(request)}, - {"id", searchdomain_.id} + {"id", searchdomain_.Id} }; - await searchdomain_.helper.ExecuteSQLNonQuery("UPDATE searchdomain set settings = @settings WHERE id = @id", parameters); - searchdomain_.settings = request; - searchdomain_.queryCache.Capacity = request.QueryCacheSize; + await searchdomain_.Helper.ExecuteSQLNonQuery("UPDATE searchdomain set settings = @settings WHERE id = @id", parameters); + searchdomain_.Settings = request; + searchdomain_.QueryCache.Capacity = request.QueryCacheSize; return Ok(new SearchdomainUpdateResults(){Success = true}); } @@ -260,8 +260,8 @@ public class SearchdomainController : ControllerBase } (Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomain, _logger); if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new SearchdomainUpdateResults(){Success = false, Message = message}); - int elementCount = searchdomain_.queryCache.Count; - int ElementMaxCount = searchdomain_.settings.QueryCacheSize; + int elementCount = searchdomain_.QueryCache.Count; + int ElementMaxCount = searchdomain_.Settings.QueryCacheSize; return Ok(new SearchdomainQueryCacheSizeResults() { SizeBytes = searchdomain_.GetSearchCacheSize(), ElementCount = elementCount, ElementMaxCount = ElementMaxCount, Success = true }); } @@ -287,7 +287,7 @@ public class SearchdomainController : ControllerBase { (Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomain, _logger); if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new SearchdomainUpdateResults(){Success = false, Message = message}); - long EmbeddingCacheUtilization = DatabaseHelper.GetSearchdomainDatabaseSize(searchdomain_.helper, searchdomain); + long EmbeddingCacheUtilization = DatabaseHelper.GetSearchdomainDatabaseSize(searchdomain_.Helper, searchdomain); return Ok(new SearchdomainGetDatabaseSizeResult() { SearchdomainDatabaseSizeBytes = EmbeddingCacheUtilization, Success = true }); } } diff --git a/src/Server/Controllers/ServerController.cs b/src/Server/Controllers/ServerController.cs index 1fb0594..9f98135 100644 --- a/src/Server/Controllers/ServerController.cs +++ b/src/Server/Controllers/ServerController.cs @@ -58,7 +58,7 @@ public class ServerController : ControllerBase long size = 0; long elementCount = 0; long embeddingsCount = 0; - EnumerableLruCache> embeddingCache = _searchdomainManager.embeddingCache; + EnumerableLruCache> embeddingCache = _searchdomainManager.EmbeddingCache; foreach (KeyValuePair> kv in embeddingCache) { @@ -68,7 +68,7 @@ public class ServerController : ControllerBase elementCount++; embeddingsCount += entry.Keys.Count; } - var sqlHelper = _searchdomainManager.helper; + var sqlHelper = _searchdomainManager.Helper; var databaseTotalSize = DatabaseHelper.GetTotalDatabaseSize(sqlHelper); Task entityCountTask = DatabaseHelper.CountEntities(sqlHelper); long queryCacheUtilization = 0; @@ -82,9 +82,9 @@ public class ServerController : ControllerBase (Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_searchdomainManager, searchdomain, _logger); if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new ServerGetStatsResult(){Success = false, Message = message}); queryCacheUtilization += searchdomain_.GetSearchCacheSize(); - queryCacheElementCount += searchdomain_.queryCache.Count; - queryCacheMaxElementCountAll += searchdomain_.queryCache.Capacity; - queryCacheMaxElementCountLoadedSearchdomainsOnly += searchdomain_.queryCache.Capacity; + queryCacheElementCount += searchdomain_.QueryCache.Count; + queryCacheMaxElementCountAll += searchdomain_.QueryCache.Capacity; + queryCacheMaxElementCountLoadedSearchdomainsOnly += searchdomain_.QueryCache.Capacity; } else { var searchdomainSettings = DatabaseHelper.GetSearchdomainSettings(sqlHelper, searchdomain); diff --git a/src/Server/Datapoint.cs b/src/Server/Datapoint.cs index 91f3062..86710fd 100644 --- a/src/Server/Datapoint.cs +++ b/src/Server/Datapoint.cs @@ -6,36 +6,36 @@ namespace Server; public class Datapoint { - public string name; - public ProbMethod probMethod; - public SimilarityMethod similarityMethod; - public List<(string, float[])> embeddings; - public string hash; - public int id; + public string Name; + public ProbMethod ProbMethod; + public SimilarityMethod SimilarityMethod; + public List<(string, float[])> Embeddings; + public string Hash; + public int Id; public Datapoint(string name, ProbMethodEnum probMethod, SimilarityMethodEnum similarityMethod, string hash, List<(string, float[])> embeddings, int id) { - this.name = name; - this.probMethod = new ProbMethod(probMethod); - this.similarityMethod = new SimilarityMethod(similarityMethod); - this.hash = hash; - this.embeddings = embeddings; - this.id = id; + Name = name; + ProbMethod = new ProbMethod(probMethod); + SimilarityMethod = new SimilarityMethod(similarityMethod); + Hash = hash; + Embeddings = embeddings; + Id = id; } public Datapoint(string name, ProbMethod probMethod, SimilarityMethod similarityMethod, string hash, List<(string, float[])> embeddings, int id) { - this.name = name; - this.probMethod = probMethod; - this.similarityMethod = similarityMethod; - this.hash = hash; - this.embeddings = embeddings; - this.id = id; + Name = name; + ProbMethod = probMethod; + SimilarityMethod = similarityMethod; + Hash = hash; + Embeddings = embeddings; + Id = id; } public float CalcProbability(List<(string, float)> probabilities) { - return probMethod.method(probabilities); + return ProbMethod.Method(probabilities); } public static Dictionary GetEmbeddings(string content, ConcurrentBag models, AIProvider aIProvider, EnumerableLruCache> embeddingCache) diff --git a/src/Server/Entity.cs b/src/Server/Entity.cs index ad5cb1c..5e69939 100644 --- a/src/Server/Entity.cs +++ b/src/Server/Entity.cs @@ -2,12 +2,13 @@ using System.Collections.Concurrent; namespace Server; -public class Entity(Dictionary attributes, Probmethods.probMethodDelegate probMethod, string probMethodName, ConcurrentBag datapoints, string name) +public class Entity(Dictionary attributes, Probmethods.ProbMethodDelegate probMethod, string probMethodName, ConcurrentBag datapoints, string name, string searchdomain) { - public Dictionary attributes = attributes; - public Probmethods.probMethodDelegate probMethod = probMethod; - public string probMethodName = probMethodName; - public ConcurrentBag datapoints = datapoints; - public int id; - public string name = name; + public Dictionary Attributes = attributes; + public Probmethods.ProbMethodDelegate ProbMethod = probMethod; + public string ProbMethodName = probMethodName; + public ConcurrentBag Datapoints = datapoints; + public int Id; + public string Name = name; + public string Searchdomain = searchdomain; } \ No newline at end of file diff --git a/src/Server/HealthChecks/ServerHealthChecks.cs b/src/Server/HealthChecks/ServerHealthChecks.cs index fef432b..bf8cb33 100644 --- a/src/Server/HealthChecks/ServerHealthChecks.cs +++ b/src/Server/HealthChecks/ServerHealthChecks.cs @@ -17,7 +17,7 @@ public class DatabaseHealthCheck : IHealthCheck { try { - DatabaseMigrations.DatabaseGetVersion(_searchdomainManager.helper); + DatabaseMigrations.DatabaseGetVersion(_searchdomainManager.Helper); } catch (Exception ex) { @@ -28,8 +28,8 @@ public class DatabaseHealthCheck : IHealthCheck try { - await _searchdomainManager.helper.ExecuteSQLNonQuery("INSERT INTO settings (name, value) VALUES ('test', 'x');", []); - await _searchdomainManager.helper.ExecuteSQLNonQuery("DELETE FROM settings WHERE name = 'test';", []); + await _searchdomainManager.Helper.ExecuteSQLNonQuery("INSERT INTO settings (name, value) VALUES ('test', 'x');", []); + await _searchdomainManager.Helper.ExecuteSQLNonQuery("DELETE FROM settings WHERE name = 'test';", []); } catch (Exception ex) { diff --git a/src/Server/Helper/DatabaseHelper.cs b/src/Server/Helper/DatabaseHelper.cs index d1113e0..1d72a0e 100644 --- a/src/Server/Helper/DatabaseHelper.cs +++ b/src/Server/Helper/DatabaseHelper.cs @@ -205,7 +205,7 @@ public class DatabaseHelper(ILogger logger) await helper.ExecuteSQLNonQuery("DELETE datapoint.* FROM datapoint JOIN entity ON id_entity = entity.id WHERE entity.name = @name AND entity.id_searchdomain = @searchdomain", parameters); await helper.ExecuteSQLNonQuery("DELETE attribute.* FROM attribute JOIN entity ON id_entity = entity.id WHERE entity.name = @name AND entity.id_searchdomain = @searchdomain", parameters); await helper.ExecuteSQLNonQuery("DELETE FROM entity WHERE name = @name AND entity.id_searchdomain = @searchdomain", parameters); - entityCache.RemoveAll(entity => entity.name == name); + entityCache.RemoveAll(entity => entity.Name == name); } public async Task RemoveAllEntities(SQLHelper helper, string searchdomain) @@ -243,7 +243,7 @@ public class DatabaseHelper(ILogger logger) { "name", name }, { "searchdomain", await GetSearchdomainID(helper, searchdomain)} }; - lock (helper.connection) + lock (helper.Connection) { DbDataReader reader = helper.ExecuteSQLCommand("SELECT COUNT(*) FROM entity WHERE name = @name AND id_searchdomain = @searchdomain", parameters); try @@ -273,7 +273,7 @@ public class DatabaseHelper(ILogger logger) { "name", name }, { "searchdomain", await GetSearchdomainID(helper, searchdomain)} }; - lock (helper.connection) + lock (helper.Connection) { DbDataReader reader = helper.ExecuteSQLCommand("SELECT id FROM entity WHERE name = @name AND id_searchdomain = @searchdomain", parameters); try diff --git a/src/Server/Helper/SQLHelper.cs b/src/Server/Helper/SQLHelper.cs index 3158b77..0972b06 100644 --- a/src/Server/Helper/SQLHelper.cs +++ b/src/Server/Helper/SQLHelper.cs @@ -6,18 +6,18 @@ namespace Server.Helper; public class SQLHelper:IDisposable { - public MySqlConnection connection; - public DbDataReader? dbDataReader; - public MySqlConnectionPoolElement[] connectionPool; - public string connectionString; + public MySqlConnection Connection; + public DbDataReader? DbDataReader; + public MySqlConnectionPoolElement[] ConnectionPool; + public string ConnectionString; public SQLHelper(MySqlConnection connection, string connectionString) { - this.connection = connection; - this.connectionString = connectionString; - connectionPool = new MySqlConnectionPoolElement[50]; - for (int i = 0; i < connectionPool.Length; i++) + Connection = connection; + ConnectionString = connectionString; + ConnectionPool = new MySqlConnectionPoolElement[50]; + for (int i = 0; i < ConnectionPool.Length; i++) { - connectionPool[i] = new MySqlConnectionPoolElement(new MySqlConnection(connectionString), new(1, 1)); + ConnectionPool[i] = new MySqlConnectionPoolElement(new MySqlConnection(connectionString), new(1, 1)); } } @@ -28,24 +28,24 @@ public class SQLHelper:IDisposable public void Dispose() { - connection.Close(); + Connection.Close(); GC.SuppressFinalize(this); } public DbDataReader ExecuteSQLCommand(string query, Dictionary parameters) { - lock (connection) + lock (Connection) { EnsureConnected(); EnsureDbReaderIsClosed(); - using MySqlCommand command = connection.CreateCommand(); + using MySqlCommand command = Connection.CreateCommand(); command.CommandText = query; foreach (KeyValuePair parameter in parameters) { command.Parameters.AddWithValue($"@{parameter.Key}", parameter.Value); } - dbDataReader = command.ExecuteReader(); - return dbDataReader; + DbDataReader = command.ExecuteReader(); + return DbDataReader; } } @@ -55,7 +55,7 @@ public class SQLHelper:IDisposable Func map) { var poolElement = await GetMySqlConnectionPoolElement(); - var connection = poolElement.connection; + var connection = poolElement.Connection; try { await using var command = connection.CreateCommand(); @@ -83,7 +83,7 @@ public class SQLHelper:IDisposable public async Task ExecuteSQLNonQuery(string query, Dictionary parameters) { var poolElement = await GetMySqlConnectionPoolElement(); - var connection = poolElement.connection; + var connection = poolElement.Connection; try { using MySqlCommand command = connection.CreateCommand(); @@ -103,7 +103,7 @@ public class SQLHelper:IDisposable public async Task ExecuteSQLCommandGetInsertedID(string query, Dictionary parameters) { var poolElement = await GetMySqlConnectionPoolElement(); - var connection = poolElement.connection; + var connection = poolElement.Connection; try { using MySqlCommand command = connection.CreateCommand(); @@ -125,7 +125,7 @@ public class SQLHelper:IDisposable public async Task BulkExecuteNonQuery(string sql, IEnumerable parameterSets) { var poolElement = await GetMySqlConnectionPoolElement(); - var connection = poolElement.connection; + var connection = poolElement.Connection; try { int affectedRows = 0; @@ -173,14 +173,14 @@ public class SQLHelper:IDisposable int sleepTime = 10; do { - foreach (var element in connectionPool) + foreach (var element in ConnectionPool) { if (element.Semaphore.Wait(0)) { - if (element.connection.State == ConnectionState.Closed) + if (element.Connection.State == ConnectionState.Closed) { - await element.connection.CloseAsync(); - await element.connection.OpenAsync(); + await element.Connection.CloseAsync(); + await element.Connection.OpenAsync(); } return element; } @@ -194,12 +194,12 @@ public class SQLHelper:IDisposable public bool EnsureConnected() { - if (connection.State != System.Data.ConnectionState.Open) + if (Connection.State != System.Data.ConnectionState.Open) { try { - connection.Close(); - connection.Open(); + Connection.Close(); + Connection.Open(); } catch (Exception ex) { @@ -215,7 +215,7 @@ public class SQLHelper:IDisposable int counter = 0; int sleepTime = 10; int timeout = 5000; - while (!(dbDataReader?.IsClosed ?? true)) + while (!(DbDataReader?.IsClosed ?? true)) { if (counter > timeout / sleepTime) { @@ -230,12 +230,12 @@ public class SQLHelper:IDisposable public struct MySqlConnectionPoolElement { - public MySqlConnection connection; + public MySqlConnection Connection; public SemaphoreSlim Semaphore; public MySqlConnectionPoolElement(MySqlConnection connection, SemaphoreSlim semaphore) { - this.connection = connection; - this.Semaphore = semaphore; + Connection = connection; + Semaphore = semaphore; } } \ No newline at end of file diff --git a/src/Server/Helper/SearchdomainHelper.cs b/src/Server/Helper/SearchdomainHelper.cs index 5a543aa..1a896bc 100644 --- a/src/Server/Helper/SearchdomainHelper.cs +++ b/src/Server/Helper/SearchdomainHelper.cs @@ -39,7 +39,7 @@ public class SearchdomainHelper(ILogger logger, DatabaseHelp { foreach ((string _, Entity entity) in entityCache) { - if (entity.name == name) + if (entity.Name == name) { return entity; } @@ -49,9 +49,9 @@ public class SearchdomainHelper(ILogger logger, DatabaseHelp public async Task?> EntitiesFromJSON(SearchdomainManager searchdomainManager, ILogger logger, string json) { - EnumerableLruCache> embeddingCache = searchdomainManager.embeddingCache; - AIProvider aIProvider = searchdomainManager.aIProvider; - SQLHelper helper = searchdomainManager.helper; + EnumerableLruCache> embeddingCache = searchdomainManager.EmbeddingCache; + AIProvider aIProvider = searchdomainManager.AiProvider; + SQLHelper helper = searchdomainManager.Helper; List? jsonEntities = JsonSerializer.Deserialize>(json); if (jsonEntities is null) @@ -65,7 +65,7 @@ public class SearchdomainHelper(ILogger logger, DatabaseHelp foreach (JSONEntity jSONEntity in jsonEntities) { Dictionary> targetDictionary = toBeCached; - if (searchdomainManager.GetSearchdomain(jSONEntity.Searchdomain).settings.ParallelEmbeddingsPrefetch) + if (searchdomainManager.GetSearchdomain(jSONEntity.Searchdomain).Settings.ParallelEmbeddingsPrefetch) { targetDictionary = toBeCachedParallel; } @@ -126,12 +126,12 @@ public class SearchdomainHelper(ILogger logger, DatabaseHelp { var stopwatch = Stopwatch.StartNew(); - SQLHelper helper = searchdomainManager.helper; + SQLHelper helper = searchdomainManager.Helper; Searchdomain searchdomain = searchdomainManager.GetSearchdomain(jsonEntity.Searchdomain); - int id_searchdomain = searchdomain.id; - ConcurrentDictionary entityCache = searchdomain.entityCache; - AIProvider aIProvider = searchdomain.aIProvider; - EnumerableLruCache> embeddingCache = searchdomain.embeddingCache; + int id_searchdomain = searchdomain.Id; + ConcurrentDictionary entityCache = searchdomain.EntityCache; + AIProvider aIProvider = searchdomain.AiProvider; + EnumerableLruCache> embeddingCache = searchdomain.EmbeddingCache; bool invalidateSearchCache = false; @@ -140,15 +140,15 @@ public class SearchdomainHelper(ILogger logger, DatabaseHelp if (hasEntity && preexistingEntity is not null) { - int preexistingEntityID = preexistingEntity.id; + int preexistingEntityID = preexistingEntity.Id; Dictionary attributes = jsonEntity.Attributes; // Attribute - get changes - List<(string attribute, string newValue, int entityId)> updatedAttributes = new(preexistingEntity.attributes.Count); - List<(string attribute, int entityId)> deletedAttributes = new(preexistingEntity.attributes.Count); + List<(string attribute, string newValue, int entityId)> updatedAttributes = new(preexistingEntity.Attributes.Count); + List<(string attribute, int entityId)> deletedAttributes = new(preexistingEntity.Attributes.Count); List<(string attributeKey, string attribute, int entityId)> addedAttributes = new(jsonEntity.Attributes.Count); - foreach (KeyValuePair attributesKV in preexistingEntity.attributes) //.ToList()) + foreach (KeyValuePair attributesKV in preexistingEntity.Attributes) //.ToList()) { string oldAttributeKey = attributesKV.Key; string oldAttribute = attributesKV.Value; @@ -166,7 +166,7 @@ public class SearchdomainHelper(ILogger logger, DatabaseHelp { string newAttributeKey = attributesKV.Key; string newAttribute = attributesKV.Value; - bool preexistingHasAttribute = preexistingEntity.attributes.TryGetValue(newAttributeKey, out string? preexistingAttribute); + bool preexistingHasAttribute = preexistingEntity.Attributes.TryGetValue(newAttributeKey, out string? preexistingAttribute); if (!preexistingHasAttribute) { // Attribute - New @@ -175,54 +175,54 @@ public class SearchdomainHelper(ILogger logger, DatabaseHelp } if (updatedAttributes.Count != 0 || deletedAttributes.Count != 0 || addedAttributes.Count != 0) - _logger.LogDebug("EntityFromJSON - Updating existing entity. name: {name}, updatedAttributes: {updatedAttributes}, deletedAttributes: {deletedAttributes}, addedAttributes: {addedAttributes}", [preexistingEntity.name, updatedAttributes.Count, deletedAttributes.Count, addedAttributes.Count]); + _logger.LogDebug("EntityFromJSON - Updating existing entity. name: {name}, updatedAttributes: {updatedAttributes}, deletedAttributes: {deletedAttributes}, addedAttributes: {addedAttributes}", [preexistingEntity.Name, updatedAttributes.Count, deletedAttributes.Count, addedAttributes.Count]); // Attribute - apply changes if (updatedAttributes.Count != 0) { // Update await DatabaseHelper.DatabaseUpdateAttributes(helper, updatedAttributes); - lock (preexistingEntity.attributes) + lock (preexistingEntity.Attributes) { - updatedAttributes.ForEach(attribute => preexistingEntity.attributes[attribute.attribute] = attribute.newValue); + updatedAttributes.ForEach(attribute => preexistingEntity.Attributes[attribute.attribute] = attribute.newValue); } } if (deletedAttributes.Count != 0) { // Delete await DatabaseHelper.DatabaseDeleteAttributes(helper, deletedAttributes); - lock (preexistingEntity.attributes) + lock (preexistingEntity.Attributes) { - deletedAttributes.ForEach(attribute => preexistingEntity.attributes.Remove(attribute.attribute)); + deletedAttributes.ForEach(attribute => preexistingEntity.Attributes.Remove(attribute.attribute)); } } if (addedAttributes.Count != 0) { // Insert await DatabaseHelper.DatabaseInsertAttributes(helper, addedAttributes); - lock (preexistingEntity.attributes) + lock (preexistingEntity.Attributes) { - addedAttributes.ForEach(attribute => preexistingEntity.attributes.Add(attribute.attributeKey, attribute.attribute)); + addedAttributes.ForEach(attribute => preexistingEntity.Attributes.Add(attribute.attributeKey, attribute.attribute)); } } // Datapoint - get changes - List deletedDatapointInstances = new(preexistingEntity.datapoints.Count); - List deletedDatapoints = new(preexistingEntity.datapoints.Count); - List<(string datapointName, int entityId, JSONDatapoint jsonDatapoint, string hash)> updatedDatapointsText = new(preexistingEntity.datapoints.Count); - List<(string datapointName, string probMethod, string similarityMethod, int entityId, JSONDatapoint jsonDatapoint)> updatedDatapointsNonText = new(preexistingEntity.datapoints.Count); + List deletedDatapointInstances = new(preexistingEntity.Datapoints.Count); + List deletedDatapoints = new(preexistingEntity.Datapoints.Count); + List<(string datapointName, int entityId, JSONDatapoint jsonDatapoint, string hash)> updatedDatapointsText = new(preexistingEntity.Datapoints.Count); + List<(string datapointName, string probMethod, string similarityMethod, int entityId, JSONDatapoint jsonDatapoint)> updatedDatapointsNonText = new(preexistingEntity.Datapoints.Count); List createdDatapointInstances = []; List<(string name, ProbMethodEnum probmethod_embedding, SimilarityMethodEnum similarityMethod, string hash, Dictionary embeddings, JSONDatapoint datapoint)> createdDatapoints = new(jsonEntity.Datapoints.Length); - foreach (Datapoint datapoint_ in preexistingEntity.datapoints.ToList()) + foreach (Datapoint datapoint_ in preexistingEntity.Datapoints.ToList()) { Datapoint datapoint = datapoint_; // To enable replacing the datapoint reference as foreach iterators cannot be overwritten - JSONDatapoint? newEntityDatapoint = jsonEntity.Datapoints.FirstOrDefault(x => x.Name == datapoint.name); + JSONDatapoint? newEntityDatapoint = jsonEntity.Datapoints.FirstOrDefault(x => x.Name == datapoint.Name); bool newEntityHasDatapoint = newEntityDatapoint is not null; if (!newEntityHasDatapoint) { // Datapoint - Deleted deletedDatapointInstances.Add(datapoint); - deletedDatapoints.Add(datapoint.name); + deletedDatapoints.Add(datapoint.Name); invalidateSearchCache = true; } else { @@ -231,7 +231,7 @@ public class SearchdomainHelper(ILogger logger, DatabaseHelp newEntityDatapoint is not null && newEntityDatapoint.Text is not null && hash is not null - && hash != datapoint.hash) + && hash != datapoint.Hash) { // Datapoint - Updated (text) updatedDatapointsText.Add(new() @@ -245,8 +245,8 @@ public class SearchdomainHelper(ILogger logger, DatabaseHelp } if ( newEntityDatapoint is not null - && (newEntityDatapoint.Probmethod_embedding != datapoint.probMethod.probMethodEnum - || newEntityDatapoint.SimilarityMethod != datapoint.similarityMethod.similarityMethodEnum)) + && (newEntityDatapoint.Probmethod_embedding != datapoint.ProbMethod.ProbMethodEnum + || newEntityDatapoint.SimilarityMethod != datapoint.SimilarityMethod.SimilarityMethodEnum)) { // Datapoint - Updated (probmethod or similaritymethod) updatedDatapointsNonText.Add(new() @@ -264,7 +264,7 @@ public class SearchdomainHelper(ILogger logger, DatabaseHelp foreach (JSONDatapoint jsonDatapoint in jsonEntity.Datapoints) { - bool oldEntityHasDatapoint = preexistingEntity.datapoints.Any(x => x.name == jsonDatapoint.Name); + bool oldEntityHasDatapoint = preexistingEntity.Datapoints.Any(x => x.Name == jsonDatapoint.Name); if (!oldEntityHasDatapoint) { // Datapoint - New @@ -290,13 +290,13 @@ public class SearchdomainHelper(ILogger logger, DatabaseHelp if (deletedDatapointInstances.Count != 0 || createdDatapoints.Count != 0 || addedAttributes.Count != 0 || updatedDatapointsNonText.Count != 0) _logger.LogDebug( "EntityFromJSON - Updating existing entity. name: {name}, deletedDatapointInstances: {deletedDatapointInstances}, createdDatapoints: {createdDatapoints}, addedAttributes: {addedAttributes}, updatedDatapointsNonText: {updatedDatapointsNonText}", - [preexistingEntity.name, deletedDatapointInstances.Count, createdDatapoints.Count, addedAttributes.Count, updatedDatapointsNonText.Count]); + [preexistingEntity.Name, deletedDatapointInstances.Count, createdDatapoints.Count, addedAttributes.Count, updatedDatapointsNonText.Count]); // Datapoint - apply changes // Deleted if (deletedDatapointInstances.Count != 0) { await DatabaseHelper.DatabaseDeleteEmbeddingsAndDatapoints(helper, deletedDatapoints, (int)preexistingEntityID); - preexistingEntity.datapoints = [.. preexistingEntity.datapoints + preexistingEntity.Datapoints = [.. preexistingEntity.Datapoints .Where(x => !deletedDatapointInstances.Contains(x) ) @@ -306,7 +306,7 @@ public class SearchdomainHelper(ILogger logger, DatabaseHelp if (createdDatapoints.Count != 0) { List datapoint = await DatabaseInsertDatapointsWithEmbeddings(helper, searchdomain, [.. createdDatapoints.Select(element => (element.datapoint, element.hash))], (int)preexistingEntityID, id_searchdomain); - datapoint.ForEach(x => preexistingEntity.datapoints.Add(x)); + datapoint.ForEach(x => preexistingEntity.Datapoints.Add(x)); } // Datapoint - Updated (text) if (updatedDatapointsText.Count != 0) @@ -317,14 +317,14 @@ public class SearchdomainHelper(ILogger logger, DatabaseHelp .Select(d => d.datapointName) .ToHashSet(); var newBag = new ConcurrentBag( - preexistingEntity.datapoints - .Where(x => !namesToRemove.Contains(x.name)) + preexistingEntity.Datapoints + .Where(x => !namesToRemove.Contains(x.Name)) ); - preexistingEntity.datapoints = newBag; + preexistingEntity.Datapoints = newBag; // Insert into database List datapoints = await DatabaseInsertDatapointsWithEmbeddings(helper, searchdomain, [.. updatedDatapointsText.Select(element => (datapoint: element.jsonDatapoint, hash: element.hash))], (int)preexistingEntityID, id_searchdomain); // Insert into datapoints - datapoints.ForEach(datapoint => preexistingEntity.datapoints.Add(datapoint)); + datapoints.ForEach(datapoint => preexistingEntity.Datapoints.Add(datapoint)); } // Datapoint - Updated (probmethod or similaritymethod) if (updatedDatapointsNonText.Count != 0) @@ -336,9 +336,9 @@ public class SearchdomainHelper(ILogger logger, DatabaseHelp ); updatedDatapointsNonText.ForEach(element => { - Datapoint preexistingDatapoint = preexistingEntity.datapoints.First(x => x.name == element.datapointName); - preexistingDatapoint.probMethod = new(element.jsonDatapoint.Probmethod_embedding); - preexistingDatapoint.similarityMethod = new(element.jsonDatapoint.SimilarityMethod); + Datapoint preexistingDatapoint = preexistingEntity.Datapoints.First(x => x.Name == element.datapointName); + preexistingDatapoint.ProbMethod = new(element.jsonDatapoint.Probmethod_embedding); + preexistingDatapoint.SimilarityMethod = new(element.jsonDatapoint.SimilarityMethod); }); } @@ -368,7 +368,7 @@ public class SearchdomainHelper(ILogger logger, DatabaseHelp var insertAttributesTask = DatabaseHelper.DatabaseInsertAttributes(helper, toBeInsertedAttributes); List<(JSONDatapoint datapoint, string hash)> toBeInsertedDatapoints = []; - ConcurrentBag usedModels = searchdomain.modelsInUse; + ConcurrentBag usedModels = searchdomain.ModelsInUse; foreach (JSONDatapoint jsonDatapoint in jsonEntity.Datapoints) { string hash = Convert.ToBase64String(SHA256.HashData(Encoding.UTF8.GetBytes(jsonDatapoint.Text))); @@ -389,9 +389,9 @@ public class SearchdomainHelper(ILogger logger, DatabaseHelp List datapoints = await DatabaseInsertDatapointsWithEmbeddings(helper, searchdomain, toBeInsertedDatapoints, id_entity, id_searchdomain); var probMethod = Probmethods.GetMethod(jsonEntity.Probmethod) ?? throw new ProbMethodNotFoundException(jsonEntity.Probmethod); - Entity entity = new(jsonEntity.Attributes, probMethod, jsonEntity.Probmethod.ToString(), new(datapoints), jsonEntity.Name) + Entity entity = new(jsonEntity.Attributes, probMethod, jsonEntity.Probmethod.ToString(), [.. datapoints], jsonEntity.Name, jsonEntity.Searchdomain) { - id = id_entity + Id = id_entity }; entityCache[jsonEntity.Name] = entity; @@ -412,16 +412,16 @@ public class SearchdomainHelper(ILogger logger, DatabaseHelp toBeInsertedDatapoints.Add(new() { - name = datapoint.name, - probmethod_embedding = datapoint.probMethod.probMethodEnum, - similarityMethod = datapoint.similarityMethod.similarityMethodEnum, + name = datapoint.Name, + probmethod_embedding = datapoint.ProbMethod.ProbMethodEnum, + similarityMethod = datapoint.SimilarityMethod.SimilarityMethodEnum, hash = value.hash }); - foreach ((string, float[]) embedding in datapoint.embeddings) + foreach ((string, float[]) embedding in datapoint.Embeddings) { toBeInsertedEmbeddings.Add(new() { - id_datapoint = datapoint.id, + id_datapoint = datapoint.Id, model = embedding.Item1, embedding = BytesFromFloatArray(embedding.Item2) }); @@ -444,7 +444,7 @@ public class SearchdomainHelper(ILogger logger, DatabaseHelp Datapoint datapoint = await BuildDatapointFromJsonDatapoint(jsonDatapoint, id_entity, searchdomain, hash); int id_datapoint = await DatabaseHelper.DatabaseInsertDatapoint(helper, jsonDatapoint.Name, jsonDatapoint.Probmethod_embedding, jsonDatapoint.SimilarityMethod, hash, id_entity); // TODO make this a bulk add action to reduce number of queries List<(string model, byte[] embedding)> data = []; - foreach ((string, float[]) embedding in datapoint.embeddings) + foreach ((string, float[]) embedding in datapoint.Embeddings) { data.Add((embedding.Item1, BytesFromFloatArray(embedding.Item2))); } @@ -463,11 +463,11 @@ public class SearchdomainHelper(ILogger logger, DatabaseHelp { throw new Exception("jsonDatapoint.Text must not be null at this point"); } - SQLHelper helper = searchdomain.helper; - EnumerableLruCache> embeddingCache = searchdomain.embeddingCache; + SQLHelper helper = searchdomain.Helper; + EnumerableLruCache> embeddingCache = searchdomain.EmbeddingCache; hash ??= Convert.ToBase64String(SHA256.HashData(Encoding.UTF8.GetBytes(jsonDatapoint.Text))); int id = await DatabaseHelper.DatabaseInsertDatapoint(helper, jsonDatapoint.Name, jsonDatapoint.Probmethod_embedding, jsonDatapoint.SimilarityMethod, hash, entityId); - Dictionary embeddings = Datapoint.GetEmbeddings(jsonDatapoint.Text, [.. jsonDatapoint.Model], searchdomain.aIProvider, embeddingCache); + Dictionary embeddings = Datapoint.GetEmbeddings(jsonDatapoint.Text, [.. jsonDatapoint.Model], searchdomain.AiProvider, embeddingCache); var probMethod_embedding = new ProbMethod(jsonDatapoint.Probmethod_embedding) ?? throw new ProbMethodNotFoundException(jsonDatapoint.Probmethod_embedding); var similarityMethod = new SimilarityMethod(jsonDatapoint.SimilarityMethod) ?? throw new SimilarityMethodNotFoundException(jsonDatapoint.SimilarityMethod); return new Datapoint(jsonDatapoint.Name, probMethod_embedding, similarityMethod, hash, [.. embeddings.Select(kv => (kv.Key, kv.Value))], id); diff --git a/src/Server/Probmethods.cs b/src/Server/Probmethods.cs index 8794756..4210d41 100644 --- a/src/Server/Probmethods.cs +++ b/src/Server/Probmethods.cs @@ -6,29 +6,29 @@ namespace Server; public class ProbMethod { - public Probmethods.probMethodDelegate method; - public ProbMethodEnum probMethodEnum; - public string name; + public Probmethods.ProbMethodDelegate Method; + public ProbMethodEnum ProbMethodEnum; + public string Name; public ProbMethod(ProbMethodEnum probMethodEnum) { - this.probMethodEnum = probMethodEnum; - this.name = probMethodEnum.ToString(); - Probmethods.probMethodDelegate? probMethod = Probmethods.GetMethod(name) ?? throw new ProbMethodNotFoundException(probMethodEnum); - method = probMethod; + this.ProbMethodEnum = probMethodEnum; + this.Name = probMethodEnum.ToString(); + Probmethods.ProbMethodDelegate? probMethod = Probmethods.GetMethod(Name) ?? throw new ProbMethodNotFoundException(probMethodEnum); + Method = probMethod; } } public static class Probmethods { - public delegate float probMethodProtoDelegate(List<(string, float)> list, string parameters); - public delegate float probMethodDelegate(List<(string, float)> list); - public static readonly Dictionary probMethods; + public delegate float ProbMethodProtoDelegate(List<(string, float)> list, string parameters); + public delegate float ProbMethodDelegate(List<(string, float)> list); + public static readonly Dictionary ProbMethods; static Probmethods() { - probMethods = new Dictionary + ProbMethods = new Dictionary { [ProbMethodEnum.Mean] = Mean, [ProbMethodEnum.HarmonicMean] = HarmonicMean, @@ -41,12 +41,12 @@ public static class Probmethods }; } - public static probMethodDelegate? GetMethod(ProbMethodEnum probMethodEnum) + public static ProbMethodDelegate? GetMethod(ProbMethodEnum probMethodEnum) { return GetMethod(probMethodEnum.ToString()); } - public static probMethodDelegate? GetMethod(string name) + public static ProbMethodDelegate? GetMethod(string name) { string methodName = name; string? jsonArg = ""; @@ -63,7 +63,7 @@ public static class Probmethods methodName ); - if (!probMethods.TryGetValue(probMethodEnum, out probMethodProtoDelegate? method)) + if (!ProbMethods.TryGetValue(probMethodEnum, out ProbMethodProtoDelegate? method)) { return null; } diff --git a/src/Server/Searchdomain.cs b/src/Server/Searchdomain.cs index 5c0a0fe..b892e32 100644 --- a/src/Server/Searchdomain.cs +++ b/src/Server/Searchdomain.cs @@ -15,33 +15,33 @@ public class Searchdomain { private readonly string _connectionString; private readonly string _provider; - public AIProvider aIProvider; - public string searchdomain; - public int id; - public SearchdomainSettings settings; - public EnumerableLruCache queryCache; // Key: query, Value: Search results for that query (with timestamp) - public ConcurrentDictionary entityCache; - public ConcurrentBag modelsInUse; - public EnumerableLruCache> embeddingCache; - public SQLHelper helper; + public AIProvider AiProvider; + public string SearchdomainName; + public int Id; + public SearchdomainSettings Settings; + public EnumerableLruCache QueryCache; // Key: query, Value: Search results for that query (with timestamp) + public ConcurrentDictionary EntityCache; + public ConcurrentBag ModelsInUse; + public EnumerableLruCache> EmbeddingCache; + public SQLHelper Helper; private readonly ILogger _logger; public Searchdomain(string searchdomain, string connectionString, SQLHelper sqlHelper, AIProvider aIProvider, EnumerableLruCache> embeddingCache, ILogger logger, string provider = "sqlserver", bool runEmpty = false) { _connectionString = connectionString; _provider = provider.ToLower(); - this.searchdomain = searchdomain; - this.aIProvider = aIProvider; - this.embeddingCache = embeddingCache; + this.SearchdomainName = searchdomain; + this.AiProvider = aIProvider; + this.EmbeddingCache = embeddingCache; this._logger = logger; - entityCache = []; - helper = sqlHelper; - settings = GetSettings(); - queryCache = new(settings.QueryCacheSize); - modelsInUse = []; // To make the compiler shut up - it is set in UpdateSearchDomain() don't worry // yeah, about that... + EntityCache = []; + Helper = sqlHelper; + Settings = GetSettings(); + QueryCache = new(Settings.QueryCacheSize); + ModelsInUse = []; // To make the compiler shut up - it is set in UpdateSearchDomain() don't worry // yeah, about that... if (!runEmpty) { - id = GetID().Result; + Id = GetID().Result; UpdateEntityCache(); } } @@ -51,9 +51,9 @@ public class Searchdomain InvalidateSearchCache(); Dictionary parametersIDSearchdomain = new() { - ["id"] = this.id + ["id"] = this.Id }; - DbDataReader embeddingReader = helper.ExecuteSQLCommand("SELECT id, id_datapoint, model, embedding FROM embedding WHERE id_searchdomain = @id", parametersIDSearchdomain); + DbDataReader embeddingReader = Helper.ExecuteSQLCommand("SELECT id, id_datapoint, model, embedding FROM embedding WHERE id_searchdomain = @id", parametersIDSearchdomain); Dictionary> embedding_unassigned = []; try { @@ -90,7 +90,7 @@ public class Searchdomain embeddingReader.Close(); } - DbDataReader datapointReader = helper.ExecuteSQLCommand("SELECT d.id, d.id_entity, d.name, d.probmethod_embedding, d.similaritymethod, d.hash FROM datapoint d JOIN entity ent ON d.id_entity = ent.id JOIN searchdomain s ON ent.id_searchdomain = s.id WHERE s.id = @id", parametersIDSearchdomain); + DbDataReader datapointReader = Helper.ExecuteSQLCommand("SELECT d.id, d.id_entity, d.name, d.probmethod_embedding, d.similaritymethod, d.hash FROM datapoint d JOIN entity ent ON d.id_entity = ent.id JOIN searchdomain s ON ent.id_searchdomain = s.id WHERE s.id = @id", parametersIDSearchdomain); Dictionary> datapoint_unassigned = []; try { @@ -127,7 +127,7 @@ public class Searchdomain datapointReader.Close(); } - DbDataReader attributeReader = helper.ExecuteSQLCommand("SELECT a.id, a.id_entity, a.attribute, a.value FROM attribute a JOIN entity ent ON a.id_entity = ent.id JOIN searchdomain s ON ent.id_searchdomain = s.id WHERE s.id = @id", parametersIDSearchdomain); + DbDataReader attributeReader = Helper.ExecuteSQLCommand("SELECT a.id, a.id_entity, a.attribute, a.value FROM attribute a JOIN entity ent ON a.id_entity = ent.id JOIN searchdomain s ON ent.id_searchdomain = s.id WHERE s.id = @id", parametersIDSearchdomain); Dictionary> attributes_unassigned = []; try { @@ -149,8 +149,8 @@ public class Searchdomain attributeReader.Close(); } - entityCache = []; - DbDataReader entityReader = helper.ExecuteSQLCommand("SELECT entity.id, name, probmethod FROM entity WHERE id_searchdomain=@id", parametersIDSearchdomain); + EntityCache = []; + DbDataReader entityReader = Helper.ExecuteSQLCommand("SELECT entity.id, name, probmethod FROM entity WHERE id_searchdomain=@id", parametersIDSearchdomain); try { while (entityReader.Read()) @@ -163,26 +163,26 @@ public class Searchdomain { attributes = []; } - Probmethods.probMethodDelegate? probmethod = Probmethods.GetMethod(probmethodString); + Probmethods.ProbMethodDelegate? probmethod = Probmethods.GetMethod(probmethodString); if (datapoint_unassigned.TryGetValue(id, out ConcurrentBag? datapoints) && probmethod is not null) { - Entity entity = new(attributes, probmethod, probmethodString, datapoints, name) + Entity entity = new(attributes, probmethod, probmethodString, datapoints, name, SearchdomainName) { - id = id + Id = id }; - entityCache[name] = entity; + EntityCache[name] = entity; } } } finally { entityReader.Close(); } - modelsInUse = GetModels(entityCache); + ModelsInUse = GetModels(EntityCache); } public List<(float, string)> Search(string query, int? topN = null) { - if (queryCache.TryGetValue(query, out DateTimedSearchResult cachedResult)) + if (QueryCache.TryGetValue(query, out DateTimedSearchResult cachedResult)) { cachedResult.AccessDateTimes.Add(DateTime.Now); return [.. cachedResult.Results.Select(r => (r.Score, r.Name))]; @@ -191,9 +191,9 @@ public class Searchdomain Dictionary queryEmbeddings = GetQueryEmbeddings(query); List<(float, string)> result = []; - foreach ((string name, Entity entity) in entityCache) + foreach ((string name, Entity entity) in EntityCache) { - result.Add((EvaluateEntityAgainstQueryEmbeddings(entity, queryEmbeddings), entity.name)); + result.Add((EvaluateEntityAgainstQueryEmbeddings(entity, queryEmbeddings), entity.Name)); } IEnumerable<(float, string)> sortedResults = result.OrderByDescending(s => s.Item1); if (topN is not null) @@ -205,26 +205,26 @@ public class Searchdomain [.. sortedResults.Select(r => new ResultItem(r.Item1, r.Item2 ))] ); - queryCache.Set(query, new DateTimedSearchResult(DateTime.Now, searchResult)); + QueryCache.Set(query, new DateTimedSearchResult(DateTime.Now, searchResult)); return results; } public Dictionary GetQueryEmbeddings(string query) { - bool hasQuery = embeddingCache.TryGetValue(query, out Dictionary? queryEmbeddings); - bool allModelsInQuery = queryEmbeddings is not null && modelsInUse.All(model => queryEmbeddings.ContainsKey(model)); + bool hasQuery = EmbeddingCache.TryGetValue(query, out Dictionary? queryEmbeddings); + bool allModelsInQuery = queryEmbeddings is not null && ModelsInUse.All(model => queryEmbeddings.ContainsKey(model)); if (!(hasQuery && allModelsInQuery) || queryEmbeddings is null) { - queryEmbeddings = Datapoint.GetEmbeddings(query, modelsInUse, aIProvider, embeddingCache); - if (!embeddingCache.TryGetValue(query, out var embeddingCacheForCurrentQuery)) + queryEmbeddings = Datapoint.GetEmbeddings(query, ModelsInUse, AiProvider, EmbeddingCache); + if (!EmbeddingCache.TryGetValue(query, out var embeddingCacheForCurrentQuery)) { - embeddingCache.Set(query, queryEmbeddings); + EmbeddingCache.Set(query, queryEmbeddings); } else // embeddingCache already has an entry for this query, so the missing model-embedding pairs have to be filled in { foreach (KeyValuePair kvp in queryEmbeddings) // kvp.Key = model, kvp.Value = embedding { - if (!embeddingCache.TryGetValue(kvp.Key, out var _)) + if (!EmbeddingCache.TryGetValue(kvp.Key, out var _)) { embeddingCacheForCurrentQuery[kvp.Key] = kvp.Value; } @@ -236,25 +236,25 @@ public class Searchdomain public void UpdateModelsInUse() { - modelsInUse = GetModels(entityCache); + ModelsInUse = GetModels(EntityCache); } private static float EvaluateEntityAgainstQueryEmbeddings(Entity entity, Dictionary queryEmbeddings) { List<(string, float)> datapointProbs = []; - foreach (Datapoint datapoint in entity.datapoints) + foreach (Datapoint datapoint in entity.Datapoints) { - SimilarityMethod similarityMethod = datapoint.similarityMethod; + SimilarityMethod similarityMethod = datapoint.SimilarityMethod; List<(string, float)> list = []; - foreach ((string, float[]) embedding in datapoint.embeddings) + foreach ((string, float[]) embedding in datapoint.Embeddings) { string key = embedding.Item1; - float value = similarityMethod.method(queryEmbeddings[embedding.Item1], embedding.Item2); + float value = similarityMethod.Method(queryEmbeddings[embedding.Item1], embedding.Item2); list.Add((key, value)); } - datapointProbs.Add((datapoint.name, datapoint.probMethod.method(list))); + datapointProbs.Add((datapoint.Name, datapoint.ProbMethod.Method(list))); } - return entity.probMethod(datapointProbs); + return entity.ProbMethod(datapointProbs); } public static ConcurrentBag GetModels(ConcurrentDictionary entities) @@ -265,9 +265,9 @@ public class Searchdomain Entity entity = element.Value; lock (entity) { - foreach (Datapoint datapoint in entity.datapoints) + foreach (Datapoint datapoint in entity.Datapoints) { - foreach ((string, float[]) tuple in datapoint.embeddings) + foreach ((string, float[]) tuple in datapoint.Embeddings) { string model = tuple.Item1; if (!result.Contains(model)) @@ -285,21 +285,21 @@ public class Searchdomain { Dictionary parameters = new() { - { "name", this.searchdomain } + { "name", this.SearchdomainName } }; - return (await helper.ExecuteQueryAsync("SELECT id from searchdomain WHERE name = @name", parameters, x => x.GetInt32(0))).First(); + return (await Helper.ExecuteQueryAsync("SELECT id from searchdomain WHERE name = @name", parameters, x => x.GetInt32(0))).First(); } public SearchdomainSettings GetSettings() { - return DatabaseHelper.GetSearchdomainSettings(helper, searchdomain); + return DatabaseHelper.GetSearchdomainSettings(Helper, SearchdomainName); } public void ReconciliateOrInvalidateCacheForNewOrUpdatedEntity(Entity entity) { - if (settings.CacheReconciliation) + if (Settings.CacheReconciliation) { - foreach (var element in queryCache) + foreach (var element in QueryCache) { string query = element.Key; DateTimedSearchResult searchResult = element.Value; @@ -307,9 +307,9 @@ public class Searchdomain Dictionary queryEmbeddings = GetQueryEmbeddings(query); float evaluationResult = EvaluateEntityAgainstQueryEmbeddings(entity, queryEmbeddings); - searchResult.Results.RemoveAll(x => x.Name == entity.name); // If entity already exists in that results list: remove it. + searchResult.Results.RemoveAll(x => x.Name == entity.Name); // If entity already exists in that results list: remove it. - ResultItem newItem = new(evaluationResult, entity.name); + ResultItem newItem = new(evaluationResult, entity.Name); int index = searchResult.Results.BinarySearch( newItem, Comparer.Create((a, b) => b.Score.CompareTo(a.Score)) // Invert searching order @@ -327,13 +327,13 @@ public class Searchdomain public void ReconciliateOrInvalidateCacheForDeletedEntity(Entity entity) { - if (settings.CacheReconciliation) + if (Settings.CacheReconciliation) { - foreach (KeyValuePair element in queryCache) + foreach (KeyValuePair element in QueryCache) { string query = element.Key; DateTimedSearchResult searchResult = element.Value; - searchResult.Results.RemoveAll(x => x.Name == entity.name); + searchResult.Results.RemoveAll(x => x.Name == entity.Name); } } else @@ -344,13 +344,13 @@ public class Searchdomain public void InvalidateSearchCache() { - queryCache = new(settings.QueryCacheSize); + QueryCache = new(Settings.QueryCacheSize); } public long GetSearchCacheSize() { long EmbeddingCacheUtilization = 0; - foreach (var entry in queryCache) + foreach (var entry in QueryCache) { EmbeddingCacheUtilization += sizeof(int); // string length prefix EmbeddingCacheUtilization += entry.Key.Length * sizeof(char); // string characters diff --git a/src/Server/SearchdomainManager.cs b/src/Server/SearchdomainManager.cs index 266ee93..2fdcf6e 100644 --- a/src/Server/SearchdomainManager.cs +++ b/src/Server/SearchdomainManager.cs @@ -15,50 +15,50 @@ namespace Server; public class SearchdomainManager : IDisposable { - private Dictionary searchdomains = []; + private Dictionary _searchdomains = []; private readonly ILogger _logger; private readonly EmbeddingSearchOptions _options; - public readonly AIProvider aIProvider; + public readonly AIProvider AiProvider; private readonly DatabaseHelper _databaseHelper; private readonly string connectionString; - private MySqlConnection connection; - public SQLHelper helper; - public EnumerableLruCache> embeddingCache; + private MySqlConnection _connection; + public SQLHelper Helper; + public EnumerableLruCache> EmbeddingCache; public long EmbeddingCacheMaxCount; - private bool disposed = false; + private bool _disposed = false; public SearchdomainManager(ILogger logger, IOptions options, AIProvider aIProvider, DatabaseHelper databaseHelper) { _logger = logger; _options = options.Value; - this.aIProvider = aIProvider; + this.AiProvider = aIProvider; _databaseHelper = databaseHelper; EmbeddingCacheMaxCount = _options.Cache.CacheTopN; if (options.Value.Cache.StoreEmbeddingCache) { var stopwatch = Stopwatch.StartNew(); - embeddingCache = CacheHelper.GetEmbeddingStore(options.Value); + EmbeddingCache = CacheHelper.GetEmbeddingStore(options.Value); stopwatch.Stop(); _logger.LogInformation("GetEmbeddingStore completed in {ElapsedMilliseconds} ms", stopwatch.ElapsedMilliseconds); } else { - embeddingCache = new((int)EmbeddingCacheMaxCount); + EmbeddingCache = new((int)EmbeddingCacheMaxCount); } connectionString = _options.ConnectionStrings.SQL; - connection = new MySqlConnection(connectionString); - connection.Open(); - helper = new SQLHelper(connection, connectionString); + _connection = new MySqlConnection(connectionString); + _connection.Open(); + Helper = new SQLHelper(_connection, connectionString); } public Searchdomain GetSearchdomain(string searchdomain) { - if (searchdomains.TryGetValue(searchdomain, out Searchdomain? value)) + if (_searchdomains.TryGetValue(searchdomain, out Searchdomain? value)) { return value; } try { - return SetSearchdomain(searchdomain, new Searchdomain(searchdomain, connectionString, helper, aIProvider, embeddingCache, _logger)); + return SetSearchdomain(searchdomain, new Searchdomain(searchdomain, connectionString, Helper, AiProvider, EmbeddingCache, _logger)); } catch (MySqlException) { @@ -81,7 +81,7 @@ public class SearchdomainManager : IDisposable public async Task> ListSearchdomainsAsync() { - return await helper.ExecuteQueryAsync("SELECT name FROM searchdomain", [], x => x.GetString(0)); + return await Helper.ExecuteQueryAsync("SELECT name FROM searchdomain", [], x => x.GetString(0)); } public async Task CreateSearchdomain(string searchdomain, SearchdomainSettings settings) @@ -91,7 +91,7 @@ public class SearchdomainManager : IDisposable public async Task CreateSearchdomain(string searchdomain, string settings = "{}") { - if (searchdomains.TryGetValue(searchdomain, out Searchdomain? value)) + if (_searchdomains.TryGetValue(searchdomain, out Searchdomain? value)) { _logger.LogError("Searchdomain {searchdomain} could not be created, as it already exists", [searchdomain]); throw new SearchdomainAlreadyExistsException(searchdomain); @@ -101,30 +101,30 @@ public class SearchdomainManager : IDisposable { "name", searchdomain }, { "settings", settings} }; - int id = await helper.ExecuteSQLCommandGetInsertedID("INSERT INTO searchdomain (name, settings) VALUES (@name, @settings)", parameters); - searchdomains.Add(searchdomain, new(searchdomain, connectionString, helper, aIProvider, embeddingCache, _logger)); + int id = await Helper.ExecuteSQLCommandGetInsertedID("INSERT INTO searchdomain (name, settings) VALUES (@name, @settings)", parameters); + _searchdomains.Add(searchdomain, new(searchdomain, connectionString, Helper, AiProvider, EmbeddingCache, _logger)); return id; } public async Task DeleteSearchdomain(string searchdomain) { - int counter = await _databaseHelper.RemoveAllEntities(helper, searchdomain); + int counter = await _databaseHelper.RemoveAllEntities(Helper, searchdomain); _logger.LogDebug($"Number of entities deleted as part of deleting the searchdomain \"{searchdomain}\": {counter}"); - await helper.ExecuteSQLNonQuery("DELETE FROM searchdomain WHERE name = @name", new() {{"name", searchdomain}}); - searchdomains.Remove(searchdomain); + await Helper.ExecuteSQLNonQuery("DELETE FROM searchdomain WHERE name = @name", new() {{"name", searchdomain}}); + _searchdomains.Remove(searchdomain); _logger.LogDebug($"Searchdomain has been successfully removed"); return counter; } private Searchdomain SetSearchdomain(string name, Searchdomain searchdomain) { - searchdomains[name] = searchdomain; + _searchdomains[name] = searchdomain; return searchdomain; } public bool IsSearchdomainLoaded(string name) { - return searchdomains.ContainsKey(name); + return _searchdomains.ContainsKey(name); } // Cleanup procedure @@ -135,7 +135,7 @@ public class SearchdomainManager : IDisposable if (_options.Cache.StoreEmbeddingCache) { var stopwatch = Stopwatch.StartNew(); - await CacheHelper.UpdateEmbeddingStore(embeddingCache, _options); + await CacheHelper.UpdateEmbeddingStore(EmbeddingCache, _options); stopwatch.Stop(); _logger.LogInformation("UpdateEmbeddingStore completed in {ElapsedMilliseconds} ms", stopwatch.ElapsedMilliseconds); } @@ -155,10 +155,10 @@ public class SearchdomainManager : IDisposable protected virtual async Task Dispose(bool disposing) { - if (!disposed && disposing) + if (!_disposed && disposing) { await Cleanup(); - disposed = true; + _disposed = true; } } } diff --git a/src/Server/SimilarityMethods.cs b/src/Server/SimilarityMethods.cs index 7d28527..d0ab079 100644 --- a/src/Server/SimilarityMethods.cs +++ b/src/Server/SimilarityMethods.cs @@ -5,16 +5,16 @@ namespace Server; public class SimilarityMethod { - public SimilarityMethods.similarityMethodDelegate method; - public SimilarityMethodEnum similarityMethodEnum; - public string name; + public SimilarityMethods.similarityMethodDelegate Method; + public SimilarityMethodEnum SimilarityMethodEnum; + public string Name; public SimilarityMethod(SimilarityMethodEnum similarityMethodEnum) { - this.similarityMethodEnum = similarityMethodEnum; - this.name = similarityMethodEnum.ToString(); - SimilarityMethods.similarityMethodDelegate? probMethod = SimilarityMethods.GetMethod(name) ?? throw new Exception($"Unable to retrieve similarityMethod {name}"); - method = probMethod; + SimilarityMethodEnum = similarityMethodEnum; + Name = similarityMethodEnum.ToString(); + SimilarityMethods.similarityMethodDelegate? probMethod = SimilarityMethods.GetMethod(Name) ?? throw new Exception($"Unable to retrieve similarityMethod {Name}"); + Method = probMethod; } } @@ -22,11 +22,11 @@ public static class SimilarityMethods { public delegate float similarityMethodProtoDelegate(float[] vector1, float[] vector2); public delegate float similarityMethodDelegate(float[] vector1, float[] vector2); - public static readonly Dictionary probMethods; + public static readonly Dictionary ProbMethods; static SimilarityMethods() { - probMethods = new Dictionary + ProbMethods = new Dictionary { [SimilarityMethodEnum.Cosine] = CosineSimilarity, [SimilarityMethodEnum.Euclidian] = EuclidianDistance, @@ -44,7 +44,7 @@ public static class SimilarityMethods methodName ); - if (!probMethods.TryGetValue(probMethodEnum, out similarityMethodProtoDelegate? method)) + if (!ProbMethods.TryGetValue(probMethodEnum, out similarityMethodProtoDelegate? method)) { return null; }