lucretia/embeddingsearch
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixed naming convention issues

Browse Source
This commit is contained in:
LD-Reborn
2026-02-22 19:59:49 +01:00
parent 0582ff9a6c
commit cda028f213
14 changed files with 270 additions and 270 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
src/Server/AIProvider.cs
View File

@@ -13,7 +13,7 @@ public class AIProvider
{ {
private readonly ILogger<AIProvider> _logger; private readonly ILogger<AIProvider> _logger;
private readonly EmbeddingSearchOptions _configuration; private readonly EmbeddingSearchOptions _configuration;
public Dictionary<string, AiProvider> aIProvidersConfiguration; public Dictionary<string, AiProvider> AiProvidersConfiguration;
public AIProvider(ILogger<AIProvider> logger, IOptions<EmbeddingSearchOptions> configuration) public AIProvider(ILogger<AIProvider> logger, IOptions<EmbeddingSearchOptions> configuration)
{ {
@@ -27,7 +27,7 @@ public class AIProvider
} }
else else
{ {
aIProvidersConfiguration = retrievedAiProvidersConfiguration; AiProvidersConfiguration = retrievedAiProvidersConfiguration;
} }
} }
@@ -41,7 +41,7 @@ public class AIProvider
Uri uri = new(modelUri); Uri uri = new(modelUri);
string provider = uri.Scheme; string provider = uri.Scheme;
string model = uri.AbsolutePath; string model = uri.AbsolutePath;
AiProvider? aIProvider = aIProvidersConfiguration AiProvider? aIProvider = AiProvidersConfiguration
.FirstOrDefault(x => string.Equals(x.Key.ToLower(), provider.ToLower())) .FirstOrDefault(x => string.Equals(x.Key.ToLower(), provider.ToLower()))
.Value; .Value;
if (aIProvider is null) if (aIProvider is null)
@@ -134,7 +134,7 @@ public class AIProvider
public string[] GetModels() public string[] GetModels()
{ {
var aIProviders = aIProvidersConfiguration; var aIProviders = AiProvidersConfiguration;
List<string> results = []; List<string> results = [];
foreach (KeyValuePair<string, AiProvider> aIProviderKV in aIProviders) foreach (KeyValuePair<string, AiProvider> aIProviderKV in aIProviders)
{ {

38
src/Server/Controllers/EntityController.cs
View File

@@ -49,34 +49,34 @@ public class EntityController : ControllerBase
(Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomain, _logger); (Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomain, _logger);
if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new SearchdomainUpdateResults(){Success = false, Message = message}); if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new SearchdomainUpdateResults(){Success = false, Message = message});
EntityListResults entityListResults = new() {Results = [], Success = true}; EntityListResults entityListResults = new() {Results = [], Success = true};
foreach ((string _, Entity entity) in searchdomain_.entityCache) foreach ((string _, Entity entity) in searchdomain_.EntityCache)
{ {
List<AttributeResult> attributeResults = []; List<AttributeResult> attributeResults = [];
foreach (KeyValuePair<string, string> attribute in entity.attributes) foreach (KeyValuePair<string, string> attribute in entity.Attributes)
{ {
attributeResults.Add(new AttributeResult() {Name = attribute.Key, Value = attribute.Value}); attributeResults.Add(new AttributeResult() {Name = attribute.Key, Value = attribute.Value});
} }
List<DatapointResult> datapointResults = []; List<DatapointResult> datapointResults = [];
foreach (Datapoint datapoint in entity.datapoints) foreach (Datapoint datapoint in entity.Datapoints)
{ {
if (returnModels) if (returnModels)
{ {
List<EmbeddingResult> embeddingResults = []; List<EmbeddingResult> embeddingResults = [];
foreach ((string, float[]) embedding in datapoint.embeddings) foreach ((string, float[]) embedding in datapoint.Embeddings)
{ {
embeddingResults.Add(new EmbeddingResult() {Model = embedding.Item1, Embeddings = returnEmbeddings ? embedding.Item2 : []}); embeddingResults.Add(new EmbeddingResult() {Model = embedding.Item1, Embeddings = returnEmbeddings ? embedding.Item2 : []});
} }
datapointResults.Add(new DatapointResult() {Name = datapoint.name, ProbMethod = datapoint.probMethod.name, SimilarityMethod = datapoint.similarityMethod.name, Embeddings = embeddingResults}); datapointResults.Add(new DatapointResult() {Name = datapoint.Name, ProbMethod = datapoint.ProbMethod.Name, SimilarityMethod = datapoint.SimilarityMethod.Name, Embeddings = embeddingResults});
} }
else else
{ {
datapointResults.Add(new DatapointResult() {Name = datapoint.name, ProbMethod = datapoint.probMethod.name, SimilarityMethod = datapoint.similarityMethod.name, Embeddings = null}); datapointResults.Add(new DatapointResult() {Name = datapoint.Name, ProbMethod = datapoint.ProbMethod.Name, SimilarityMethod = datapoint.SimilarityMethod.Name, Embeddings = null});
} }
} }
EntityListResult entityListResult = new() EntityListResult entityListResult = new()
{ {
Name = entity.name, Name = entity.Name,
ProbMethod = entity.probMethodName, ProbMethod = entity.ProbMethodName,
Attributes = attributeResults, Attributes = attributeResults,
Datapoints = datapointResults Datapoints = datapointResults
}; };
@@ -162,20 +162,20 @@ public class EntityController : ControllerBase
private async Task EntityIndexSessionDeleteUnindexedEntities(EntityIndexSessionData session) private async Task EntityIndexSessionDeleteUnindexedEntities(EntityIndexSessionData session)
{ {
var entityGroupsBySearchdomain = session.AccumulatedEntities.GroupBy(e => e.searchdomain); var entityGroupsBySearchdomain = session.AccumulatedEntities.GroupBy(e => e.Searchdomain);
foreach (var entityGroup in entityGroupsBySearchdomain) foreach (var entityGroup in entityGroupsBySearchdomain)
{ {
string searchdomainName = entityGroup.Key; string searchdomainName = entityGroup.Key;
var entityNamesInRequest = entityGroup.Select(e => e.name).ToHashSet(); var entityNamesInRequest = entityGroup.Select(e => e.Name).ToHashSet();
(Searchdomain? searchdomain_, int? httpStatusCode, string? message) = (Searchdomain? searchdomain_, int? httpStatusCode, string? message) =
SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomainName, _logger); SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomainName, _logger);
if (searchdomain_ is not null && httpStatusCode is null) // If getting searchdomain was successful if (searchdomain_ is not null && httpStatusCode is null) // If getting searchdomain was successful
{ {
var entitiesToDelete = searchdomain_.entityCache var entitiesToDelete = searchdomain_.EntityCache
.Where(kvp => !entityNamesInRequest.Contains(kvp.Value.name)) .Where(kvp => !entityNamesInRequest.Contains(kvp.Value.Name))
.Select(kvp => kvp.Value) .Select(kvp => kvp.Value)
.ToList(); .ToList();
@@ -184,11 +184,11 @@ public class EntityController : ControllerBase
searchdomain_.ReconciliateOrInvalidateCacheForDeletedEntity(entity); searchdomain_.ReconciliateOrInvalidateCacheForDeletedEntity(entity);
await _databaseHelper.RemoveEntity( await _databaseHelper.RemoveEntity(
[], [],
_domainManager.helper, _domainManager.Helper,
entity.name, entity.Name,
searchdomainName); searchdomainName);
searchdomain_.entityCache.TryRemove(entity.name, out _); searchdomain_.EntityCache.TryRemove(entity.Name, out _);
_logger.LogInformation("Deleted entity {entityName} from {searchdomain}", entity.name, searchdomainName); _logger.LogInformation("Deleted entity {entityName} from {searchdomain}", entity.Name, searchdomainName);
} }
} }
else else
@@ -209,7 +209,7 @@ public class EntityController : ControllerBase
(Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomain, _logger); (Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomain, _logger);
if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new SearchdomainUpdateResults(){Success = false, Message = message}); if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new SearchdomainUpdateResults(){Success = false, Message = message});
Entity? entity_ = SearchdomainHelper.CacheGetEntity(searchdomain_.entityCache, entityName); Entity? entity_ = SearchdomainHelper.CacheGetEntity(searchdomain_.EntityCache, entityName);
if (entity_ is null) if (entity_ is null)
{ {
_logger.LogError("Unable to delete the entity {entityName} in {searchdomain} - it was not found under the specified name", [entityName, searchdomain]); _logger.LogError("Unable to delete the entity {entityName} in {searchdomain} - it was not found under the specified name", [entityName, searchdomain]);
@@ -221,9 +221,9 @@ public class EntityController : ControllerBase
return Ok(new EntityDeleteResults() {Success = false, Message = "Entity not found"}); return Ok(new EntityDeleteResults() {Success = false, Message = "Entity not found"});
} }
searchdomain_.ReconciliateOrInvalidateCacheForDeletedEntity(entity_); searchdomain_.ReconciliateOrInvalidateCacheForDeletedEntity(entity_);
await _databaseHelper.RemoveEntity([], _domainManager.helper, entityName, searchdomain); await _databaseHelper.RemoveEntity([], _domainManager.Helper, entityName, searchdomain);
bool success = searchdomain_.entityCache.TryRemove(entityName, out Entity? _); bool success = searchdomain_.EntityCache.TryRemove(entityName, out Entity? _);
return Ok(new EntityDeleteResults() {Success = success}); return Ok(new EntityDeleteResults() {Success = success});
} }

32
src/Server/Controllers/SearchdomainController.cs
View File

@@ -118,18 +118,18 @@ public class SearchdomainController : ControllerBase
Dictionary<string, dynamic> parameters = new() Dictionary<string, dynamic> parameters = new()
{ {
{"name", newName}, {"name", newName},
{"id", searchdomain_.id} {"id", searchdomain_.Id}
}; };
await searchdomain_.helper.ExecuteSQLNonQuery("UPDATE searchdomain set name = @name WHERE id = @id", parameters); await searchdomain_.Helper.ExecuteSQLNonQuery("UPDATE searchdomain set name = @name WHERE id = @id", parameters);
} else } else
{ {
Dictionary<string, dynamic> parameters = new() Dictionary<string, dynamic> parameters = new()
{ {
{"name", newName}, {"name", newName},
{"settings", settings}, {"settings", settings},
{"id", searchdomain_.id} {"id", searchdomain_.Id}
}; };
await searchdomain_.helper.ExecuteSQLNonQuery("UPDATE searchdomain set name = @name, settings = @settings WHERE id = @id", parameters); await searchdomain_.Helper.ExecuteSQLNonQuery("UPDATE searchdomain set name = @name, settings = @settings WHERE id = @id", parameters);
} }
return Ok(new SearchdomainUpdateResults(){Success = true}); return Ok(new SearchdomainUpdateResults(){Success = true});
} }
@@ -143,7 +143,7 @@ public class SearchdomainController : ControllerBase
{ {
(Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomain, _logger); (Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomain, _logger);
if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new SearchdomainUpdateResults(){Success = false, Message = message}); if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new SearchdomainUpdateResults(){Success = false, Message = message});
Dictionary<string, DateTimedSearchResult> searchCache = searchdomain_.queryCache.AsDictionary(); Dictionary<string, DateTimedSearchResult> searchCache = searchdomain_.QueryCache.AsDictionary();
return Ok(new SearchdomainQueriesResults() { Searches = searchCache, Success = true }); return Ok(new SearchdomainQueriesResults() { Searches = searchCache, Success = true });
} }
@@ -165,7 +165,7 @@ public class SearchdomainController : ControllerBase
{ {
Name = r.Item2, Name = r.Item2,
Value = r.Item1, Value = r.Item1,
Attributes = returnAttributes ? (searchdomain_.entityCache[r.Item2]?.attributes ?? null) : null Attributes = returnAttributes ? (searchdomain_.EntityCache[r.Item2]?.Attributes ?? null) : null
})]; })];
return Ok(new EntityQueryResults(){Results = queryResults, Success = true }); return Ok(new EntityQueryResults(){Results = queryResults, Success = true });
} }
@@ -180,7 +180,7 @@ public class SearchdomainController : ControllerBase
{ {
(Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomain, _logger); (Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomain, _logger);
if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new SearchdomainUpdateResults(){Success = false, Message = message}); if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new SearchdomainUpdateResults(){Success = false, Message = message});
EnumerableLruCache<string, DateTimedSearchResult> searchCache = searchdomain_.queryCache; EnumerableLruCache<string, DateTimedSearchResult> searchCache = searchdomain_.QueryCache;
bool containsKey = searchCache.ContainsKey(query); bool containsKey = searchCache.ContainsKey(query);
if (containsKey) if (containsKey)
{ {
@@ -201,7 +201,7 @@ public class SearchdomainController : ControllerBase
{ {
(Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomain, _logger); (Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomain, _logger);
if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new SearchdomainUpdateResults(){Success = false, Message = message}); if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new SearchdomainUpdateResults(){Success = false, Message = message});
EnumerableLruCache<string, DateTimedSearchResult> searchCache = searchdomain_.queryCache; EnumerableLruCache<string, DateTimedSearchResult> searchCache = searchdomain_.QueryCache;
bool containsKey = searchCache.ContainsKey(query); bool containsKey = searchCache.ContainsKey(query);
if (containsKey) if (containsKey)
{ {
@@ -222,7 +222,7 @@ public class SearchdomainController : ControllerBase
{ {
(Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomain, _logger); (Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomain, _logger);
if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new SearchdomainUpdateResults(){Success = false, Message = message}); if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new SearchdomainUpdateResults(){Success = false, Message = message});
SearchdomainSettings settings = searchdomain_.settings; SearchdomainSettings settings = searchdomain_.Settings;
return Ok(new SearchdomainSettingsResults() { Settings = settings, Success = true }); return Ok(new SearchdomainSettingsResults() { Settings = settings, Success = true });
} }
@@ -239,11 +239,11 @@ public class SearchdomainController : ControllerBase
Dictionary<string, dynamic> parameters = new() Dictionary<string, dynamic> parameters = new()
{ {
{"settings", JsonSerializer.Serialize(request)}, {"settings", JsonSerializer.Serialize(request)},
{"id", searchdomain_.id} {"id", searchdomain_.Id}
}; };
await searchdomain_.helper.ExecuteSQLNonQuery("UPDATE searchdomain set settings = @settings WHERE id = @id", parameters); await searchdomain_.Helper.ExecuteSQLNonQuery("UPDATE searchdomain set settings = @settings WHERE id = @id", parameters);
searchdomain_.settings = request; searchdomain_.Settings = request;
searchdomain_.queryCache.Capacity = request.QueryCacheSize; searchdomain_.QueryCache.Capacity = request.QueryCacheSize;
return Ok(new SearchdomainUpdateResults(){Success = true}); return Ok(new SearchdomainUpdateResults(){Success = true});
} }
@@ -260,8 +260,8 @@ public class SearchdomainController : ControllerBase
} }
(Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomain, _logger); (Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomain, _logger);
if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new SearchdomainUpdateResults(){Success = false, Message = message}); if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new SearchdomainUpdateResults(){Success = false, Message = message});
int elementCount = searchdomain_.queryCache.Count; int elementCount = searchdomain_.QueryCache.Count;
int ElementMaxCount = searchdomain_.settings.QueryCacheSize; int ElementMaxCount = searchdomain_.Settings.QueryCacheSize;
return Ok(new SearchdomainQueryCacheSizeResults() { SizeBytes = searchdomain_.GetSearchCacheSize(), ElementCount = elementCount, ElementMaxCount = ElementMaxCount, Success = true }); return Ok(new SearchdomainQueryCacheSizeResults() { SizeBytes = searchdomain_.GetSearchCacheSize(), ElementCount = elementCount, ElementMaxCount = ElementMaxCount, Success = true });
} }
@@ -287,7 +287,7 @@ public class SearchdomainController : ControllerBase
{ {
(Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomain, _logger); (Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomain, _logger);
if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new SearchdomainUpdateResults(){Success = false, Message = message}); if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new SearchdomainUpdateResults(){Success = false, Message = message});
long EmbeddingCacheUtilization = DatabaseHelper.GetSearchdomainDatabaseSize(searchdomain_.helper, searchdomain); long EmbeddingCacheUtilization = DatabaseHelper.GetSearchdomainDatabaseSize(searchdomain_.Helper, searchdomain);
return Ok(new SearchdomainGetDatabaseSizeResult() { SearchdomainDatabaseSizeBytes = EmbeddingCacheUtilization, Success = true }); return Ok(new SearchdomainGetDatabaseSizeResult() { SearchdomainDatabaseSizeBytes = EmbeddingCacheUtilization, Success = true });
} }
} }

10
src/Server/Controllers/ServerController.cs
View File

@@ -58,7 +58,7 @@ public class ServerController : ControllerBase
long size = 0; long size = 0;
long elementCount = 0; long elementCount = 0;
long embeddingsCount = 0; long embeddingsCount = 0;
EnumerableLruCache<string, Dictionary<string, float[]>> embeddingCache = _searchdomainManager.embeddingCache; EnumerableLruCache<string, Dictionary<string, float[]>> embeddingCache = _searchdomainManager.EmbeddingCache;
foreach (KeyValuePair<string, Dictionary<string, float[]>> kv in embeddingCache) foreach (KeyValuePair<string, Dictionary<string, float[]>> kv in embeddingCache)
{ {
@@ -68,7 +68,7 @@ public class ServerController : ControllerBase
elementCount++; elementCount++;
embeddingsCount += entry.Keys.Count; embeddingsCount += entry.Keys.Count;
} }
var sqlHelper = _searchdomainManager.helper; var sqlHelper = _searchdomainManager.Helper;
var databaseTotalSize = DatabaseHelper.GetTotalDatabaseSize(sqlHelper); var databaseTotalSize = DatabaseHelper.GetTotalDatabaseSize(sqlHelper);
Task<long> entityCountTask = DatabaseHelper.CountEntities(sqlHelper); Task<long> entityCountTask = DatabaseHelper.CountEntities(sqlHelper);
long queryCacheUtilization = 0; long queryCacheUtilization = 0;
@@ -82,9 +82,9 @@ public class ServerController : ControllerBase
(Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_searchdomainManager, searchdomain, _logger); (Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_searchdomainManager, searchdomain, _logger);
if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new ServerGetStatsResult(){Success = false, Message = message}); if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new ServerGetStatsResult(){Success = false, Message = message});
queryCacheUtilization += searchdomain_.GetSearchCacheSize(); queryCacheUtilization += searchdomain_.GetSearchCacheSize();
queryCacheElementCount += searchdomain_.queryCache.Count; queryCacheElementCount += searchdomain_.QueryCache.Count;
queryCacheMaxElementCountAll += searchdomain_.queryCache.Capacity; queryCacheMaxElementCountAll += searchdomain_.QueryCache.Capacity;
queryCacheMaxElementCountLoadedSearchdomainsOnly += searchdomain_.queryCache.Capacity; queryCacheMaxElementCountLoadedSearchdomainsOnly += searchdomain_.QueryCache.Capacity;
} else } else
{ {
var searchdomainSettings = DatabaseHelper.GetSearchdomainSettings(sqlHelper, searchdomain); var searchdomainSettings = DatabaseHelper.GetSearchdomainSettings(sqlHelper, searchdomain);

38
src/Server/Datapoint.cs
View File

@@ -6,36 +6,36 @@ namespace Server;
public class Datapoint public class Datapoint
{ {
public string name; public string Name;
public ProbMethod probMethod; public ProbMethod ProbMethod;
public SimilarityMethod similarityMethod; public SimilarityMethod SimilarityMethod;
public List<(string, float[])> embeddings; public List<(string, float[])> Embeddings;
public string hash; public string Hash;
public int id; public int Id;
public Datapoint(string name, ProbMethodEnum probMethod, SimilarityMethodEnum similarityMethod, string hash, List<(string, float[])> embeddings, int id) public Datapoint(string name, ProbMethodEnum probMethod, SimilarityMethodEnum similarityMethod, string hash, List<(string, float[])> embeddings, int id)
{ {
this.name = name; Name = name;
this.probMethod = new ProbMethod(probMethod); ProbMethod = new ProbMethod(probMethod);
this.similarityMethod = new SimilarityMethod(similarityMethod); SimilarityMethod = new SimilarityMethod(similarityMethod);
this.hash = hash; Hash = hash;
this.embeddings = embeddings; Embeddings = embeddings;
this.id = id; Id = id;
} }
public Datapoint(string name, ProbMethod probMethod, SimilarityMethod similarityMethod, string hash, List<(string, float[])> embeddings, int id) public Datapoint(string name, ProbMethod probMethod, SimilarityMethod similarityMethod, string hash, List<(string, float[])> embeddings, int id)
{ {
this.name = name; Name = name;
this.probMethod = probMethod; ProbMethod = probMethod;
this.similarityMethod = similarityMethod; SimilarityMethod = similarityMethod;
this.hash = hash; Hash = hash;
this.embeddings = embeddings; Embeddings = embeddings;
this.id = id; Id = id;
} }
public float CalcProbability(List<(string, float)> probabilities) public float CalcProbability(List<(string, float)> probabilities)
{ {
return probMethod.method(probabilities); return ProbMethod.Method(probabilities);
} }
public static Dictionary<string, float[]> GetEmbeddings(string content, ConcurrentBag<string> models, AIProvider aIProvider, EnumerableLruCache<string, Dictionary<string, float[]>> embeddingCache) public static Dictionary<string, float[]> GetEmbeddings(string content, ConcurrentBag<string> models, AIProvider aIProvider, EnumerableLruCache<string, Dictionary<string, float[]>> embeddingCache)

16
src/Server/Entity.cs
View File

@@ -2,13 +2,13 @@ using System.Collections.Concurrent;
namespace Server; namespace Server;
public class Entity(Dictionary<string, string> attributes, Probmethods.probMethodDelegate probMethod, string probMethodName, ConcurrentBag<Datapoint> datapoints, string name, string searchdomain) public class Entity(Dictionary<string, string> attributes, Probmethods.ProbMethodDelegate probMethod, string probMethodName, ConcurrentBag<Datapoint> datapoints, string name, string searchdomain)
{ {
public Dictionary<string, string> attributes = attributes; public Dictionary<string, string> Attributes = attributes;
public Probmethods.probMethodDelegate probMethod = probMethod; public Probmethods.ProbMethodDelegate ProbMethod = probMethod;
public string probMethodName = probMethodName; public string ProbMethodName = probMethodName;
public ConcurrentBag<Datapoint> datapoints = datapoints; public ConcurrentBag<Datapoint> Datapoints = datapoints;
public int id; public int Id;
public string name = name; public string Name = name;
public string searchdomain = searchdomain; public string Searchdomain = searchdomain;
} }

6
src/Server/HealthChecks/ServerHealthChecks.cs
View File

@@ -17,7 +17,7 @@ public class DatabaseHealthCheck : IHealthCheck
{ {
try try
{ {
DatabaseMigrations.DatabaseGetVersion(_searchdomainManager.helper); DatabaseMigrations.DatabaseGetVersion(_searchdomainManager.Helper);
} }
catch (Exception ex) catch (Exception ex)
{ {
@@ -28,8 +28,8 @@ public class DatabaseHealthCheck : IHealthCheck
try try
{ {
await _searchdomainManager.helper.ExecuteSQLNonQuery("INSERT INTO settings (name, value) VALUES ('test', 'x');", []); await _searchdomainManager.Helper.ExecuteSQLNonQuery("INSERT INTO settings (name, value) VALUES ('test', 'x');", []);
await _searchdomainManager.helper.ExecuteSQLNonQuery("DELETE FROM settings WHERE name = 'test';", []); await _searchdomainManager.Helper.ExecuteSQLNonQuery("DELETE FROM settings WHERE name = 'test';", []);
} }
catch (Exception ex) catch (Exception ex)
{ {

6
src/Server/Helper/DatabaseHelper.cs
View File

@@ -205,7 +205,7 @@ public class DatabaseHelper(ILogger<DatabaseHelper> logger)
await helper.ExecuteSQLNonQuery("DELETE datapoint.* FROM datapoint JOIN entity ON id_entity = entity.id WHERE entity.name = @name AND entity.id_searchdomain = @searchdomain", parameters); await helper.ExecuteSQLNonQuery("DELETE datapoint.* FROM datapoint JOIN entity ON id_entity = entity.id WHERE entity.name = @name AND entity.id_searchdomain = @searchdomain", parameters);
await helper.ExecuteSQLNonQuery("DELETE attribute.* FROM attribute JOIN entity ON id_entity = entity.id WHERE entity.name = @name AND entity.id_searchdomain = @searchdomain", parameters); await helper.ExecuteSQLNonQuery("DELETE attribute.* FROM attribute JOIN entity ON id_entity = entity.id WHERE entity.name = @name AND entity.id_searchdomain = @searchdomain", parameters);
await helper.ExecuteSQLNonQuery("DELETE FROM entity WHERE name = @name AND entity.id_searchdomain = @searchdomain", parameters); await helper.ExecuteSQLNonQuery("DELETE FROM entity WHERE name = @name AND entity.id_searchdomain = @searchdomain", parameters);
entityCache.RemoveAll(entity => entity.name == name); entityCache.RemoveAll(entity => entity.Name == name);
} }
public async Task<int> RemoveAllEntities(SQLHelper helper, string searchdomain) public async Task<int> RemoveAllEntities(SQLHelper helper, string searchdomain)
@@ -243,7 +243,7 @@ public class DatabaseHelper(ILogger<DatabaseHelper> logger)
{ "name", name }, { "name", name },
{ "searchdomain", await GetSearchdomainID(helper, searchdomain)} { "searchdomain", await GetSearchdomainID(helper, searchdomain)}
}; };
lock (helper.connection) lock (helper.Connection)
{ {
DbDataReader reader = helper.ExecuteSQLCommand("SELECT COUNT(*) FROM entity WHERE name = @name AND id_searchdomain = @searchdomain", parameters); DbDataReader reader = helper.ExecuteSQLCommand("SELECT COUNT(*) FROM entity WHERE name = @name AND id_searchdomain = @searchdomain", parameters);
try try
@@ -273,7 +273,7 @@ public class DatabaseHelper(ILogger<DatabaseHelper> logger)
{ "name", name }, { "name", name },
{ "searchdomain", await GetSearchdomainID(helper, searchdomain)} { "searchdomain", await GetSearchdomainID(helper, searchdomain)}
}; };
lock (helper.connection) lock (helper.Connection)
{ {
DbDataReader reader = helper.ExecuteSQLCommand("SELECT id FROM entity WHERE name = @name AND id_searchdomain = @searchdomain", parameters); DbDataReader reader = helper.ExecuteSQLCommand("SELECT id FROM entity WHERE name = @name AND id_searchdomain = @searchdomain", parameters);
try try

58
src/Server/Helper/SQLHelper.cs
View File

@@ -6,18 +6,18 @@ namespace Server.Helper;
public class SQLHelper:IDisposable public class SQLHelper:IDisposable
{ {
public MySqlConnection connection; public MySqlConnection Connection;
public DbDataReader? dbDataReader; public DbDataReader? DbDataReader;
public MySqlConnectionPoolElement[] connectionPool; public MySqlConnectionPoolElement[] ConnectionPool;
public string connectionString; public string ConnectionString;
public SQLHelper(MySqlConnection connection, string connectionString) public SQLHelper(MySqlConnection connection, string connectionString)
{ {
this.connection = connection; Connection = connection;
this.connectionString = connectionString; ConnectionString = connectionString;
connectionPool = new MySqlConnectionPoolElement[50]; ConnectionPool = new MySqlConnectionPoolElement[50];
for (int i = 0; i < connectionPool.Length; i++) for (int i = 0; i < ConnectionPool.Length; i++)
{ {
connectionPool[i] = new MySqlConnectionPoolElement(new MySqlConnection(connectionString), new(1, 1)); ConnectionPool[i] = new MySqlConnectionPoolElement(new MySqlConnection(connectionString), new(1, 1));
} }
} }
@@ -28,24 +28,24 @@ public class SQLHelper:IDisposable
public void Dispose() public void Dispose()
{ {
connection.Close(); Connection.Close();
GC.SuppressFinalize(this); GC.SuppressFinalize(this);
} }
public DbDataReader ExecuteSQLCommand(string query, Dictionary<string, dynamic> parameters) public DbDataReader ExecuteSQLCommand(string query, Dictionary<string, dynamic> parameters)
{ {
lock (connection) lock (Connection)
{ {
EnsureConnected(); EnsureConnected();
EnsureDbReaderIsClosed(); EnsureDbReaderIsClosed();
using MySqlCommand command = connection.CreateCommand(); using MySqlCommand command = Connection.CreateCommand();
command.CommandText = query; command.CommandText = query;
foreach (KeyValuePair<string, dynamic> parameter in parameters) foreach (KeyValuePair<string, dynamic> parameter in parameters)
{ {
command.Parameters.AddWithValue($"@{parameter.Key}", parameter.Value); command.Parameters.AddWithValue($"@{parameter.Key}", parameter.Value);
} }
dbDataReader = command.ExecuteReader(); DbDataReader = command.ExecuteReader();
return dbDataReader; return DbDataReader;
} }
} }
@@ -55,7 +55,7 @@ public class SQLHelper:IDisposable
Func<DbDataReader, T> map) Func<DbDataReader, T> map)
{ {
var poolElement = await GetMySqlConnectionPoolElement(); var poolElement = await GetMySqlConnectionPoolElement();
var connection = poolElement.connection; var connection = poolElement.Connection;
try try
{ {
await using var command = connection.CreateCommand(); await using var command = connection.CreateCommand();
@@ -83,7 +83,7 @@ public class SQLHelper:IDisposable
public async Task<int> ExecuteSQLNonQuery(string query, Dictionary<string, dynamic> parameters) public async Task<int> ExecuteSQLNonQuery(string query, Dictionary<string, dynamic> parameters)
{ {
var poolElement = await GetMySqlConnectionPoolElement(); var poolElement = await GetMySqlConnectionPoolElement();
var connection = poolElement.connection; var connection = poolElement.Connection;
try try
{ {
using MySqlCommand command = connection.CreateCommand(); using MySqlCommand command = connection.CreateCommand();
@@ -103,7 +103,7 @@ public class SQLHelper:IDisposable
public async Task<int> ExecuteSQLCommandGetInsertedID(string query, Dictionary<string, dynamic> parameters) public async Task<int> ExecuteSQLCommandGetInsertedID(string query, Dictionary<string, dynamic> parameters)
{ {
var poolElement = await GetMySqlConnectionPoolElement(); var poolElement = await GetMySqlConnectionPoolElement();
var connection = poolElement.connection; var connection = poolElement.Connection;
try try
{ {
using MySqlCommand command = connection.CreateCommand(); using MySqlCommand command = connection.CreateCommand();
@@ -125,7 +125,7 @@ public class SQLHelper:IDisposable
public async Task<int> BulkExecuteNonQuery(string sql, IEnumerable<object[]> parameterSets) public async Task<int> BulkExecuteNonQuery(string sql, IEnumerable<object[]> parameterSets)
{ {
var poolElement = await GetMySqlConnectionPoolElement(); var poolElement = await GetMySqlConnectionPoolElement();
var connection = poolElement.connection; var connection = poolElement.Connection;
try try
{ {
int affectedRows = 0; int affectedRows = 0;
@@ -173,14 +173,14 @@ public class SQLHelper:IDisposable
int sleepTime = 10; int sleepTime = 10;
do do
{ {
foreach (var element in connectionPool) foreach (var element in ConnectionPool)
{ {
if (element.Semaphore.Wait(0)) if (element.Semaphore.Wait(0))
{ {
if (element.connection.State == ConnectionState.Closed) if (element.Connection.State == ConnectionState.Closed)
{ {
await element.connection.CloseAsync(); await element.Connection.CloseAsync();
await element.connection.OpenAsync(); await element.Connection.OpenAsync();
} }
return element; return element;
} }
@@ -194,12 +194,12 @@ public class SQLHelper:IDisposable
public bool EnsureConnected() public bool EnsureConnected()
{ {
if (connection.State != System.Data.ConnectionState.Open) if (Connection.State != System.Data.ConnectionState.Open)
{ {
try try
{ {
connection.Close(); Connection.Close();
connection.Open(); Connection.Open();
} }
catch (Exception ex) catch (Exception ex)
{ {
@@ -215,7 +215,7 @@ public class SQLHelper:IDisposable
int counter = 0; int counter = 0;
int sleepTime = 10; int sleepTime = 10;
int timeout = 5000; int timeout = 5000;
while (!(dbDataReader?.IsClosed ?? true)) while (!(DbDataReader?.IsClosed ?? true))
{ {
if (counter > timeout / sleepTime) if (counter > timeout / sleepTime)
{ {
@@ -230,12 +230,12 @@ public class SQLHelper:IDisposable
public struct MySqlConnectionPoolElement public struct MySqlConnectionPoolElement
{ {
public MySqlConnection connection; public MySqlConnection Connection;
public SemaphoreSlim Semaphore; public SemaphoreSlim Semaphore;
public MySqlConnectionPoolElement(MySqlConnection connection, SemaphoreSlim semaphore) public MySqlConnectionPoolElement(MySqlConnection connection, SemaphoreSlim semaphore)
{ {
this.connection = connection; Connection = connection;
this.Semaphore = semaphore; Semaphore = semaphore;
} }
} }

108
src/Server/Helper/SearchdomainHelper.cs
View File

@@ -39,7 +39,7 @@ public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelp
{ {
foreach ((string _, Entity entity) in entityCache) foreach ((string _, Entity entity) in entityCache)
{ {
if (entity.name == name) if (entity.Name == name)
{ {
return entity; return entity;
} }
@@ -49,9 +49,9 @@ public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelp
public async Task<List<Entity>?> EntitiesFromJSON(SearchdomainManager searchdomainManager, ILogger logger, string json) public async Task<List<Entity>?> EntitiesFromJSON(SearchdomainManager searchdomainManager, ILogger logger, string json)
{ {
EnumerableLruCache<string, Dictionary<string, float[]>> embeddingCache = searchdomainManager.embeddingCache; EnumerableLruCache<string, Dictionary<string, float[]>> embeddingCache = searchdomainManager.EmbeddingCache;
AIProvider aIProvider = searchdomainManager.aIProvider; AIProvider aIProvider = searchdomainManager.AiProvider;
SQLHelper helper = searchdomainManager.helper; SQLHelper helper = searchdomainManager.Helper;
List<JSONEntity>? jsonEntities = JsonSerializer.Deserialize<List<JSONEntity>>(json); List<JSONEntity>? jsonEntities = JsonSerializer.Deserialize<List<JSONEntity>>(json);
if (jsonEntities is null) if (jsonEntities is null)
@@ -65,7 +65,7 @@ public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelp
foreach (JSONEntity jSONEntity in jsonEntities) foreach (JSONEntity jSONEntity in jsonEntities)
{ {
Dictionary<string, List<string>> targetDictionary = toBeCached; Dictionary<string, List<string>> targetDictionary = toBeCached;
if (searchdomainManager.GetSearchdomain(jSONEntity.Searchdomain).settings.ParallelEmbeddingsPrefetch) if (searchdomainManager.GetSearchdomain(jSONEntity.Searchdomain).Settings.ParallelEmbeddingsPrefetch)
{ {
targetDictionary = toBeCachedParallel; targetDictionary = toBeCachedParallel;
} }
@@ -126,12 +126,12 @@ public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelp
{ {
var stopwatch = Stopwatch.StartNew(); var stopwatch = Stopwatch.StartNew();
SQLHelper helper = searchdomainManager.helper; SQLHelper helper = searchdomainManager.Helper;
Searchdomain searchdomain = searchdomainManager.GetSearchdomain(jsonEntity.Searchdomain); Searchdomain searchdomain = searchdomainManager.GetSearchdomain(jsonEntity.Searchdomain);
int id_searchdomain = searchdomain.id; int id_searchdomain = searchdomain.Id;
ConcurrentDictionary<string, Entity> entityCache = searchdomain.entityCache; ConcurrentDictionary<string, Entity> entityCache = searchdomain.EntityCache;
AIProvider aIProvider = searchdomain.aIProvider; AIProvider aIProvider = searchdomain.AiProvider;
EnumerableLruCache<string, Dictionary<string, float[]>> embeddingCache = searchdomain.embeddingCache; EnumerableLruCache<string, Dictionary<string, float[]>> embeddingCache = searchdomain.EmbeddingCache;
bool invalidateSearchCache = false; bool invalidateSearchCache = false;
@@ -140,15 +140,15 @@ public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelp
if (hasEntity && preexistingEntity is not null) if (hasEntity && preexistingEntity is not null)
{ {
int preexistingEntityID = preexistingEntity.id; int preexistingEntityID = preexistingEntity.Id;
Dictionary<string, string> attributes = jsonEntity.Attributes; Dictionary<string, string> attributes = jsonEntity.Attributes;
// Attribute - get changes // Attribute - get changes
List<(string attribute, string newValue, int entityId)> updatedAttributes = new(preexistingEntity.attributes.Count); List<(string attribute, string newValue, int entityId)> updatedAttributes = new(preexistingEntity.Attributes.Count);
List<(string attribute, int entityId)> deletedAttributes = new(preexistingEntity.attributes.Count); List<(string attribute, int entityId)> deletedAttributes = new(preexistingEntity.Attributes.Count);
List<(string attributeKey, string attribute, int entityId)> addedAttributes = new(jsonEntity.Attributes.Count); List<(string attributeKey, string attribute, int entityId)> addedAttributes = new(jsonEntity.Attributes.Count);
foreach (KeyValuePair<string, string> attributesKV in preexistingEntity.attributes) //.ToList()) foreach (KeyValuePair<string, string> attributesKV in preexistingEntity.Attributes) //.ToList())
{ {
string oldAttributeKey = attributesKV.Key; string oldAttributeKey = attributesKV.Key;
string oldAttribute = attributesKV.Value; string oldAttribute = attributesKV.Value;
@@ -166,7 +166,7 @@ public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelp
{ {
string newAttributeKey = attributesKV.Key; string newAttributeKey = attributesKV.Key;
string newAttribute = attributesKV.Value; string newAttribute = attributesKV.Value;
bool preexistingHasAttribute = preexistingEntity.attributes.TryGetValue(newAttributeKey, out string? preexistingAttribute); bool preexistingHasAttribute = preexistingEntity.Attributes.TryGetValue(newAttributeKey, out string? preexistingAttribute);
if (!preexistingHasAttribute) if (!preexistingHasAttribute)
{ {
// Attribute - New // Attribute - New
@@ -175,54 +175,54 @@ public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelp
} }
if (updatedAttributes.Count != 0 || deletedAttributes.Count != 0 || addedAttributes.Count != 0) if (updatedAttributes.Count != 0 || deletedAttributes.Count != 0 || addedAttributes.Count != 0)
_logger.LogDebug("EntityFromJSON - Updating existing entity. name: {name}, updatedAttributes: {updatedAttributes}, deletedAttributes: {deletedAttributes}, addedAttributes: {addedAttributes}", [preexistingEntity.name, updatedAttributes.Count, deletedAttributes.Count, addedAttributes.Count]); _logger.LogDebug("EntityFromJSON - Updating existing entity. name: {name}, updatedAttributes: {updatedAttributes}, deletedAttributes: {deletedAttributes}, addedAttributes: {addedAttributes}", [preexistingEntity.Name, updatedAttributes.Count, deletedAttributes.Count, addedAttributes.Count]);
// Attribute - apply changes // Attribute - apply changes
if (updatedAttributes.Count != 0) if (updatedAttributes.Count != 0)
{ {
// Update // Update
await DatabaseHelper.DatabaseUpdateAttributes(helper, updatedAttributes); await DatabaseHelper.DatabaseUpdateAttributes(helper, updatedAttributes);
lock (preexistingEntity.attributes) lock (preexistingEntity.Attributes)
{ {
updatedAttributes.ForEach(attribute => preexistingEntity.attributes[attribute.attribute] = attribute.newValue); updatedAttributes.ForEach(attribute => preexistingEntity.Attributes[attribute.attribute] = attribute.newValue);
} }
} }
if (deletedAttributes.Count != 0) if (deletedAttributes.Count != 0)
{ {
// Delete // Delete
await DatabaseHelper.DatabaseDeleteAttributes(helper, deletedAttributes); await DatabaseHelper.DatabaseDeleteAttributes(helper, deletedAttributes);
lock (preexistingEntity.attributes) lock (preexistingEntity.Attributes)
{ {
deletedAttributes.ForEach(attribute => preexistingEntity.attributes.Remove(attribute.attribute)); deletedAttributes.ForEach(attribute => preexistingEntity.Attributes.Remove(attribute.attribute));
} }
} }
if (addedAttributes.Count != 0) if (addedAttributes.Count != 0)
{ {
// Insert // Insert
await DatabaseHelper.DatabaseInsertAttributes(helper, addedAttributes); await DatabaseHelper.DatabaseInsertAttributes(helper, addedAttributes);
lock (preexistingEntity.attributes) lock (preexistingEntity.Attributes)
{ {
addedAttributes.ForEach(attribute => preexistingEntity.attributes.Add(attribute.attributeKey, attribute.attribute)); addedAttributes.ForEach(attribute => preexistingEntity.Attributes.Add(attribute.attributeKey, attribute.attribute));
} }
} }
// Datapoint - get changes // Datapoint - get changes
List<Datapoint> deletedDatapointInstances = new(preexistingEntity.datapoints.Count); List<Datapoint> deletedDatapointInstances = new(preexistingEntity.Datapoints.Count);
List<string> deletedDatapoints = new(preexistingEntity.datapoints.Count); List<string> deletedDatapoints = new(preexistingEntity.Datapoints.Count);
List<(string datapointName, int entityId, JSONDatapoint jsonDatapoint, string hash)> updatedDatapointsText = new(preexistingEntity.datapoints.Count); List<(string datapointName, int entityId, JSONDatapoint jsonDatapoint, string hash)> updatedDatapointsText = new(preexistingEntity.Datapoints.Count);
List<(string datapointName, string probMethod, string similarityMethod, int entityId, JSONDatapoint jsonDatapoint)> updatedDatapointsNonText = new(preexistingEntity.datapoints.Count); List<(string datapointName, string probMethod, string similarityMethod, int entityId, JSONDatapoint jsonDatapoint)> updatedDatapointsNonText = new(preexistingEntity.Datapoints.Count);
List<Datapoint> createdDatapointInstances = []; List<Datapoint> createdDatapointInstances = [];
List<(string name, ProbMethodEnum probmethod_embedding, SimilarityMethodEnum similarityMethod, string hash, Dictionary<string, float[]> embeddings, JSONDatapoint datapoint)> createdDatapoints = new(jsonEntity.Datapoints.Length); List<(string name, ProbMethodEnum probmethod_embedding, SimilarityMethodEnum similarityMethod, string hash, Dictionary<string, float[]> embeddings, JSONDatapoint datapoint)> createdDatapoints = new(jsonEntity.Datapoints.Length);
foreach (Datapoint datapoint_ in preexistingEntity.datapoints.ToList()) foreach (Datapoint datapoint_ in preexistingEntity.Datapoints.ToList())
{ {
Datapoint datapoint = datapoint_; // To enable replacing the datapoint reference as foreach iterators cannot be overwritten Datapoint datapoint = datapoint_; // To enable replacing the datapoint reference as foreach iterators cannot be overwritten
JSONDatapoint? newEntityDatapoint = jsonEntity.Datapoints.FirstOrDefault(x => x.Name == datapoint.name); JSONDatapoint? newEntityDatapoint = jsonEntity.Datapoints.FirstOrDefault(x => x.Name == datapoint.Name);
bool newEntityHasDatapoint = newEntityDatapoint is not null; bool newEntityHasDatapoint = newEntityDatapoint is not null;
if (!newEntityHasDatapoint) if (!newEntityHasDatapoint)
{ {
// Datapoint - Deleted // Datapoint - Deleted
deletedDatapointInstances.Add(datapoint); deletedDatapointInstances.Add(datapoint);
deletedDatapoints.Add(datapoint.name); deletedDatapoints.Add(datapoint.Name);
invalidateSearchCache = true; invalidateSearchCache = true;
} else } else
{ {
@@ -231,7 +231,7 @@ public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelp
newEntityDatapoint is not null newEntityDatapoint is not null
&& newEntityDatapoint.Text is not null && newEntityDatapoint.Text is not null
&& hash is not null && hash is not null
&& hash != datapoint.hash) && hash != datapoint.Hash)
{ {
// Datapoint - Updated (text) // Datapoint - Updated (text)
updatedDatapointsText.Add(new() updatedDatapointsText.Add(new()
@@ -245,8 +245,8 @@ public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelp
} }
if ( if (
newEntityDatapoint is not null newEntityDatapoint is not null
&& (newEntityDatapoint.Probmethod_embedding != datapoint.probMethod.probMethodEnum && (newEntityDatapoint.Probmethod_embedding != datapoint.ProbMethod.ProbMethodEnum
|| newEntityDatapoint.SimilarityMethod != datapoint.similarityMethod.similarityMethodEnum)) || newEntityDatapoint.SimilarityMethod != datapoint.SimilarityMethod.SimilarityMethodEnum))
{ {
// Datapoint - Updated (probmethod or similaritymethod) // Datapoint - Updated (probmethod or similaritymethod)
updatedDatapointsNonText.Add(new() updatedDatapointsNonText.Add(new()
@@ -264,7 +264,7 @@ public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelp
foreach (JSONDatapoint jsonDatapoint in jsonEntity.Datapoints) foreach (JSONDatapoint jsonDatapoint in jsonEntity.Datapoints)
{ {
bool oldEntityHasDatapoint = preexistingEntity.datapoints.Any(x => x.name == jsonDatapoint.Name); bool oldEntityHasDatapoint = preexistingEntity.Datapoints.Any(x => x.Name == jsonDatapoint.Name);
if (!oldEntityHasDatapoint) if (!oldEntityHasDatapoint)
{ {
// Datapoint - New // Datapoint - New
@@ -290,13 +290,13 @@ public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelp
if (deletedDatapointInstances.Count != 0 || createdDatapoints.Count != 0 || addedAttributes.Count != 0 || updatedDatapointsNonText.Count != 0) if (deletedDatapointInstances.Count != 0 || createdDatapoints.Count != 0 || addedAttributes.Count != 0 || updatedDatapointsNonText.Count != 0)
_logger.LogDebug( _logger.LogDebug(
"EntityFromJSON - Updating existing entity. name: {name}, deletedDatapointInstances: {deletedDatapointInstances}, createdDatapoints: {createdDatapoints}, addedAttributes: {addedAttributes}, updatedDatapointsNonText: {updatedDatapointsNonText}", "EntityFromJSON - Updating existing entity. name: {name}, deletedDatapointInstances: {deletedDatapointInstances}, createdDatapoints: {createdDatapoints}, addedAttributes: {addedAttributes}, updatedDatapointsNonText: {updatedDatapointsNonText}",
[preexistingEntity.name, deletedDatapointInstances.Count, createdDatapoints.Count, addedAttributes.Count, updatedDatapointsNonText.Count]); [preexistingEntity.Name, deletedDatapointInstances.Count, createdDatapoints.Count, addedAttributes.Count, updatedDatapointsNonText.Count]);
// Datapoint - apply changes // Datapoint - apply changes
// Deleted // Deleted
if (deletedDatapointInstances.Count != 0) if (deletedDatapointInstances.Count != 0)
{ {
await DatabaseHelper.DatabaseDeleteEmbeddingsAndDatapoints(helper, deletedDatapoints, (int)preexistingEntityID); await DatabaseHelper.DatabaseDeleteEmbeddingsAndDatapoints(helper, deletedDatapoints, (int)preexistingEntityID);
preexistingEntity.datapoints = [.. preexistingEntity.datapoints preexistingEntity.Datapoints = [.. preexistingEntity.Datapoints
.Where(x => .Where(x =>
!deletedDatapointInstances.Contains(x) !deletedDatapointInstances.Contains(x)
) )
@@ -306,7 +306,7 @@ public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelp
if (createdDatapoints.Count != 0) if (createdDatapoints.Count != 0)
{ {
List<Datapoint> datapoint = await DatabaseInsertDatapointsWithEmbeddings(helper, searchdomain, [.. createdDatapoints.Select(element => (element.datapoint, element.hash))], (int)preexistingEntityID, id_searchdomain); List<Datapoint> datapoint = await DatabaseInsertDatapointsWithEmbeddings(helper, searchdomain, [.. createdDatapoints.Select(element => (element.datapoint, element.hash))], (int)preexistingEntityID, id_searchdomain);
datapoint.ForEach(x => preexistingEntity.datapoints.Add(x)); datapoint.ForEach(x => preexistingEntity.Datapoints.Add(x));
} }
// Datapoint - Updated (text) // Datapoint - Updated (text)
if (updatedDatapointsText.Count != 0) if (updatedDatapointsText.Count != 0)
@@ -317,14 +317,14 @@ public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelp
.Select(d => d.datapointName) .Select(d => d.datapointName)
.ToHashSet(); .ToHashSet();
var newBag = new ConcurrentBag<Datapoint>( var newBag = new ConcurrentBag<Datapoint>(
preexistingEntity.datapoints preexistingEntity.Datapoints
.Where(x => !namesToRemove.Contains(x.name)) .Where(x => !namesToRemove.Contains(x.Name))
); );
preexistingEntity.datapoints = newBag; preexistingEntity.Datapoints = newBag;
// Insert into database // Insert into database
List<Datapoint> datapoints = await DatabaseInsertDatapointsWithEmbeddings(helper, searchdomain, [.. updatedDatapointsText.Select(element => (datapoint: element.jsonDatapoint, hash: element.hash))], (int)preexistingEntityID, id_searchdomain); List<Datapoint> datapoints = await DatabaseInsertDatapointsWithEmbeddings(helper, searchdomain, [.. updatedDatapointsText.Select(element => (datapoint: element.jsonDatapoint, hash: element.hash))], (int)preexistingEntityID, id_searchdomain);
// Insert into datapoints // Insert into datapoints
datapoints.ForEach(datapoint => preexistingEntity.datapoints.Add(datapoint)); datapoints.ForEach(datapoint => preexistingEntity.Datapoints.Add(datapoint));
} }
// Datapoint - Updated (probmethod or similaritymethod) // Datapoint - Updated (probmethod or similaritymethod)
if (updatedDatapointsNonText.Count != 0) if (updatedDatapointsNonText.Count != 0)
@@ -336,9 +336,9 @@ public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelp
); );
updatedDatapointsNonText.ForEach(element => updatedDatapointsNonText.ForEach(element =>
{ {
Datapoint preexistingDatapoint = preexistingEntity.datapoints.First(x => x.name == element.datapointName); Datapoint preexistingDatapoint = preexistingEntity.Datapoints.First(x => x.Name == element.datapointName);
preexistingDatapoint.probMethod = new(element.jsonDatapoint.Probmethod_embedding); preexistingDatapoint.ProbMethod = new(element.jsonDatapoint.Probmethod_embedding);
preexistingDatapoint.similarityMethod = new(element.jsonDatapoint.SimilarityMethod); preexistingDatapoint.SimilarityMethod = new(element.jsonDatapoint.SimilarityMethod);
}); });
} }
@@ -368,7 +368,7 @@ public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelp
var insertAttributesTask = DatabaseHelper.DatabaseInsertAttributes(helper, toBeInsertedAttributes); var insertAttributesTask = DatabaseHelper.DatabaseInsertAttributes(helper, toBeInsertedAttributes);
List<(JSONDatapoint datapoint, string hash)> toBeInsertedDatapoints = []; List<(JSONDatapoint datapoint, string hash)> toBeInsertedDatapoints = [];
ConcurrentBag<string> usedModels = searchdomain.modelsInUse; ConcurrentBag<string> usedModels = searchdomain.ModelsInUse;
foreach (JSONDatapoint jsonDatapoint in jsonEntity.Datapoints) foreach (JSONDatapoint jsonDatapoint in jsonEntity.Datapoints)
{ {
string hash = Convert.ToBase64String(SHA256.HashData(Encoding.UTF8.GetBytes(jsonDatapoint.Text))); string hash = Convert.ToBase64String(SHA256.HashData(Encoding.UTF8.GetBytes(jsonDatapoint.Text)));
@@ -391,7 +391,7 @@ public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelp
var probMethod = Probmethods.GetMethod(jsonEntity.Probmethod) ?? throw new ProbMethodNotFoundException(jsonEntity.Probmethod); var probMethod = Probmethods.GetMethod(jsonEntity.Probmethod) ?? throw new ProbMethodNotFoundException(jsonEntity.Probmethod);
Entity entity = new(jsonEntity.Attributes, probMethod, jsonEntity.Probmethod.ToString(), [.. datapoints], jsonEntity.Name, jsonEntity.Searchdomain) Entity entity = new(jsonEntity.Attributes, probMethod, jsonEntity.Probmethod.ToString(), [.. datapoints], jsonEntity.Name, jsonEntity.Searchdomain)
{ {
id = id_entity Id = id_entity
}; };
entityCache[jsonEntity.Name] = entity; entityCache[jsonEntity.Name] = entity;
@@ -412,16 +412,16 @@ public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelp
toBeInsertedDatapoints.Add(new() toBeInsertedDatapoints.Add(new()
{ {
name = datapoint.name, name = datapoint.Name,
probmethod_embedding = datapoint.probMethod.probMethodEnum, probmethod_embedding = datapoint.ProbMethod.ProbMethodEnum,
similarityMethod = datapoint.similarityMethod.similarityMethodEnum, similarityMethod = datapoint.SimilarityMethod.SimilarityMethodEnum,
hash = value.hash hash = value.hash
}); });
foreach ((string, float[]) embedding in datapoint.embeddings) foreach ((string, float[]) embedding in datapoint.Embeddings)
{ {
toBeInsertedEmbeddings.Add(new() toBeInsertedEmbeddings.Add(new()
{ {
id_datapoint = datapoint.id, id_datapoint = datapoint.Id,
model = embedding.Item1, model = embedding.Item1,
embedding = BytesFromFloatArray(embedding.Item2) embedding = BytesFromFloatArray(embedding.Item2)
}); });
@@ -444,7 +444,7 @@ public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelp
Datapoint datapoint = await BuildDatapointFromJsonDatapoint(jsonDatapoint, id_entity, searchdomain, hash); Datapoint datapoint = await BuildDatapointFromJsonDatapoint(jsonDatapoint, id_entity, searchdomain, hash);
int id_datapoint = await DatabaseHelper.DatabaseInsertDatapoint(helper, jsonDatapoint.Name, jsonDatapoint.Probmethod_embedding, jsonDatapoint.SimilarityMethod, hash, id_entity); // TODO make this a bulk add action to reduce number of queries int id_datapoint = await DatabaseHelper.DatabaseInsertDatapoint(helper, jsonDatapoint.Name, jsonDatapoint.Probmethod_embedding, jsonDatapoint.SimilarityMethod, hash, id_entity); // TODO make this a bulk add action to reduce number of queries
List<(string model, byte[] embedding)> data = []; List<(string model, byte[] embedding)> data = [];
foreach ((string, float[]) embedding in datapoint.embeddings) foreach ((string, float[]) embedding in datapoint.Embeddings)
{ {
data.Add((embedding.Item1, BytesFromFloatArray(embedding.Item2))); data.Add((embedding.Item1, BytesFromFloatArray(embedding.Item2)));
} }
@@ -463,11 +463,11 @@ public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelp
{ {
throw new Exception("jsonDatapoint.Text must not be null at this point"); throw new Exception("jsonDatapoint.Text must not be null at this point");
} }
SQLHelper helper = searchdomain.helper; SQLHelper helper = searchdomain.Helper;
EnumerableLruCache<string, Dictionary<string, float[]>> embeddingCache = searchdomain.embeddingCache; EnumerableLruCache<string, Dictionary<string, float[]>> embeddingCache = searchdomain.EmbeddingCache;
hash ??= Convert.ToBase64String(SHA256.HashData(Encoding.UTF8.GetBytes(jsonDatapoint.Text))); hash ??= Convert.ToBase64String(SHA256.HashData(Encoding.UTF8.GetBytes(jsonDatapoint.Text)));
int id = await DatabaseHelper.DatabaseInsertDatapoint(helper, jsonDatapoint.Name, jsonDatapoint.Probmethod_embedding, jsonDatapoint.SimilarityMethod, hash, entityId); int id = await DatabaseHelper.DatabaseInsertDatapoint(helper, jsonDatapoint.Name, jsonDatapoint.Probmethod_embedding, jsonDatapoint.SimilarityMethod, hash, entityId);
Dictionary<string, float[]> embeddings = Datapoint.GetEmbeddings(jsonDatapoint.Text, [.. jsonDatapoint.Model], searchdomain.aIProvider, embeddingCache); Dictionary<string, float[]> embeddings = Datapoint.GetEmbeddings(jsonDatapoint.Text, [.. jsonDatapoint.Model], searchdomain.AiProvider, embeddingCache);
var probMethod_embedding = new ProbMethod(jsonDatapoint.Probmethod_embedding) ?? throw new ProbMethodNotFoundException(jsonDatapoint.Probmethod_embedding); var probMethod_embedding = new ProbMethod(jsonDatapoint.Probmethod_embedding) ?? throw new ProbMethodNotFoundException(jsonDatapoint.Probmethod_embedding);
var similarityMethod = new SimilarityMethod(jsonDatapoint.SimilarityMethod) ?? throw new SimilarityMethodNotFoundException(jsonDatapoint.SimilarityMethod); var similarityMethod = new SimilarityMethod(jsonDatapoint.SimilarityMethod) ?? throw new SimilarityMethodNotFoundException(jsonDatapoint.SimilarityMethod);
return new Datapoint(jsonDatapoint.Name, probMethod_embedding, similarityMethod, hash, [.. embeddings.Select(kv => (kv.Key, kv.Value))], id); return new Datapoint(jsonDatapoint.Name, probMethod_embedding, similarityMethod, hash, [.. embeddings.Select(kv => (kv.Key, kv.Value))], id);

28
src/Server/Probmethods.cs
View File

@@ -6,29 +6,29 @@ namespace Server;
public class ProbMethod public class ProbMethod
{ {
public Probmethods.probMethodDelegate method; public Probmethods.ProbMethodDelegate Method;
public ProbMethodEnum probMethodEnum; public ProbMethodEnum ProbMethodEnum;
public string name; public string Name;
public ProbMethod(ProbMethodEnum probMethodEnum) public ProbMethod(ProbMethodEnum probMethodEnum)
{ {
this.probMethodEnum = probMethodEnum; this.ProbMethodEnum = probMethodEnum;
this.name = probMethodEnum.ToString(); this.Name = probMethodEnum.ToString();
Probmethods.probMethodDelegate? probMethod = Probmethods.GetMethod(name) ?? throw new ProbMethodNotFoundException(probMethodEnum); Probmethods.ProbMethodDelegate? probMethod = Probmethods.GetMethod(Name) ?? throw new ProbMethodNotFoundException(probMethodEnum);
method = probMethod; Method = probMethod;
} }
} }
public static class Probmethods public static class Probmethods
{ {
public delegate float probMethodProtoDelegate(List<(string, float)> list, string parameters); public delegate float ProbMethodProtoDelegate(List<(string, float)> list, string parameters);
public delegate float probMethodDelegate(List<(string, float)> list); public delegate float ProbMethodDelegate(List<(string, float)> list);
public static readonly Dictionary<ProbMethodEnum, probMethodProtoDelegate> probMethods; public static readonly Dictionary<ProbMethodEnum, ProbMethodProtoDelegate> ProbMethods;
static Probmethods() static Probmethods()
{ {
probMethods = new Dictionary<ProbMethodEnum, probMethodProtoDelegate> ProbMethods = new Dictionary<ProbMethodEnum, ProbMethodProtoDelegate>
{ {
[ProbMethodEnum.Mean] = Mean, [ProbMethodEnum.Mean] = Mean,
[ProbMethodEnum.HarmonicMean] = HarmonicMean, [ProbMethodEnum.HarmonicMean] = HarmonicMean,
@@ -41,12 +41,12 @@ public static class Probmethods
}; };
} }
public static probMethodDelegate? GetMethod(ProbMethodEnum probMethodEnum) public static ProbMethodDelegate? GetMethod(ProbMethodEnum probMethodEnum)
{ {
return GetMethod(probMethodEnum.ToString()); return GetMethod(probMethodEnum.ToString());
} }
public static probMethodDelegate? GetMethod(string name) public static ProbMethodDelegate? GetMethod(string name)
{ {
string methodName = name; string methodName = name;
string? jsonArg = ""; string? jsonArg = "";
@@ -63,7 +63,7 @@ public static class Probmethods
methodName methodName
); );
if (!probMethods.TryGetValue(probMethodEnum, out probMethodProtoDelegate? method)) if (!ProbMethods.TryGetValue(probMethodEnum, out ProbMethodProtoDelegate? method))
{ {
return null; return null;
} }

120
src/Server/Searchdomain.cs
View File

@@ -15,33 +15,33 @@ public class Searchdomain
{ {
private readonly string _connectionString; private readonly string _connectionString;
private readonly string _provider; private readonly string _provider;
public AIProvider aIProvider; public AIProvider AiProvider;
public string searchdomain; public string SearchdomainName;
public int id; public int Id;
public SearchdomainSettings settings; public SearchdomainSettings Settings;
public EnumerableLruCache<string, DateTimedSearchResult> queryCache; // Key: query, Value: Search results for that query (with timestamp) public EnumerableLruCache<string, DateTimedSearchResult> QueryCache; // Key: query, Value: Search results for that query (with timestamp)
public ConcurrentDictionary<string, Entity> entityCache; public ConcurrentDictionary<string, Entity> EntityCache;
public ConcurrentBag<string> modelsInUse; public ConcurrentBag<string> ModelsInUse;
public EnumerableLruCache<string, Dictionary<string, float[]>> embeddingCache; public EnumerableLruCache<string, Dictionary<string, float[]>> EmbeddingCache;
public SQLHelper helper; public SQLHelper Helper;
private readonly ILogger _logger; private readonly ILogger _logger;
public Searchdomain(string searchdomain, string connectionString, SQLHelper sqlHelper, AIProvider aIProvider, EnumerableLruCache<string, Dictionary<string, float[]>> embeddingCache, ILogger logger, string provider = "sqlserver", bool runEmpty = false) public Searchdomain(string searchdomain, string connectionString, SQLHelper sqlHelper, AIProvider aIProvider, EnumerableLruCache<string, Dictionary<string, float[]>> embeddingCache, ILogger logger, string provider = "sqlserver", bool runEmpty = false)
{ {
_connectionString = connectionString; _connectionString = connectionString;
_provider = provider.ToLower(); _provider = provider.ToLower();
this.searchdomain = searchdomain; this.SearchdomainName = searchdomain;
this.aIProvider = aIProvider; this.AiProvider = aIProvider;
this.embeddingCache = embeddingCache; this.EmbeddingCache = embeddingCache;
this._logger = logger; this._logger = logger;
entityCache = []; EntityCache = [];
helper = sqlHelper; Helper = sqlHelper;
settings = GetSettings(); Settings = GetSettings();
queryCache = new(settings.QueryCacheSize); QueryCache = new(Settings.QueryCacheSize);
modelsInUse = []; // To make the compiler shut up - it is set in UpdateSearchDomain() don't worry // yeah, about that... ModelsInUse = []; // To make the compiler shut up - it is set in UpdateSearchDomain() don't worry // yeah, about that...
if (!runEmpty) if (!runEmpty)
{ {
id = GetID().Result; Id = GetID().Result;
UpdateEntityCache(); UpdateEntityCache();
} }
} }
@@ -51,9 +51,9 @@ public class Searchdomain
InvalidateSearchCache(); InvalidateSearchCache();
Dictionary<string, dynamic> parametersIDSearchdomain = new() Dictionary<string, dynamic> parametersIDSearchdomain = new()
{ {
["id"] = this.id ["id"] = this.Id
}; };
DbDataReader embeddingReader = helper.ExecuteSQLCommand("SELECT id, id_datapoint, model, embedding FROM embedding WHERE id_searchdomain = @id", parametersIDSearchdomain); DbDataReader embeddingReader = Helper.ExecuteSQLCommand("SELECT id, id_datapoint, model, embedding FROM embedding WHERE id_searchdomain = @id", parametersIDSearchdomain);
Dictionary<int, Dictionary<string, float[]>> embedding_unassigned = []; Dictionary<int, Dictionary<string, float[]>> embedding_unassigned = [];
try try
{ {
@@ -90,7 +90,7 @@ public class Searchdomain
embeddingReader.Close(); embeddingReader.Close();
} }
DbDataReader datapointReader = helper.ExecuteSQLCommand("SELECT d.id, d.id_entity, d.name, d.probmethod_embedding, d.similaritymethod, d.hash FROM datapoint d JOIN entity ent ON d.id_entity = ent.id JOIN searchdomain s ON ent.id_searchdomain = s.id WHERE s.id = @id", parametersIDSearchdomain); DbDataReader datapointReader = Helper.ExecuteSQLCommand("SELECT d.id, d.id_entity, d.name, d.probmethod_embedding, d.similaritymethod, d.hash FROM datapoint d JOIN entity ent ON d.id_entity = ent.id JOIN searchdomain s ON ent.id_searchdomain = s.id WHERE s.id = @id", parametersIDSearchdomain);
Dictionary<int, ConcurrentBag<Datapoint>> datapoint_unassigned = []; Dictionary<int, ConcurrentBag<Datapoint>> datapoint_unassigned = [];
try try
{ {
@@ -127,7 +127,7 @@ public class Searchdomain
datapointReader.Close(); datapointReader.Close();
} }
DbDataReader attributeReader = helper.ExecuteSQLCommand("SELECT a.id, a.id_entity, a.attribute, a.value FROM attribute a JOIN entity ent ON a.id_entity = ent.id JOIN searchdomain s ON ent.id_searchdomain = s.id WHERE s.id = @id", parametersIDSearchdomain); DbDataReader attributeReader = Helper.ExecuteSQLCommand("SELECT a.id, a.id_entity, a.attribute, a.value FROM attribute a JOIN entity ent ON a.id_entity = ent.id JOIN searchdomain s ON ent.id_searchdomain = s.id WHERE s.id = @id", parametersIDSearchdomain);
Dictionary<int, Dictionary<string, string>> attributes_unassigned = []; Dictionary<int, Dictionary<string, string>> attributes_unassigned = [];
try try
{ {
@@ -149,8 +149,8 @@ public class Searchdomain
attributeReader.Close(); attributeReader.Close();
} }
entityCache = []; EntityCache = [];
DbDataReader entityReader = helper.ExecuteSQLCommand("SELECT entity.id, name, probmethod FROM entity WHERE id_searchdomain=@id", parametersIDSearchdomain); DbDataReader entityReader = Helper.ExecuteSQLCommand("SELECT entity.id, name, probmethod FROM entity WHERE id_searchdomain=@id", parametersIDSearchdomain);
try try
{ {
while (entityReader.Read()) while (entityReader.Read())
@@ -163,26 +163,26 @@ public class Searchdomain
{ {
attributes = []; attributes = [];
} }
Probmethods.probMethodDelegate? probmethod = Probmethods.GetMethod(probmethodString); Probmethods.ProbMethodDelegate? probmethod = Probmethods.GetMethod(probmethodString);
if (datapoint_unassigned.TryGetValue(id, out ConcurrentBag<Datapoint>? datapoints) && probmethod is not null) if (datapoint_unassigned.TryGetValue(id, out ConcurrentBag<Datapoint>? datapoints) && probmethod is not null)
{ {
Entity entity = new(attributes, probmethod, probmethodString, datapoints, name, searchdomain) Entity entity = new(attributes, probmethod, probmethodString, datapoints, name, SearchdomainName)
{ {
id = id Id = id
}; };
entityCache[name] = entity; EntityCache[name] = entity;
} }
} }
} finally } finally
{ {
entityReader.Close(); entityReader.Close();
} }
modelsInUse = GetModels(entityCache); ModelsInUse = GetModels(EntityCache);
} }
public List<(float, string)> Search(string query, int? topN = null) public List<(float, string)> Search(string query, int? topN = null)
{ {
if (queryCache.TryGetValue(query, out DateTimedSearchResult cachedResult)) if (QueryCache.TryGetValue(query, out DateTimedSearchResult cachedResult))
{ {
cachedResult.AccessDateTimes.Add(DateTime.Now); cachedResult.AccessDateTimes.Add(DateTime.Now);
return [.. cachedResult.Results.Select(r => (r.Score, r.Name))]; return [.. cachedResult.Results.Select(r => (r.Score, r.Name))];
@@ -191,9 +191,9 @@ public class Searchdomain
Dictionary<string, float[]> queryEmbeddings = GetQueryEmbeddings(query); Dictionary<string, float[]> queryEmbeddings = GetQueryEmbeddings(query);
List<(float, string)> result = []; List<(float, string)> result = [];
foreach ((string name, Entity entity) in entityCache) foreach ((string name, Entity entity) in EntityCache)
{ {
result.Add((EvaluateEntityAgainstQueryEmbeddings(entity, queryEmbeddings), entity.name)); result.Add((EvaluateEntityAgainstQueryEmbeddings(entity, queryEmbeddings), entity.Name));
} }
IEnumerable<(float, string)> sortedResults = result.OrderByDescending(s => s.Item1); IEnumerable<(float, string)> sortedResults = result.OrderByDescending(s => s.Item1);
if (topN is not null) if (topN is not null)
@@ -205,26 +205,26 @@ public class Searchdomain
[.. sortedResults.Select(r => [.. sortedResults.Select(r =>
new ResultItem(r.Item1, r.Item2 ))] new ResultItem(r.Item1, r.Item2 ))]
); );
queryCache.Set(query, new DateTimedSearchResult(DateTime.Now, searchResult)); QueryCache.Set(query, new DateTimedSearchResult(DateTime.Now, searchResult));
return results; return results;
} }
public Dictionary<string, float[]> GetQueryEmbeddings(string query) public Dictionary<string, float[]> GetQueryEmbeddings(string query)
{ {
bool hasQuery = embeddingCache.TryGetValue(query, out Dictionary<string, float[]>? queryEmbeddings); bool hasQuery = EmbeddingCache.TryGetValue(query, out Dictionary<string, float[]>? queryEmbeddings);
bool allModelsInQuery = queryEmbeddings is not null && modelsInUse.All(model => queryEmbeddings.ContainsKey(model)); bool allModelsInQuery = queryEmbeddings is not null && ModelsInUse.All(model => queryEmbeddings.ContainsKey(model));
if (!(hasQuery && allModelsInQuery) || queryEmbeddings is null) if (!(hasQuery && allModelsInQuery) || queryEmbeddings is null)
{ {
queryEmbeddings = Datapoint.GetEmbeddings(query, modelsInUse, aIProvider, embeddingCache); queryEmbeddings = Datapoint.GetEmbeddings(query, ModelsInUse, AiProvider, EmbeddingCache);
if (!embeddingCache.TryGetValue(query, out var embeddingCacheForCurrentQuery)) if (!EmbeddingCache.TryGetValue(query, out var embeddingCacheForCurrentQuery))
{ {
embeddingCache.Set(query, queryEmbeddings); EmbeddingCache.Set(query, queryEmbeddings);
} }
else // embeddingCache already has an entry for this query, so the missing model-embedding pairs have to be filled in else // embeddingCache already has an entry for this query, so the missing model-embedding pairs have to be filled in
{ {
foreach (KeyValuePair<string, float[]> kvp in queryEmbeddings) // kvp.Key = model, kvp.Value = embedding foreach (KeyValuePair<string, float[]> kvp in queryEmbeddings) // kvp.Key = model, kvp.Value = embedding
{ {
if (!embeddingCache.TryGetValue(kvp.Key, out var _)) if (!EmbeddingCache.TryGetValue(kvp.Key, out var _))
{ {
embeddingCacheForCurrentQuery[kvp.Key] = kvp.Value; embeddingCacheForCurrentQuery[kvp.Key] = kvp.Value;
} }
@@ -236,25 +236,25 @@ public class Searchdomain
public void UpdateModelsInUse() public void UpdateModelsInUse()
{ {
modelsInUse = GetModels(entityCache); ModelsInUse = GetModels(EntityCache);
} }
private static float EvaluateEntityAgainstQueryEmbeddings(Entity entity, Dictionary<string, float[]> queryEmbeddings) private static float EvaluateEntityAgainstQueryEmbeddings(Entity entity, Dictionary<string, float[]> queryEmbeddings)
{ {
List<(string, float)> datapointProbs = []; List<(string, float)> datapointProbs = [];
foreach (Datapoint datapoint in entity.datapoints) foreach (Datapoint datapoint in entity.Datapoints)
{ {
SimilarityMethod similarityMethod = datapoint.similarityMethod; SimilarityMethod similarityMethod = datapoint.SimilarityMethod;
List<(string, float)> list = []; List<(string, float)> list = [];
foreach ((string, float[]) embedding in datapoint.embeddings) foreach ((string, float[]) embedding in datapoint.Embeddings)
{ {
string key = embedding.Item1; string key = embedding.Item1;
float value = similarityMethod.method(queryEmbeddings[embedding.Item1], embedding.Item2); float value = similarityMethod.Method(queryEmbeddings[embedding.Item1], embedding.Item2);
list.Add((key, value)); list.Add((key, value));
} }
datapointProbs.Add((datapoint.name, datapoint.probMethod.method(list))); datapointProbs.Add((datapoint.Name, datapoint.ProbMethod.Method(list)));
} }
return entity.probMethod(datapointProbs); return entity.ProbMethod(datapointProbs);
} }
public static ConcurrentBag<string> GetModels(ConcurrentDictionary<string, Entity> entities) public static ConcurrentBag<string> GetModels(ConcurrentDictionary<string, Entity> entities)
@@ -265,9 +265,9 @@ public class Searchdomain
Entity entity = element.Value; Entity entity = element.Value;
lock (entity) lock (entity)
{ {
foreach (Datapoint datapoint in entity.datapoints) foreach (Datapoint datapoint in entity.Datapoints)
{ {
foreach ((string, float[]) tuple in datapoint.embeddings) foreach ((string, float[]) tuple in datapoint.Embeddings)
{ {
string model = tuple.Item1; string model = tuple.Item1;
if (!result.Contains(model)) if (!result.Contains(model))
@@ -285,21 +285,21 @@ public class Searchdomain
{ {
Dictionary<string, object?> parameters = new() Dictionary<string, object?> parameters = new()
{ {
{ "name", this.searchdomain } { "name", this.SearchdomainName }
}; };
return (await helper.ExecuteQueryAsync("SELECT id from searchdomain WHERE name = @name", parameters, x => x.GetInt32(0))).First(); return (await Helper.ExecuteQueryAsync("SELECT id from searchdomain WHERE name = @name", parameters, x => x.GetInt32(0))).First();
} }
public SearchdomainSettings GetSettings() public SearchdomainSettings GetSettings()
{ {
return DatabaseHelper.GetSearchdomainSettings(helper, searchdomain); return DatabaseHelper.GetSearchdomainSettings(Helper, SearchdomainName);
} }
public void ReconciliateOrInvalidateCacheForNewOrUpdatedEntity(Entity entity) public void ReconciliateOrInvalidateCacheForNewOrUpdatedEntity(Entity entity)
{ {
if (settings.CacheReconciliation) if (Settings.CacheReconciliation)
{ {
foreach (var element in queryCache) foreach (var element in QueryCache)
{ {
string query = element.Key; string query = element.Key;
DateTimedSearchResult searchResult = element.Value; DateTimedSearchResult searchResult = element.Value;
@@ -307,9 +307,9 @@ public class Searchdomain
Dictionary<string, float[]> queryEmbeddings = GetQueryEmbeddings(query); Dictionary<string, float[]> queryEmbeddings = GetQueryEmbeddings(query);
float evaluationResult = EvaluateEntityAgainstQueryEmbeddings(entity, queryEmbeddings); float evaluationResult = EvaluateEntityAgainstQueryEmbeddings(entity, queryEmbeddings);
searchResult.Results.RemoveAll(x => x.Name == entity.name); // If entity already exists in that results list: remove it. searchResult.Results.RemoveAll(x => x.Name == entity.Name); // If entity already exists in that results list: remove it.
ResultItem newItem = new(evaluationResult, entity.name); ResultItem newItem = new(evaluationResult, entity.Name);
int index = searchResult.Results.BinarySearch( int index = searchResult.Results.BinarySearch(
newItem, newItem,
Comparer<ResultItem>.Create((a, b) => b.Score.CompareTo(a.Score)) // Invert searching order Comparer<ResultItem>.Create((a, b) => b.Score.CompareTo(a.Score)) // Invert searching order
@@ -327,13 +327,13 @@ public class Searchdomain
public void ReconciliateOrInvalidateCacheForDeletedEntity(Entity entity) public void ReconciliateOrInvalidateCacheForDeletedEntity(Entity entity)
{ {
if (settings.CacheReconciliation) if (Settings.CacheReconciliation)
{ {
foreach (KeyValuePair<string, DateTimedSearchResult> element in queryCache) foreach (KeyValuePair<string, DateTimedSearchResult> element in QueryCache)
{ {
string query = element.Key; string query = element.Key;
DateTimedSearchResult searchResult = element.Value; DateTimedSearchResult searchResult = element.Value;
searchResult.Results.RemoveAll(x => x.Name == entity.name); searchResult.Results.RemoveAll(x => x.Name == entity.Name);
} }
} }
else else
@@ -344,13 +344,13 @@ public class Searchdomain
public void InvalidateSearchCache() public void InvalidateSearchCache()
{ {
queryCache = new(settings.QueryCacheSize); QueryCache = new(Settings.QueryCacheSize);
} }
public long GetSearchCacheSize() public long GetSearchCacheSize()
{ {
long EmbeddingCacheUtilization = 0; long EmbeddingCacheUtilization = 0;
foreach (var entry in queryCache) foreach (var entry in QueryCache)
{ {
EmbeddingCacheUtilization += sizeof(int); // string length prefix EmbeddingCacheUtilization += sizeof(int); // string length prefix
EmbeddingCacheUtilization += entry.Key.Length * sizeof(char); // string characters EmbeddingCacheUtilization += entry.Key.Length * sizeof(char); // string characters

52
src/Server/SearchdomainManager.cs
View File

@@ -15,50 +15,50 @@ namespace Server;
public class SearchdomainManager : IDisposable public class SearchdomainManager : IDisposable
{ {
private Dictionary<string, Searchdomain> searchdomains = []; private Dictionary<string, Searchdomain> _searchdomains = [];
private readonly ILogger<SearchdomainManager> _logger; private readonly ILogger<SearchdomainManager> _logger;
private readonly EmbeddingSearchOptions _options; private readonly EmbeddingSearchOptions _options;
public readonly AIProvider aIProvider; public readonly AIProvider AiProvider;
private readonly DatabaseHelper _databaseHelper; private readonly DatabaseHelper _databaseHelper;
private readonly string connectionString; private readonly string connectionString;
private MySqlConnection connection; private MySqlConnection _connection;
public SQLHelper helper; public SQLHelper Helper;
public EnumerableLruCache<string, Dictionary<string, float[]>> embeddingCache; public EnumerableLruCache<string, Dictionary<string, float[]>> EmbeddingCache;
public long EmbeddingCacheMaxCount; public long EmbeddingCacheMaxCount;
private bool disposed = false; private bool _disposed = false;
public SearchdomainManager(ILogger<SearchdomainManager> logger, IOptions<EmbeddingSearchOptions> options, AIProvider aIProvider, DatabaseHelper databaseHelper) public SearchdomainManager(ILogger<SearchdomainManager> logger, IOptions<EmbeddingSearchOptions> options, AIProvider aIProvider, DatabaseHelper databaseHelper)
{ {
_logger = logger; _logger = logger;
_options = options.Value; _options = options.Value;
this.aIProvider = aIProvider; this.AiProvider = aIProvider;
_databaseHelper = databaseHelper; _databaseHelper = databaseHelper;
EmbeddingCacheMaxCount = _options.Cache.CacheTopN; EmbeddingCacheMaxCount = _options.Cache.CacheTopN;
if (options.Value.Cache.StoreEmbeddingCache) if (options.Value.Cache.StoreEmbeddingCache)
{ {
var stopwatch = Stopwatch.StartNew(); var stopwatch = Stopwatch.StartNew();
embeddingCache = CacheHelper.GetEmbeddingStore(options.Value); EmbeddingCache = CacheHelper.GetEmbeddingStore(options.Value);
stopwatch.Stop(); stopwatch.Stop();
_logger.LogInformation("GetEmbeddingStore completed in {ElapsedMilliseconds} ms", stopwatch.ElapsedMilliseconds); _logger.LogInformation("GetEmbeddingStore completed in {ElapsedMilliseconds} ms", stopwatch.ElapsedMilliseconds);
} else } else
{ {
embeddingCache = new((int)EmbeddingCacheMaxCount); EmbeddingCache = new((int)EmbeddingCacheMaxCount);
} }
connectionString = _options.ConnectionStrings.SQL; connectionString = _options.ConnectionStrings.SQL;
connection = new MySqlConnection(connectionString); _connection = new MySqlConnection(connectionString);
connection.Open(); _connection.Open();
helper = new SQLHelper(connection, connectionString); Helper = new SQLHelper(_connection, connectionString);
} }
public Searchdomain GetSearchdomain(string searchdomain) public Searchdomain GetSearchdomain(string searchdomain)
{ {
if (searchdomains.TryGetValue(searchdomain, out Searchdomain? value)) if (_searchdomains.TryGetValue(searchdomain, out Searchdomain? value))
{ {
return value; return value;
} }
try try
{ {
return SetSearchdomain(searchdomain, new Searchdomain(searchdomain, connectionString, helper, aIProvider, embeddingCache, _logger)); return SetSearchdomain(searchdomain, new Searchdomain(searchdomain, connectionString, Helper, AiProvider, EmbeddingCache, _logger));
} }
catch (MySqlException) catch (MySqlException)
{ {
@@ -81,7 +81,7 @@ public class SearchdomainManager : IDisposable
public async Task<List<string>> ListSearchdomainsAsync() public async Task<List<string>> ListSearchdomainsAsync()
{ {
return await helper.ExecuteQueryAsync("SELECT name FROM searchdomain", [], x => x.GetString(0)); return await Helper.ExecuteQueryAsync("SELECT name FROM searchdomain", [], x => x.GetString(0));
} }
public async Task<int> CreateSearchdomain(string searchdomain, SearchdomainSettings settings) public async Task<int> CreateSearchdomain(string searchdomain, SearchdomainSettings settings)
@@ -91,7 +91,7 @@ public class SearchdomainManager : IDisposable
public async Task<int> CreateSearchdomain(string searchdomain, string settings = "{}") public async Task<int> CreateSearchdomain(string searchdomain, string settings = "{}")
{ {
if (searchdomains.TryGetValue(searchdomain, out Searchdomain? value)) if (_searchdomains.TryGetValue(searchdomain, out Searchdomain? value))
{ {
_logger.LogError("Searchdomain {searchdomain} could not be created, as it already exists", [searchdomain]); _logger.LogError("Searchdomain {searchdomain} could not be created, as it already exists", [searchdomain]);
throw new SearchdomainAlreadyExistsException(searchdomain); throw new SearchdomainAlreadyExistsException(searchdomain);
@@ -101,30 +101,30 @@ public class SearchdomainManager : IDisposable
{ "name", searchdomain }, { "name", searchdomain },
{ "settings", settings} { "settings", settings}
}; };
int id = await helper.ExecuteSQLCommandGetInsertedID("INSERT INTO searchdomain (name, settings) VALUES (@name, @settings)", parameters); int id = await Helper.ExecuteSQLCommandGetInsertedID("INSERT INTO searchdomain (name, settings) VALUES (@name, @settings)", parameters);
searchdomains.Add(searchdomain, new(searchdomain, connectionString, helper, aIProvider, embeddingCache, _logger)); _searchdomains.Add(searchdomain, new(searchdomain, connectionString, Helper, AiProvider, EmbeddingCache, _logger));
return id; return id;
} }
public async Task<int> DeleteSearchdomain(string searchdomain) public async Task<int> DeleteSearchdomain(string searchdomain)
{ {
int counter = await _databaseHelper.RemoveAllEntities(helper, searchdomain); int counter = await _databaseHelper.RemoveAllEntities(Helper, searchdomain);
_logger.LogDebug($"Number of entities deleted as part of deleting the searchdomain \"{searchdomain}\": {counter}"); _logger.LogDebug($"Number of entities deleted as part of deleting the searchdomain \"{searchdomain}\": {counter}");
await helper.ExecuteSQLNonQuery("DELETE FROM searchdomain WHERE name = @name", new() {{"name", searchdomain}}); await Helper.ExecuteSQLNonQuery("DELETE FROM searchdomain WHERE name = @name", new() {{"name", searchdomain}});
searchdomains.Remove(searchdomain); _searchdomains.Remove(searchdomain);
_logger.LogDebug($"Searchdomain has been successfully removed"); _logger.LogDebug($"Searchdomain has been successfully removed");
return counter; return counter;
} }
private Searchdomain SetSearchdomain(string name, Searchdomain searchdomain) private Searchdomain SetSearchdomain(string name, Searchdomain searchdomain)
{ {
searchdomains[name] = searchdomain; _searchdomains[name] = searchdomain;
return searchdomain; return searchdomain;
} }
public bool IsSearchdomainLoaded(string name) public bool IsSearchdomainLoaded(string name)
{ {
return searchdomains.ContainsKey(name); return _searchdomains.ContainsKey(name);
} }
// Cleanup procedure // Cleanup procedure
@@ -135,7 +135,7 @@ public class SearchdomainManager : IDisposable
if (_options.Cache.StoreEmbeddingCache) if (_options.Cache.StoreEmbeddingCache)
{ {
var stopwatch = Stopwatch.StartNew(); var stopwatch = Stopwatch.StartNew();
await CacheHelper.UpdateEmbeddingStore(embeddingCache, _options); await CacheHelper.UpdateEmbeddingStore(EmbeddingCache, _options);
stopwatch.Stop(); stopwatch.Stop();
_logger.LogInformation("UpdateEmbeddingStore completed in {ElapsedMilliseconds} ms", stopwatch.ElapsedMilliseconds); _logger.LogInformation("UpdateEmbeddingStore completed in {ElapsedMilliseconds} ms", stopwatch.ElapsedMilliseconds);
} }
@@ -155,10 +155,10 @@ public class SearchdomainManager : IDisposable
protected virtual async Task Dispose(bool disposing) protected virtual async Task Dispose(bool disposing)
{ {
if (!disposed && disposing) if (!_disposed && disposing)
{ {
await Cleanup(); await Cleanup();
disposed = true; _disposed = true;
} }
} }
} }

20
src/Server/SimilarityMethods.cs
View File

@@ -5,16 +5,16 @@ namespace Server;
public class SimilarityMethod public class SimilarityMethod
{ {
public SimilarityMethods.similarityMethodDelegate method; public SimilarityMethods.similarityMethodDelegate Method;
public SimilarityMethodEnum similarityMethodEnum; public SimilarityMethodEnum SimilarityMethodEnum;
public string name; public string Name;
public SimilarityMethod(SimilarityMethodEnum similarityMethodEnum) public SimilarityMethod(SimilarityMethodEnum similarityMethodEnum)
{ {
this.similarityMethodEnum = similarityMethodEnum; SimilarityMethodEnum = similarityMethodEnum;
this.name = similarityMethodEnum.ToString(); Name = similarityMethodEnum.ToString();
SimilarityMethods.similarityMethodDelegate? probMethod = SimilarityMethods.GetMethod(name) ?? throw new Exception($"Unable to retrieve similarityMethod {name}"); SimilarityMethods.similarityMethodDelegate? probMethod = SimilarityMethods.GetMethod(Name) ?? throw new Exception($"Unable to retrieve similarityMethod {Name}");
method = probMethod; Method = probMethod;
} }
} }
@@ -22,11 +22,11 @@ public static class SimilarityMethods
{ {
public delegate float similarityMethodProtoDelegate(float[] vector1, float[] vector2); public delegate float similarityMethodProtoDelegate(float[] vector1, float[] vector2);
public delegate float similarityMethodDelegate(float[] vector1, float[] vector2); public delegate float similarityMethodDelegate(float[] vector1, float[] vector2);
public static readonly Dictionary<SimilarityMethodEnum, similarityMethodProtoDelegate> probMethods; public static readonly Dictionary<SimilarityMethodEnum, similarityMethodProtoDelegate> ProbMethods;
static SimilarityMethods() static SimilarityMethods()
{ {
probMethods = new Dictionary<SimilarityMethodEnum, similarityMethodProtoDelegate> ProbMethods = new Dictionary<SimilarityMethodEnum, similarityMethodProtoDelegate>
{ {
[SimilarityMethodEnum.Cosine] = CosineSimilarity, [SimilarityMethodEnum.Cosine] = CosineSimilarity,
[SimilarityMethodEnum.Euclidian] = EuclidianDistance, [SimilarityMethodEnum.Euclidian] = EuclidianDistance,
@@ -44,7 +44,7 @@ public static class SimilarityMethods
methodName methodName
); );
if (!probMethods.TryGetValue(probMethodEnum, out similarityMethodProtoDelegate? method)) if (!ProbMethods.TryGetValue(probMethodEnum, out similarityMethodProtoDelegate? method))
{ {
return null; return null;
} }