28 Commits

Author SHA1 Message Date
LD50
d647bedb33 Merge pull request #60 from LD-Reborn/44-fix-controller-endpoint-naming-and-http-methods
Fixed endpoint naming and http methods
2025-12-28 17:36:17 +01:00
fe6bbfe9e5 Fixed endpoint naming and http methods 2025-12-28 17:36:01 +01:00
LD50
6f7afca195 Merge pull request #58 from LD-Reborn/56-bug-exception-when-update-indexing-entity
Fixed datapoint stale reference causing issues when updating datapoin…
2025-12-28 00:44:12 +01:00
3fa71a8d8b Fixed datapoint stale reference causing issues when updating datapoint text and probmethod or similaritymethod, fixe probmethod and similaritymethod not being applied in-memory 2025-12-28 00:43:55 +01:00
LD50
8921121078 Merge pull request #57 from LD-Reborn/54-properly-implement-embeddings-cache-size-limit-global
Implemented cache reconciliation
2025-12-28 00:22:15 +01:00
baf76685b7 Implemented cache reconciliation 2025-12-28 00:19:18 +01:00
LD50
4030e4a824 Merge pull request #55 from LD-Reborn/54-properly-implement-embeddings-cache-size-limit-global
Moved embeddingCache from Dictionary to LRUCache
2025-12-27 18:40:45 +01:00
7b4a3bd2c8 Moved embeddingCache from Dictionary to LRUCache 2025-12-27 18:40:03 +01:00
LD50
5eabb0d924 Merge pull request #53 from LD-Reborn/33-move-query-from-entity-to-searchdomain
33 move query from entity to searchdomain
2025-12-27 17:26:29 +01:00
40424053da Merge branch '33-move-query-from-entity-to-searchdomain' of https://github.com/LD-Reborn/embeddingsearch into 33-move-query-from-entity-to-searchdomain 2025-12-27 17:26:13 +01:00
f3a4665153 Moved query action from EntityController to SearchdomainController 2025-12-27 17:26:06 +01:00
a358eaea86 Moved query action from EntityController to SearchdomainController 2025-12-27 17:25:12 +01:00
665a392b5a Fixed redundant Searchdomain retrieval error messages 2025-12-25 15:25:23 +01:00
26d0561c3b Fixed wrong return model returned in EntityController methods 2025-12-25 14:55:58 +01:00
cc93a76546 Fixed DRY violations regarding result models 2025-12-25 14:55:30 +01:00
LD50
7298593341 Merge pull request #51 from LD-Reborn/35-implement-enums-for-probmethods-in-the-shared-models
Fixed embeddingCache not yet global
2025-12-25 14:19:41 +01:00
25723cb7a4 Fixed embeddingCache not yet global 2025-12-25 14:19:25 +01:00
LD50
84d83206cb Merge pull request #50 from LD-Reborn/35-implement-enums-for-probmethods-in-the-shared-models
Removed unused GenerateEmbeddings method
2025-12-25 14:18:41 +01:00
b6e01a3f66 Removed unused GenerateEmbeddings method 2025-12-25 14:13:28 +01:00
LD50
e4cfcb1030 Merge pull request #49 from LD-Reborn/35-implement-enums-for-probmethods-in-the-shared-models
Added enums to JSONEntity and JSONDatapoint
2025-12-25 13:20:44 +01:00
6d1cffe2db Added enums to JSONEntity and JSONDatapoint 2025-12-25 13:20:24 +01:00
LD50
dd0019b1c1 Merge pull request #48 from LD-Reborn/40-add-attributes-to-query-result
40 add attributes to query result
2025-12-25 12:39:18 +01:00
5877ebaff2 Added attributes to query results 2025-12-25 12:39:01 +01:00
040d4f916a Fixed views showing in swagger 2025-12-25 12:21:50 +01:00
LD50
57beddd70f Merge pull request #47 from LD-Reborn/42-create-a-front-end---localization
Added localization
2025-12-23 22:40:42 +01:00
8416d7f404 Added localization 2025-12-23 22:40:04 +01:00
16f08aa8a7 Removed privacy page 2025-12-23 21:37:10 +01:00
LD50
cce42d8ec3 Merge pull request #46 from LD-Reborn/41-create-a-front-end---toasts
41 create a front end   toasts
2025-12-23 14:55:39 +01:00
27 changed files with 868 additions and 499 deletions

View File

@@ -36,7 +36,7 @@ public class Client
public async Task<SearchdomainListResults> SearchdomainListAsync()
{
return await GetUrlAndProcessJson<SearchdomainListResults>(GetUrl($"{baseUri}/Searchdomain", "List", apiKey, []));
return await GetUrlAndProcessJson<SearchdomainListResults>(GetUrl($"{baseUri}", "Searchdomains", apiKey, []));
}
public async Task<SearchdomainDeleteResults> SearchdomainDeleteAsync()
@@ -46,7 +46,7 @@ public class Client
public async Task<SearchdomainDeleteResults> SearchdomainDeleteAsync(string searchdomain)
{
return await GetUrlAndProcessJson<SearchdomainDeleteResults>(GetUrl($"{baseUri}/Searchdomain", "Delete", apiKey, new Dictionary<string, string>()
return await DeleteUrlAndProcessJson<SearchdomainDeleteResults>(GetUrl($"{baseUri}", "Searchdomain", apiKey, new Dictionary<string, string>()
{
{"searchdomain", searchdomain}
}));
@@ -57,12 +57,12 @@ public class Client
return await SearchdomainCreateAsync(searchdomain);
}
public async Task<SearchdomainCreateResults> SearchdomainCreateAsync(string searchdomain)
public async Task<SearchdomainCreateResults> SearchdomainCreateAsync(string searchdomain, SearchdomainSettings searchdomainSettings = new())
{
return await GetUrlAndProcessJson<SearchdomainCreateResults>(GetUrl($"{baseUri}/Searchdomain", "Create", apiKey, new Dictionary<string, string>()
return await PostUrlAndProcessJson<SearchdomainCreateResults>(GetUrl($"{baseUri}", "Searchdomain", apiKey, new Dictionary<string, string>()
{
{"searchdomain", searchdomain}
}));
}), new StringContent(JsonSerializer.Serialize(searchdomainSettings), Encoding.UTF8, "application/json"));
}
public async Task<SearchdomainUpdateResults> SearchdomainUpdateAsync(string newName, string settings = "{}")
@@ -72,14 +72,18 @@ public class Client
return updateResults;
}
public async Task<SearchdomainUpdateResults> SearchdomainUpdateAsync(string searchdomain, string newName, SearchdomainSettings settings = new())
{
return await SearchdomainUpdateAsync(searchdomain, newName, JsonSerializer.Serialize(settings));
}
public async Task<SearchdomainUpdateResults> SearchdomainUpdateAsync(string searchdomain, string newName, string settings = "{}")
{
return await GetUrlAndProcessJson<SearchdomainUpdateResults>(GetUrl($"{baseUri}/Searchdomain", "Update", apiKey, new Dictionary<string, string>()
return await PutUrlAndProcessJson<SearchdomainUpdateResults>(GetUrl($"{baseUri}", "Searchdomain", apiKey, new Dictionary<string, string>()
{
{"searchdomain", searchdomain},
{"newName", newName},
{"settings", settings}
}));
{"newName", newName}
}), new StringContent(settings, Encoding.UTF8, "application/json"));
}
public async Task<EntityQueryResults> EntityQueryAsync(string query)
@@ -89,11 +93,11 @@ public class Client
public async Task<EntityQueryResults> EntityQueryAsync(string searchdomain, string query)
{
return await GetUrlAndProcessJson<EntityQueryResults>(GetUrl($"{baseUri}/Entity", "Query", apiKey, new Dictionary<string, string>()
return await PostUrlAndProcessJson<EntityQueryResults>(GetUrl($"{baseUri}/Searchdomain", "Query", apiKey, new Dictionary<string, string>()
{
{"searchdomain", searchdomain},
{"query", query}
}));
}), null);
}
public async Task<EntityIndexResult> EntityIndexAsync(List<JSONEntity> jsonEntity)
@@ -104,7 +108,7 @@ public class Client
public async Task<EntityIndexResult> EntityIndexAsync(string jsonEntity)
{
var content = new StringContent(jsonEntity, Encoding.UTF8, "application/json");
return await PostUrlAndProcessJson<EntityIndexResult>(GetUrl($"{baseUri}/Entity", "Index", apiKey, []), content);//new FormUrlEncodedContent(values));
return await PutUrlAndProcessJson<EntityIndexResult>(GetUrl($"{baseUri}", "Entity", apiKey, []), content);
}
public async Task<EntityListResults> EntityListAsync(bool returnEmbeddings = false)
@@ -114,7 +118,7 @@ public class Client
public async Task<EntityListResults> EntityListAsync(string searchdomain, bool returnEmbeddings = false)
{
var url = $"{baseUri}/Entity/List?apiKey={HttpUtility.UrlEncode(apiKey)}&searchdomain={HttpUtility.UrlEncode(searchdomain)}&returnEmbeddings={HttpUtility.UrlEncode(returnEmbeddings.ToString())}";
var url = $"{baseUri}/Entities?apiKey={HttpUtility.UrlEncode(apiKey)}&searchdomain={HttpUtility.UrlEncode(searchdomain)}&returnEmbeddings={HttpUtility.UrlEncode(returnEmbeddings.ToString())}";
return await GetUrlAndProcessJson<EntityListResults>(url);
}
@@ -125,8 +129,8 @@ public class Client
public async Task<EntityDeleteResults> EntityDeleteAsync(string searchdomain, string entityName)
{
var url = $"{baseUri}/Entity/Delete?apiKey={HttpUtility.UrlEncode(apiKey)}&searchdomain={HttpUtility.UrlEncode(searchdomain)}&entity={HttpUtility.UrlEncode(entityName)}";
return await GetUrlAndProcessJson<EntityDeleteResults>(url);
var url = $"{baseUri}/Entity?apiKey={HttpUtility.UrlEncode(apiKey)}&searchdomain={HttpUtility.UrlEncode(searchdomain)}&entity={HttpUtility.UrlEncode(entityName)}";
return await DeleteUrlAndProcessJson<EntityDeleteResults>(url);
}
private static async Task<T> GetUrlAndProcessJson<T>(string url)
@@ -138,7 +142,8 @@ public class Client
?? throw new Exception($"Failed to deserialize JSON to type {typeof(T).Name}");
return result;
}
private static async Task<T> PostUrlAndProcessJson<T>(string url, HttpContent content)
private static async Task<T> PostUrlAndProcessJson<T>(string url, HttpContent? content)
{
using var client = new HttpClient();
var response = await client.PostAsync(url, content);
@@ -148,6 +153,26 @@ public class Client
return result;
}
private static async Task<T> PutUrlAndProcessJson<T>(string url, HttpContent content)
{
using var client = new HttpClient();
var response = await client.PutAsync(url, content);
string responseContent = await response.Content.ReadAsStringAsync();
var result = JsonSerializer.Deserialize<T>(responseContent)
?? throw new Exception($"Failed to deserialize JSON to type {typeof(T).Name}");
return result;
}
private static async Task<T> DeleteUrlAndProcessJson<T>(string url)
{
using var client = new HttpClient();
var response = await client.DeleteAsync(url);
string responseContent = await response.Content.ReadAsStringAsync();
var result = JsonSerializer.Deserialize<T>(responseContent)
?? throw new Exception($"Failed to deserialize JSON to type {typeof(T).Name}");
return result;
}
public static string GetUrl(string baseUri, string endpoint, string apiKey, Dictionary<string, string> parameters)
{
var uriBuilder = new UriBuilder($"{baseUri}/{endpoint}");

View File

@@ -6,6 +6,7 @@ using Server.Models;
namespace Server.Controllers;
[ApiExplorerSettings(IgnoreApi = true)]
[Route("[Controller]")]
public class AccountController : Controller
{

View File

@@ -24,32 +24,7 @@ public class EntityController : ControllerBase
_databaseHelper = databaseHelper;
}
[HttpGet("Query")]
public ActionResult<EntityQueryResults> Query(string searchdomain, string query, int? topN)
{
Searchdomain searchdomain_;
try
{
searchdomain_ = _domainManager.GetSearchdomain(searchdomain);
} catch (SearchdomainNotFoundException)
{
_logger.LogError("Unable to retrieve the searchdomain {searchdomain} - it likely does not exist yet", [searchdomain]);
return Ok(new EntityQueryResults() {Results = [], Success = false, Message = "Searchdomain not found" });
} catch (Exception ex)
{
_logger.LogError("Unable to retrieve the searchdomain {searchdomain} - {ex.Message} - {ex.StackTrace}", [searchdomain, ex.Message, ex.StackTrace]);
return Ok(new EntityQueryResults() {Results = [], Success = false, Message = "Unable to retrieve the searchdomain - it likely exists, but some other error happened." });
}
List<(float, string)> results = searchdomain_.Search(query, topN);
List<EntityQueryResult> queryResults = [.. results.Select(r => new EntityQueryResult
{
Name = r.Item2,
Value = r.Item1
})];
return Ok(new EntityQueryResults(){Results = queryResults, Success = true });
}
[HttpPost("Index")]
[HttpPut]
public ActionResult<EntityIndexResult> Index([FromBody] List<JSONEntity>? jsonEntities)
{
try
@@ -69,7 +44,6 @@ public class EntityController : ControllerBase
&& !invalidatedSearchdomains.Contains(jsonEntitySearchdomainName))
{
invalidatedSearchdomains.Add(jsonEntitySearchdomainName);
_domainManager.InvalidateSearchdomainCache(jsonEntitySearchdomainName);
}
}
return Ok(new EntityIndexResult() { Success = true });
@@ -88,27 +62,16 @@ public class EntityController : ControllerBase
}
[HttpGet("List")]
[HttpGet("/Entities")]
public ActionResult<EntityListResults> List(string searchdomain, bool returnModels = false, bool returnEmbeddings = false)
{
if (returnEmbeddings && !returnModels)
{
_logger.LogError("Invalid request for {searchdomain} - embeddings return requested but without models - not possible!", [searchdomain]);
return Ok(new EntityQueryResults() {Results = [], Success = false, Message = "Invalid request" });
}
Searchdomain searchdomain_;
try
{
searchdomain_ = _domainManager.GetSearchdomain(searchdomain);
} catch (SearchdomainNotFoundException)
{
_logger.LogError("Unable to retrieve the searchdomain {searchdomain} - it likely does not exist yet", [searchdomain]);
return Ok(new EntityQueryResults() {Results = [], Success = false, Message = "Searchdomain not found" });
} catch (Exception ex)
{
_logger.LogError("Unable to retrieve the searchdomain {searchdomain} - {ex.Message} - {ex.StackTrace}", [searchdomain, ex.Message, ex.StackTrace]);
return Ok(new EntityQueryResults() {Results = [], Success = false, Message = "Unable to retrieve the searchdomain - it likely exists, but some other error happened." });
return BadRequest(new EntityListResults() {Results = [], Success = false, Message = "Invalid request" });
}
(Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomain, _logger);
if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new SearchdomainUpdateResults(){Success = false, Message = message});
EntityListResults entityListResults = new() {Results = [], Success = true};
foreach (Entity entity in searchdomain_.entityCache)
{
@@ -146,22 +109,11 @@ public class EntityController : ControllerBase
return Ok(entityListResults);
}
[HttpGet("Delete")]
[HttpDelete]
public ActionResult<EntityDeleteResults> Delete(string searchdomain, string entityName)
{
Searchdomain searchdomain_;
try
{
searchdomain_ = _domainManager.GetSearchdomain(searchdomain);
} catch (SearchdomainNotFoundException)
{
_logger.LogError("Unable to retrieve the searchdomain {searchdomain} - it likely does not exist yet", [searchdomain]);
return Ok(new EntityQueryResults() {Results = [], Success = false, Message = "Searchdomain not found" });
} catch (Exception ex)
{
_logger.LogError("Unable to retrieve the searchdomain {searchdomain} - {ex.Message} - {ex.StackTrace}", [searchdomain, ex.Message, ex.StackTrace]);
return Ok(new EntityQueryResults() {Results = [], Success = false, Message = "Unable to retrieve the searchdomain - it likely exists, but some other error happened." });
}
(Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomain, _logger);
if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new SearchdomainUpdateResults(){Success = false, Message = message});
Entity? entity_ = SearchdomainHelper.CacheGetEntity(searchdomain_.entityCache, entityName);
if (entity_ is null)
@@ -169,6 +121,7 @@ public class EntityController : ControllerBase
_logger.LogError("Unable to delete the entity {entityName} in {searchdomain} - it was not found under the specified name", [entityName, searchdomain]);
return Ok(new EntityDeleteResults() {Success = false, Message = "Entity not found"});
}
searchdomain_.ReconciliateOrInvalidateCacheForDeletedEntity(entity_);
_databaseHelper.RemoveEntity([], _domainManager.helper, entityName, searchdomain);
searchdomain_.entityCache.RemoveAll(entity => entity.name == entityName);
return Ok(new EntityDeleteResults() {Success = true});

View File

@@ -7,7 +7,7 @@ using Server.Exceptions;
using Server.Models;
namespace Server.Controllers;
[ApiController]
[ApiExplorerSettings(IgnoreApi = true)]
[Route("/")]
public class HomeController : Controller
{

View File

@@ -23,7 +23,7 @@ public class SearchdomainController : ControllerBase
_domainManager = domainManager;
}
[HttpGet("List")]
[HttpGet("/Searchdomains")]
public ActionResult<SearchdomainListResults> List()
{
List<string> results;
@@ -40,8 +40,8 @@ public class SearchdomainController : ControllerBase
return Ok(searchdomainListResults);
}
[HttpGet("Create")]
public ActionResult<SearchdomainCreateResults> Create(string searchdomain, string settings = "{}")
[HttpPost]
public ActionResult<SearchdomainCreateResults> Create(string searchdomain, [FromBody]SearchdomainSettings settings = new())
{
try
{
@@ -54,7 +54,7 @@ public class SearchdomainController : ControllerBase
}
}
[HttpGet("Delete")]
[HttpDelete]
public ActionResult<SearchdomainDeleteResults> Delete(string searchdomain)
{
bool success;
@@ -84,12 +84,21 @@ public class SearchdomainController : ControllerBase
return Ok(new SearchdomainDeleteResults(){Success = success, DeletedEntities = deletedEntries, Message = message});
}
[HttpGet("Update")]
public ActionResult<SearchdomainUpdateResults> Update(string searchdomain, string newName, string settings = "{}")
[HttpPut]
public ActionResult<SearchdomainUpdateResults> Update(string searchdomain, string newName, [FromBody]string? settings = "{}")
{
try
(Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomain, _logger);
if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new SearchdomainUpdateResults(){Success = false, Message = message});
if (settings is null)
{
Dictionary<string, dynamic> parameters = new()
{
{"name", newName},
{"id", searchdomain_.id}
};
searchdomain_.helper.ExecuteSQLNonQuery("UPDATE searchdomain set name = @name WHERE id = @id", parameters);
} else
{
Searchdomain searchdomain_ = _domainManager.GetSearchdomain(searchdomain);
Dictionary<string, dynamic> parameters = new()
{
{"name", newName},
@@ -97,84 +106,55 @@ public class SearchdomainController : ControllerBase
{"id", searchdomain_.id}
};
searchdomain_.helper.ExecuteSQLNonQuery("UPDATE searchdomain set name = @name, settings = @settings WHERE id = @id", parameters);
} catch (SearchdomainNotFoundException)
{
_logger.LogError("Unable to update searchdomain {searchdomain} - not found", [searchdomain]);
return Ok(new SearchdomainUpdateResults() { Success = false, Message = $"Unable to update searchdomain {searchdomain} - not found" });
} catch (Exception ex)
{
_logger.LogError("Unable to update searchdomain {searchdomain} - Exception: {ex.Message} - {ex.StackTrace}", [searchdomain, ex.Message, ex.StackTrace]);
return Ok(new SearchdomainUpdateResults() { Success = false, Message = $"Unable to update searchdomain {searchdomain}" });
}
return Ok(new SearchdomainUpdateResults(){Success = true});
}
[HttpPost("UpdateSettings")]
[HttpPost("Query")]
public ActionResult<EntityQueryResults> Query(string searchdomain, string query, int? topN, bool returnAttributes = false)
{
(Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomain, _logger);
if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new SearchdomainUpdateResults(){Success = false, Message = message});
List<(float, string)> results = searchdomain_.Search(query, topN);
List<EntityQueryResult> queryResults = [.. results.Select(r => new EntityQueryResult
{
Name = r.Item2,
Value = r.Item1,
Attributes = returnAttributes ? (searchdomain_.entityCache.FirstOrDefault(x => x.name == r.Item2)?.attributes ?? null) : null
})];
return Ok(new EntityQueryResults(){Results = queryResults, Success = true });
}
[HttpPut("Settings")]
public ActionResult<SearchdomainUpdateResults> UpdateSettings(string searchdomain, [FromBody] SearchdomainSettings request)
{
try
(Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomain, _logger);
if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new SearchdomainUpdateResults(){Success = false, Message = message});
Dictionary<string, dynamic> parameters = new()
{
Searchdomain searchdomain_ = _domainManager.GetSearchdomain(searchdomain);
Dictionary<string, dynamic> parameters = new()
{
{"settings", JsonSerializer.Serialize(request)},
{"id", searchdomain_.id}
};
searchdomain_.helper.ExecuteSQLNonQuery("UPDATE searchdomain set settings = @settings WHERE id = @id", parameters);
searchdomain_.settings = request;
} catch (SearchdomainNotFoundException)
{
_logger.LogError("Unable to update settings for searchdomain {searchdomain} - not found", [searchdomain]);
return Ok(new SearchdomainUpdateResults() { Success = false, Message = $"Unable to update settings for searchdomain {searchdomain} - not found" });
} catch (Exception ex)
{
_logger.LogError("Unable to update settings for searchdomain {searchdomain} - Exception: {ex.Message} - {ex.StackTrace}", [searchdomain, ex.Message, ex.StackTrace]);
return Ok(new SearchdomainUpdateResults() { Success = false, Message = $"Unable to update settings for searchdomain {searchdomain}" });
}
{"settings", JsonSerializer.Serialize(request)},
{"id", searchdomain_.id}
};
searchdomain_.helper.ExecuteSQLNonQuery("UPDATE searchdomain set settings = @settings WHERE id = @id", parameters);
searchdomain_.settings = request;
return Ok(new SearchdomainUpdateResults(){Success = true});
}
[HttpGet("GetSearches")]
public ActionResult<SearchdomainSearchesResults> GetSearches(string searchdomain)
[HttpGet("Queries")]
public ActionResult<SearchdomainSearchesResults> GetQueries(string searchdomain)
{
Searchdomain searchdomain_;
try
{
searchdomain_ = _domainManager.GetSearchdomain(searchdomain);
}
catch (SearchdomainNotFoundException)
{
_logger.LogError("Unable to retrieve the searchdomain {searchdomain} - it likely does not exist yet", [searchdomain]);
return Ok(new SearchdomainSearchesResults() { Searches = [], Success = false, Message = "Searchdomain not found" });
}
catch (Exception ex)
{
_logger.LogError("Unable to retrieve the searchdomain {searchdomain} - {ex.Message} - {ex.StackTrace}", [searchdomain, ex.Message, ex.StackTrace]);
return Ok(new SearchdomainSearchesResults() { Searches = [], Success = false, Message = ex.Message });
}
(Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomain, _logger);
if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new SearchdomainUpdateResults(){Success = false, Message = message});
Dictionary<string, DateTimedSearchResult> searchCache = searchdomain_.searchCache;
return Ok(new SearchdomainSearchesResults() { Searches = searchCache, Success = true });
}
[HttpDelete("Searches")]
public ActionResult<SearchdomainDeleteSearchResult> DeleteSearch(string searchdomain, string query)
[HttpDelete("Query")]
public ActionResult<SearchdomainDeleteSearchResult> DeleteQuery(string searchdomain, string query)
{
Searchdomain searchdomain_;
try
{
searchdomain_ = _domainManager.GetSearchdomain(searchdomain);
}
catch (SearchdomainNotFoundException)
{
_logger.LogError("Unable to retrieve the searchdomain {searchdomain} - it likely does not exist yet", [searchdomain]);
return Ok(new SearchdomainDeleteSearchResult() { Success = false, Message = "Searchdomain not found" });
}
catch (Exception ex)
{
_logger.LogError("Unable to retrieve the searchdomain {searchdomain} - {ex.Message} - {ex.StackTrace}", [searchdomain, ex.Message, ex.StackTrace]);
return Ok(new SearchdomainDeleteSearchResult() { Success = false, Message = ex.Message });
}
(Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomain, _logger);
if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new SearchdomainUpdateResults(){Success = false, Message = message});
Dictionary<string, DateTimedSearchResult> searchCache = searchdomain_.searchCache;
bool containsKey = searchCache.ContainsKey(query);
if (containsKey)
@@ -185,24 +165,11 @@ public class SearchdomainController : ControllerBase
return Ok(new SearchdomainDeleteSearchResult() {Success = false, Message = "Query not found in search cache"});
}
[HttpPatch("Searches")]
public ActionResult<SearchdomainUpdateSearchResult> UpdateSearch(string searchdomain, string query, [FromBody]List<ResultItem> results)
[HttpPatch("Query")]
public ActionResult<SearchdomainUpdateSearchResult> UpdateQuery(string searchdomain, string query, [FromBody]List<ResultItem> results)
{
Searchdomain searchdomain_;
try
{
searchdomain_ = _domainManager.GetSearchdomain(searchdomain);
}
catch (SearchdomainNotFoundException)
{
_logger.LogError("Unable to retrieve the searchdomain {searchdomain} - it likely does not exist yet", [searchdomain]);
return Ok(new SearchdomainUpdateSearchResult() { Success = false, Message = "Searchdomain not found" });
}
catch (Exception ex)
{
_logger.LogError("Unable to retrieve the searchdomain {searchdomain} - {ex.Message} - {ex.StackTrace}", [searchdomain, ex.Message, ex.StackTrace]);
return Ok(new SearchdomainUpdateSearchResult() { Success = false, Message = ex.Message });
}
(Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomain, _logger);
if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new SearchdomainUpdateResults(){Success = false, Message = message});
Dictionary<string, DateTimedSearchResult> searchCache = searchdomain_.searchCache;
bool containsKey = searchCache.ContainsKey(query);
if (containsKey)
@@ -215,46 +182,20 @@ public class SearchdomainController : ControllerBase
return Ok(new SearchdomainUpdateSearchResult() {Success = false, Message = "Query not found in search cache"});
}
[HttpGet("GetSettings")]
[HttpGet("Settings")]
public ActionResult<SearchdomainSettingsResults> GetSettings(string searchdomain)
{
Searchdomain searchdomain_;
try
{
searchdomain_ = _domainManager.GetSearchdomain(searchdomain);
}
catch (SearchdomainNotFoundException)
{
_logger.LogError("Unable to retrieve the searchdomain {searchdomain} - it likely does not exist yet", [searchdomain]);
return Ok(new SearchdomainSettingsResults() { Settings = null, Success = false, Message = "Searchdomain not found" });
}
catch (Exception ex)
{
_logger.LogError("Unable to retrieve the searchdomain {searchdomain} - {ex.Message} - {ex.StackTrace}", [searchdomain, ex.Message, ex.StackTrace]);
return Ok(new SearchdomainSettingsResults() { Settings = null, Success = false, Message = ex.Message });
}
(Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomain, _logger);
if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new SearchdomainUpdateResults(){Success = false, Message = message});
SearchdomainSettings settings = searchdomain_.settings;
return Ok(new SearchdomainSettingsResults() { Settings = settings, Success = true });
}
[HttpGet("GetSearchCacheSize")]
[HttpGet("SearchCache/Size")]
public ActionResult<SearchdomainSearchCacheSizeResults> GetSearchCacheSize(string searchdomain)
{
Searchdomain searchdomain_;
try
{
searchdomain_ = _domainManager.GetSearchdomain(searchdomain);
}
catch (SearchdomainNotFoundException)
{
_logger.LogError("Unable to retrieve the searchdomain {searchdomain} - it likely does not exist yet", [searchdomain]);
return Ok(new SearchdomainSearchCacheSizeResults() { SearchCacheSizeBytes = null, Success = false, Message = "Searchdomain not found" });
}
catch (Exception ex)
{
_logger.LogError("Unable to retrieve the searchdomain {searchdomain} - {ex.Message} - {ex.StackTrace}", [searchdomain, ex.Message, ex.StackTrace]);
return Ok(new SearchdomainSearchCacheSizeResults() { SearchCacheSizeBytes = null, Success = false, Message = ex.Message });
}
(Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomain, _logger);
if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new SearchdomainUpdateResults(){Success = false, Message = message});
Dictionary<string, DateTimedSearchResult> searchCache = searchdomain_.searchCache;
long sizeInBytes = 0;
foreach (var entry in searchCache)
@@ -266,43 +207,20 @@ public class SearchdomainController : ControllerBase
return Ok(new SearchdomainSearchCacheSizeResults() { SearchCacheSizeBytes = sizeInBytes, Success = true });
}
[HttpGet("ClearSearchCache")]
[HttpPost("SearchCache/Clear")]
public ActionResult<SearchdomainInvalidateCacheResults> InvalidateSearchCache(string searchdomain)
{
try
{
Searchdomain searchdomain_ = _domainManager.GetSearchdomain(searchdomain);
searchdomain_.InvalidateSearchCache();
} catch (SearchdomainNotFoundException)
{
_logger.LogError("Unable to invalidate search cache for searchdomain {searchdomain} - not found", [searchdomain]);
return Ok(new SearchdomainInvalidateCacheResults() { Success = false, Message = $"Unable to invalidate search cache for searchdomain {searchdomain} - not found" });
} catch (Exception ex)
{
_logger.LogError("Unable to invalidate search cache for searchdomain {searchdomain} - Exception: {ex.Message} - {ex.StackTrace}", [searchdomain, ex.Message, ex.StackTrace]);
return Ok(new SearchdomainInvalidateCacheResults() { Success = false, Message = $"Unable to invalidate search cache for searchdomain {searchdomain}" });
}
(Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomain, _logger);
if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new SearchdomainUpdateResults(){Success = false, Message = message});
searchdomain_.InvalidateSearchCache();
return Ok(new SearchdomainInvalidateCacheResults(){Success = true});
}
[HttpGet("GetDatabaseSize")]
[HttpGet("Database/Size")]
public ActionResult<SearchdomainGetDatabaseSizeResult> GetDatabaseSize(string searchdomain)
{
Searchdomain searchdomain_;
try
{
searchdomain_ = _domainManager.GetSearchdomain(searchdomain);
}
catch (SearchdomainNotFoundException)
{
_logger.LogError("Unable to retrieve the searchdomain {searchdomain} - it likely does not exist yet", [searchdomain]);
return Ok(new SearchdomainGetDatabaseSizeResult() { SearchdomainDatabaseSizeBytes = null, Success = false, Message = "Searchdomain not found" });
}
catch (Exception ex)
{
_logger.LogError("Unable to retrieve the searchdomain {searchdomain} - {ex.Message} - {ex.StackTrace}", [searchdomain, ex.Message, ex.StackTrace]);
return Ok(new SearchdomainGetDatabaseSizeResult() { SearchdomainDatabaseSizeBytes = null, Success = false, Message = ex.Message });
}
(Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomain, _logger);
if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new SearchdomainUpdateResults(){Success = false, Message = message});
long sizeInBytes = DatabaseHelper.GetSearchdomainDatabaseSize(searchdomain_.helper, searchdomain);
return Ok(new SearchdomainGetDatabaseSizeResult() { SearchdomainDatabaseSizeBytes = sizeInBytes, Success = true });
}

View File

@@ -22,7 +22,7 @@ public class ServerController : ControllerBase
_aIProvider = aIProvider;
}
[HttpGet("GetModels")]
[HttpGet("Models")]
public ActionResult<ServerGetModelsResult> GetModels()
{
try

View File

@@ -1,3 +1,4 @@
using AdaptiveExpressions;
using OllamaSharp;
using OllamaSharp.Models;
@@ -27,74 +28,31 @@ public class Datapoint
public static Dictionary<string, float[]> GenerateEmbeddings(string content, List<string> models, AIProvider aIProvider)
{
return GenerateEmbeddings(content, models, aIProvider, []);
return GenerateEmbeddings(content, models, aIProvider, new());
}
public static Dictionary<string, float[]> GenerateEmbeddings(List<string> contents, string model, OllamaApiClient ollama, Dictionary<string, Dictionary<string, float[]>> embeddingCache)
{
Dictionary<string, float[]> retVal = [];
List<string> remainingContents = new List<string>(contents);
for (int i = contents.Count - 1; i >= 0; i--) // Compare against cache and remove accordingly
{
string content = contents[i];
if (embeddingCache.ContainsKey(model) && embeddingCache[model].ContainsKey(content))
{
retVal[content] = embeddingCache[model][content];
remainingContents.RemoveAt(i);
}
}
if (remainingContents.Count == 0)
{
return retVal;
}
EmbedRequest request = new()
{
Model = model,
Input = remainingContents
};
EmbedResponse response = ollama.EmbedAsync(request).Result;
for (int i = 0; i < response.Embeddings.Count; i++)
{
string content = remainingContents.ElementAt(i);
float[] embeddings = response.Embeddings.ElementAt(i);
retVal[content] = embeddings;
if (!embeddingCache.ContainsKey(model))
{
embeddingCache[model] = [];
}
if (!embeddingCache[model].ContainsKey(content))
{
embeddingCache[model][content] = embeddings;
}
}
return retVal;
}
public static Dictionary<string, float[]> GenerateEmbeddings(string content, List<string> models, AIProvider aIProvider, Dictionary<string, Dictionary<string, float[]>> embeddingCache)
public static Dictionary<string, float[]> GenerateEmbeddings(string content, List<string> models, AIProvider aIProvider, LRUCache<string, Dictionary<string, float[]>> embeddingCache)
{
Dictionary<string, float[]> retVal = [];
foreach (string model in models)
{
if (embeddingCache.ContainsKey(model) && embeddingCache[model].ContainsKey(content))
bool embeddingCacheHasModel = embeddingCache.TryGet(model, out var embeddingCacheForModel);
if (embeddingCacheHasModel && embeddingCacheForModel.ContainsKey(content))
{
retVal[model] = embeddingCache[model][content];
retVal[model] = embeddingCacheForModel[content];
continue;
}
var response = aIProvider.GenerateEmbeddings(model, [content]);
if (response is not null)
{
retVal[model] = response;
if (!embeddingCache.ContainsKey(model))
if (!embeddingCacheHasModel)
{
embeddingCache[model] = [];
embeddingCacheForModel = [];
}
if (!embeddingCache[model].ContainsKey(content))
if (!embeddingCacheForModel.ContainsKey(content))
{
embeddingCache[model][content] = response;
embeddingCacheForModel[content] = response;
}
}
}

View File

@@ -1,7 +1,9 @@
using Shared.Models;
namespace Server.Exceptions;
public class ProbMethodNotFoundException(string probMethod) : Exception($"Unknown probMethod name {probMethod}") { }
public class ProbMethodNotFoundException(ProbMethodEnum probMethod) : Exception($"Unknown probMethod name {probMethod}") { }
public class SimilarityMethodNotFoundException(string similarityMethod) : Exception($"Unknown similarityMethod name \"{similarityMethod}\"") { }
public class SimilarityMethodNotFoundException(SimilarityMethodEnum similarityMethod) : Exception($"Unknown similarityMethod name \"{similarityMethod}\"") { }
public class JSONPathSelectionException(string path, string testedContent) : Exception($"Unable to select tokens using JSONPath {path} for string: {testedContent}.") { }

View File

@@ -38,12 +38,12 @@ public class DatabaseHelper(ILogger<DatabaseHelper> logger)
return helper.ExecuteSQLCommandGetInsertedID("INSERT INTO searchdomain (name, settings) VALUES (@name, @settings)", parameters);
}
public static int DatabaseInsertEntity(SQLHelper helper, string name, string probmethod, int id_searchdomain)
public static int DatabaseInsertEntity(SQLHelper helper, string name, ProbMethodEnum probmethod, int id_searchdomain)
{
Dictionary<string, dynamic> parameters = new()
{
{ "name", name },
{ "probmethod", probmethod },
{ "probmethod", probmethod.ToString() },
{ "id_searchdomain", id_searchdomain }
};
return helper.ExecuteSQLCommandGetInsertedID("INSERT INTO entity (name, probmethod, id_searchdomain) VALUES (@name, @probmethod, @id_searchdomain)", parameters);
@@ -60,13 +60,13 @@ public class DatabaseHelper(ILogger<DatabaseHelper> logger)
return helper.ExecuteSQLCommandGetInsertedID("INSERT INTO attribute (attribute, value, id_entity) VALUES (@attribute, @value, @id_entity)", parameters);
}
public static int DatabaseInsertDatapoint(SQLHelper helper, string name, string probmethod_embedding, string similarityMethod, string hash, int id_entity)
public static int DatabaseInsertDatapoint(SQLHelper helper, string name, ProbMethodEnum probmethod_embedding, SimilarityMethodEnum similarityMethod, string hash, int id_entity)
{
Dictionary<string, dynamic> parameters = new()
{
{ "name", name },
{ "probmethod_embedding", probmethod_embedding },
{ "similaritymethod", similarityMethod },
{ "probmethod_embedding", probmethod_embedding.ToString() },
{ "similaritymethod", similarityMethod.ToString() },
{ "hash", hash },
{ "id_entity", id_entity }
};

View File

@@ -2,6 +2,7 @@ using System.Collections.Concurrent;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
using AdaptiveExpressions;
using Server.Exceptions;
using Shared.Models;
@@ -46,7 +47,7 @@ public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelp
public List<Entity>? EntitiesFromJSON(SearchdomainManager searchdomainManager, ILogger logger, string json)
{
Dictionary<string, Dictionary<string, float[]>> embeddingCache = searchdomainManager.embeddingCache;
LRUCache<string, Dictionary<string, float[]>> embeddingCache = searchdomainManager.embeddingCache;
AIProvider aIProvider = searchdomainManager.aIProvider;
SQLHelper helper = searchdomainManager.helper;
@@ -91,8 +92,9 @@ public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelp
Searchdomain searchdomain = searchdomainManager.GetSearchdomain(jsonEntity.Searchdomain);
List<Entity> entityCache = searchdomain.entityCache;
AIProvider aIProvider = searchdomain.aIProvider;
Dictionary<string, Dictionary<string, float[]>> embeddingCache = searchdomain.embeddingCache;
LRUCache<string, Dictionary<string, float[]>> embeddingCache = searchdomain.embeddingCache;
Entity? preexistingEntity = entityCache.FirstOrDefault(entity => entity.name == jsonEntity.Name);
bool invalidateSearchCache = false;
if (preexistingEntity is not null)
{
@@ -147,8 +149,9 @@ public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelp
}
// Datapoint
foreach (Datapoint datapoint in preexistingEntity.datapoints.ToList())
foreach (Datapoint datapoint_ in preexistingEntity.datapoints.ToList())
{
Datapoint datapoint = datapoint_; // To enable replacing the datapoint reference as foreach iterators cannot be overwritten
bool newEntityHasDatapoint = jsonEntity.Datapoints.Any(x => x.Name == datapoint.name);
if (!newEntityHasDatapoint)
{
@@ -161,6 +164,7 @@ public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelp
helper.ExecuteSQLNonQuery("DELETE e FROM embedding e JOIN datapoint d ON e.id_datapoint=d.id WHERE d.name=@datapointName AND d.id_entity=@entityId", parameters);
helper.ExecuteSQLNonQuery("DELETE FROM datapoint WHERE id_entity=@entityId AND name=@datapointName", parameters);
preexistingEntity.datapoints.Remove(datapoint);
invalidateSearchCache = true;
} else
{
JSONDatapoint? newEntityDatapoint = jsonEntity.Datapoints.FirstOrDefault(x => x.Name == datapoint.name);
@@ -177,22 +181,24 @@ public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelp
preexistingEntity.datapoints.Remove(datapoint);
Datapoint newDatapoint = DatabaseInsertDatapointWithEmbeddings(helper, searchdomain, newEntityDatapoint, (int)preexistingEntityID);
preexistingEntity.datapoints.Add(newDatapoint);
datapoint = newDatapoint;
invalidateSearchCache = true;
}
if (newEntityDatapoint is not null && (newEntityDatapoint.Probmethod_embedding != datapoint.probMethod.name || newEntityDatapoint.SimilarityMethod != datapoint.similarityMethod.name))
if (newEntityDatapoint is not null && (newEntityDatapoint.Probmethod_embedding != datapoint.probMethod.probMethodEnum || newEntityDatapoint.SimilarityMethod != datapoint.similarityMethod.similarityMethodEnum))
{
// Datapoint - Updated (probmethod or similaritymethod)
Dictionary<string, dynamic> parameters = new()
{
{ "probmethod", newEntityDatapoint.Probmethod_embedding },
{ "similaritymethod", newEntityDatapoint.SimilarityMethod },
{ "probmethod", newEntityDatapoint.Probmethod_embedding.ToString() },
{ "similaritymethod", newEntityDatapoint.SimilarityMethod.ToString() },
{ "datapointName", datapoint.name },
{ "entityId", preexistingEntityID}
};
helper.ExecuteSQLNonQuery("UPDATE datapoint SET probmethod_embedding=@probmethod, similaritymethod=@similaritymethod WHERE id_entity=@entityId AND name=@datapointName", parameters);
Datapoint preexistingDatapoint = preexistingEntity.datapoints.First(x => x == datapoint); // The for loop is a copy. This retrieves the original such that it can be updated.
preexistingDatapoint.probMethod = datapoint.probMethod;
preexistingDatapoint.similarityMethod = datapoint.similarityMethod;
preexistingDatapoint.probMethod = new(newEntityDatapoint.Probmethod_embedding, _logger);
preexistingDatapoint.similarityMethod = new(newEntityDatapoint.SimilarityMethod, _logger);
invalidateSearchCache = true;
}
}
}
@@ -204,10 +210,14 @@ public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelp
// Datapoint - New
Datapoint datapoint = DatabaseInsertDatapointWithEmbeddings(helper, searchdomain, jsonDatapoint, (int)preexistingEntityID);
preexistingEntity.datapoints.Add(datapoint);
invalidateSearchCache = true;
}
}
if (invalidateSearchCache)
{
searchdomain.ReconciliateOrInvalidateCacheForNewOrUpdatedEntity(preexistingEntity);
}
return preexistingEntity;
}
else
@@ -227,11 +237,12 @@ public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelp
}
var probMethod = Probmethods.GetMethod(jsonEntity.Probmethod) ?? throw new ProbMethodNotFoundException(jsonEntity.Probmethod);
Entity entity = new(jsonEntity.Attributes, probMethod, jsonEntity.Probmethod, datapoints, jsonEntity.Name)
Entity entity = new(jsonEntity.Attributes, probMethod, jsonEntity.Probmethod.ToString(), datapoints, jsonEntity.Name)
{
id = id_entity
};
entityCache.Add(entity);
searchdomain.ReconciliateOrInvalidateCacheForNewOrUpdatedEntity(entity);
return entity;
}
}
@@ -261,7 +272,7 @@ public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelp
throw new Exception("jsonDatapoint.Text must not be null at this point");
}
using SQLHelper helper = searchdomain.helper.DuplicateConnection();
Dictionary<string, Dictionary<string, float[]>> embeddingCache = searchdomain.embeddingCache;
LRUCache<string, Dictionary<string, float[]>> embeddingCache = searchdomain.embeddingCache;
hash ??= Convert.ToBase64String(SHA256.HashData(Encoding.UTF8.GetBytes(jsonDatapoint.Text)));
DatabaseHelper.DatabaseInsertDatapoint(helper, jsonDatapoint.Name, jsonDatapoint.Probmethod_embedding, jsonDatapoint.SimilarityMethod, hash, entityId);
Dictionary<string, float[]> embeddings = Datapoint.GenerateEmbeddings(jsonDatapoint.Text, [.. jsonDatapoint.Model], searchdomain.aIProvider, embeddingCache);
@@ -269,4 +280,21 @@ public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelp
var similarityMethod = new SimilarityMethod(jsonDatapoint.SimilarityMethod, logger) ?? throw new SimilarityMethodNotFoundException(jsonDatapoint.SimilarityMethod);
return new Datapoint(jsonDatapoint.Name, probMethod_embedding, similarityMethod, hash, [.. embeddings.Select(kv => (kv.Key, kv.Value))]);
}
public static (Searchdomain?, int?, string?) TryGetSearchdomain(SearchdomainManager searchdomainManager, string searchdomain, ILogger logger)
{
try
{
Searchdomain searchdomain_ = searchdomainManager.GetSearchdomain(searchdomain);
return (searchdomain_, null, null);
} catch (SearchdomainNotFoundException)
{
logger.LogError("Unable to update searchdomain {searchdomain} - not found", [searchdomain]);
return (null, 500, $"Unable to update searchdomain {searchdomain} - not found");
} catch (Exception ex)
{
logger.LogError("Unable to update searchdomain {searchdomain} - Exception: {ex.Message} - {ex.StackTrace}", [searchdomain, ex.Message, ex.StackTrace]);
return (null, 404, $"Unable to update searchdomain {searchdomain}");
}
}
}

View File

@@ -1,37 +1,29 @@
using System.Text.Json;
using Server.Exceptions;
using Shared.Models;
namespace Server;
public class ProbMethod
{
public Probmethods.probMethodDelegate method;
public ProbMethodEnum probMethodEnum;
public string name;
public ProbMethod(string name, ILogger logger)
public ProbMethod(ProbMethodEnum probMethodEnum, ILogger logger)
{
this.name = name;
this.probMethodEnum = probMethodEnum;
this.name = probMethodEnum.ToString();
Probmethods.probMethodDelegate? probMethod = Probmethods.GetMethod(name);
if (probMethod is null)
{
logger.LogError("Unable to retrieve probMethod {name}", [name]);
throw new ProbMethodNotFoundException(name);
throw new ProbMethodNotFoundException(probMethodEnum);
}
method = probMethod;
}
}
public enum ProbMethodEnum
{
Mean,
HarmonicMean,
QuadraticMean,
GeometricMean,
EVEWAvg,
HVEWAvg,
LVEWAvg,
DictionaryWeightedAverage
}
public static class Probmethods
{
@@ -54,6 +46,11 @@ public static class Probmethods
};
}
public static probMethodDelegate? GetMethod(ProbMethodEnum probMethodEnum)
{
return GetMethod(probMethodEnum.ToString());
}
public static probMethodDelegate? GetMethod(string name)
{
string methodName = name;

View File

@@ -8,12 +8,19 @@ using Server.HealthChecks;
using Server.Helper;
using Server.Models;
using Server.Services;
using System.Text.Json.Serialization;
var builder = WebApplication.CreateBuilder(args);
// Add services to the container.
builder.Services.AddControllersWithViews();
builder.Services.AddControllersWithViews()
.AddJsonOptions(options =>
{
options.JsonSerializerOptions.Converters.Add(
new JsonStringEnumConverter()
);
});
// Add Localization
builder.Services.AddLocalization(options => options.ResourcesPath = "Resources");

View File

@@ -24,4 +24,223 @@
<data name="IrreversibleActionWarning" xml:space="preserve">
<value>Diese Aktion kann nicht rückgängig gemacht werden.</value>
</data>
<data name="Searchdomain selection" xml:space="preserve">
<value>Searchdomain Auswahl</value>
</data>
<data name="Create" xml:space="preserve">
<value>Erstellen</value>
</data>
<data name="Searchdomain information and settings" xml:space="preserve">
<value>Searchdomain Informationen und Einstellungen</value>
</data>
<data name="Actions" xml:space="preserve">
<value>Aktionen</value>
</data>
<data name="Rename" xml:space="preserve">
<value>Umbenennen</value>
</data>
<data name="Delete" xml:space="preserve">
<value>Löschen</value>
</data>
<data name="Settings" xml:space="preserve">
<value>Einstellungen</value>
</data>
<data name="Cache reconciliation" xml:space="preserve">
<value>Cache Abgleich</value>
</data>
<data name="Update" xml:space="preserve">
<value>Anpassen</value>
</data>
<data name="Search cache" xml:space="preserve">
<value>Such-Cache</value>
</data>
<data name="Search cache utilization" xml:space="preserve">
<value>Such-Cache Speicherauslastung</value>
</data>
<data name="Clear" xml:space="preserve">
<value>Leeren</value>
</data>
<data name="Database size" xml:space="preserve">
<value>Größe in der Datenbank</value>
</data>
<data name="Add new entity" xml:space="preserve">
<value>Neue Entity erstellen</value>
</data>
<data name="Entity Details" xml:space="preserve">
<value>Entity Details</value>
</data>
<data name="Attributes" xml:space="preserve">
<value>Attribute</value>
</data>
<data name="Key" xml:space="preserve">
<value>Schlüssel</value>
</data>
<data name="Value" xml:space="preserve">
<value>Wert</value>
</data>
<data name="Datapoints" xml:space="preserve">
<value>Datapoints</value>
</data>
<data name="Name" xml:space="preserve">
<value>Name</value>
</data>
<data name="ProbMethod" xml:space="preserve">
<value>ProbMethod</value>
</data>
<data name="SimilarityMethod" xml:space="preserve">
<value>SimilarityMethod</value>
</data>
<data name="Close" xml:space="preserve">
<value>Schließen</value>
</data>
<data name="Query Details" xml:space="preserve">
<value>Suchanfrage Details</value>
</data>
<data name="Access times" xml:space="preserve">
<value>Zugriffszeiten</value>
</data>
<data name="Results" xml:space="preserve">
<value>Ergebnisse</value>
</data>
<data name="Score" xml:space="preserve">
<value>Bewertung</value>
</data>
<data name="Query Update" xml:space="preserve">
<value>Suchanfrage anpassen</value>
</data>
<data name="Rename searchdomain" xml:space="preserve">
<value>Searchdomain umbenennen</value>
</data>
<data name="Delete searchdomain" xml:space="preserve">
<value>Searchdomain löschen</value>
</data>
<data name="Create searchdomain" xml:space="preserve">
<value>Searchdomain anlegen</value>
</data>
<data name="Searchdomain name" xml:space="preserve">
<value>Searchdomain Name</value>
</data>
<data name="Enable cache reconciliation" xml:space="preserve">
<value>Cache Abgleich verwenden</value>
</data>
<data name="Create entity" xml:space="preserve">
<value>Entity erstellen</value>
</data>
<data name="Entity name" xml:space="preserve">
<value>Entity Name</value>
</data>
<data name="Probmethod" xml:space="preserve">
<value>Probmethod</value>
</data>
<data name="Add attribute" xml:space="preserve">
<value>Attribut hinzufügen</value>
</data>
<data name="Probmethod_embedding" xml:space="preserve">
<value>Probmethod_embedding</value>
</data>
<data name="Similarity method" xml:space="preserve">
<value>Similarity method</value>
</data>
<data name="Model" xml:space="preserve">
<value>Modell</value>
</data>
<data name="Add datapoint" xml:space="preserve">
<value>Datapoint hinzufügen</value>
</data>
<data name="Delete entity" xml:space="preserve">
<value>Entity löschen</value>
</data>
<data name="Update entity" xml:space="preserve">
<value>Entity anpassen</value>
</data>
<data name="Action" xml:space="preserve">
<value>Aktion</value>
</data>
<data name="Delete query" xml:space="preserve">
<value>Suchanfrage löschen</value>
</data>
<data name="Creating entity" xml:space="preserve">
<value>Erstelle Entity</value>
</data>
<data name="Entity was created successfully" xml:space="preserve">
<value>Entity wurde erfolgreich erstellt</value>
</data>
<data name="Failed to create entity" xml:space="preserve">
<value>Entity konnte nicht erstellt werden</value>
</data>
<data name="Searchdomain was created successfully" xml:space="preserve">
<value>Searchdomain wurde erfolgreich erstellt</value>
</data>
<data name="Failed to create searchdomain" xml:space="preserve">
<value>Searchdomain konnte nicht erstellt werden</value>
</data>
<data name="Searchdomain cache was cleared successfully" xml:space="preserve">
<value>Searchdomain Cache wurde erfolgreich geleert</value>
</data>
<data name="Failed to clear searchdomain cache" xml:space="preserve">
<value>Searchdomain Cache konnte nicht geleert werden</value>
</data>
<data name="Entity was deleted successfully" xml:space="preserve">
<value>Entity wurde erfolgreich gelöscht</value>
</data>
<data name="Failed to delete entity" xml:space="preserve">
<value>Entity konnte nicht gelöscht werden</value>
</data>
<data name="Updating entity" xml:space="preserve">
<value>Entity wird angepasst</value>
</data>
<data name="Entity was updated successfully" xml:space="preserve">
<value>Entity wurde erfolgreich angepasst</value>
</data>
<data name="Failed to update entity" xml:space="preserve">
<value>Entity konnte nicht angepasst werden</value>
</data>
<data name="Search query was deleted successfully" xml:space="preserve">
<value>Suchanfrage wurde erfolgreich gelöscht</value>
</data>
<data name="Failed to delete search query" xml:space="preserve">
<value>Suchanfrage konnte nicht gelöscht werden</value>
</data>
<data name="Searchdomain was created successfully" xml:space="preserve">
<value>Searchdomain wurde erfolgreich erstellt</value>
</data>
<data name="Updating search query failed" xml:space="preserve">
<value>Suchanfrage konnte nicht angepasst werden</value>
</data>
<data name="Searchdomain was deleted successfully" xml:space="preserve">
<value>Searchdomain wurde erfolgreich gelöscht</value>
</data>
<data name="Failed to delete searchdomain" xml:space="preserve">
<value>Konnte Searchdomain nicht löschen</value>
</data>
<data name="Searchdomain was renamed successfully" xml:space="preserve">
<value>Searchdomain wurde erfolgreich umbenannt</value>
</data>
<data name="Failed to rename searchdomain" xml:space="preserve">
<value>Searchdomain konnte nicht umbenannt werden</value>
</data>
<data name="Searchdomain settings were updated successfully" xml:space="preserve">
<value>Searchdomain Einstellungen wurden erfolgreich angepasst</value>
</data>
<data name="Updating searchdomain settings failed" xml:space="preserve">
<value>Searchdomain Einstellungen konnten nicht angepasst werden</value>
</data>
<data name="Unable to fetch searchdomain config" xml:space="preserve">
<value>Searchdomain Einstellungen konnten nicht abgerufen werden</value>
</data>
<data name="Unable to fetch searchdomain cache utilization" xml:space="preserve">
<value>Searchdomain Cache-Auslastung konnte nicht abgerufen werden</value>
</data>
<data name="Details" xml:space="preserve">
<value>Details</value>
</data>
<data name="Remove attribute" xml:space="preserve">
<value>Attribut entfernen</value>
</data>
<data name="Remove" xml:space="preserve">
<value>Entfernen</value>
</data>
<data name="Close alert" xml:space="preserve">
<value>Benachrichtigung schließen</value>
</data>
</root>

View File

@@ -24,4 +24,223 @@
<data name="IrreversibleActionWarning" xml:space="preserve">
<value>This action cannot be undone.</value>
</data>
<data name="Searchdomain selection" xml:space="preserve">
<value>Searchdomain selection</value>
</data>
<data name="Create" xml:space="preserve">
<value>Create</value>
</data>
<data name="Searchdomain information and settings" xml:space="preserve">
<value>Searchdomain information and settings</value>
</data>
<data name="Actions" xml:space="preserve">
<value>Actions</value>
</data>
<data name="Rename" xml:space="preserve">
<value>Rename</value>
</data>
<data name="Delete" xml:space="preserve">
<value>Delete</value>
</data>
<data name="Settings" xml:space="preserve">
<value>Settings</value>
</data>
<data name="Cache reconciliation" xml:space="preserve">
<value>Cache reconciliation</value>
</data>
<data name="Update" xml:space="preserve">
<value>Update</value>
</data>
<data name="Search cache" xml:space="preserve">
<value>Search cache</value>
</data>
<data name="Search cache utilization" xml:space="preserve">
<value>Search cache utilization</value>
</data>
<data name="Clear" xml:space="preserve">
<value>Clear</value>
</data>
<data name="Database size" xml:space="preserve">
<value>Database size</value>
</data>
<data name="Add new entity" xml:space="preserve">
<value>Add new entity</value>
</data>
<data name="Entity Details" xml:space="preserve">
<value>Entity Details</value>
</data>
<data name="Attributes" xml:space="preserve">
<value>Attributes</value>
</data>
<data name="Key" xml:space="preserve">
<value>Key</value>
</data>
<data name="Value" xml:space="preserve">
<value>Value</value>
</data>
<data name="Datapoints" xml:space="preserve">
<value>Datapoints</value>
</data>
<data name="Name" xml:space="preserve">
<value>Name</value>
</data>
<data name="ProbMethod" xml:space="preserve">
<value>ProbMethod</value>
</data>
<data name="SimilarityMethod" xml:space="preserve">
<value>SimilarityMethod</value>
</data>
<data name="Close" xml:space="preserve">
<value>Close</value>
</data>
<data name="Query Details" xml:space="preserve">
<value>Query Details</value>
</data>
<data name="Access times" xml:space="preserve">
<value>Access times</value>
</data>
<data name="Results" xml:space="preserve">
<value>Results</value>
</data>
<data name="Score" xml:space="preserve">
<value>Score</value>
</data>
<data name="Query Update" xml:space="preserve">
<value>Query Update</value>
</data>
<data name="Rename searchdomain" xml:space="preserve">
<value>Rename searchdomain</value>
</data>
<data name="Delete searchdomain" xml:space="preserve">
<value>Delete searchdomain</value>
</data>
<data name="Create searchdomain" xml:space="preserve">
<value>Create searchdomain</value>
</data>
<data name="Searchdomain name" xml:space="preserve">
<value>Searchdomain name</value>
</data>
<data name="Enable cache reconciliation" xml:space="preserve">
<value>Enable cache reconciliation</value>
</data>
<data name="Create entity" xml:space="preserve">
<value>Create entity</value>
</data>
<data name="Entity name" xml:space="preserve">
<value>Entity name</value>
</data>
<data name="Probmethod" xml:space="preserve">
<value>Probmethod</value>
</data>
<data name="Add attribute" xml:space="preserve">
<value>Add attribute</value>
</data>
<data name="Probmethod_embedding" xml:space="preserve">
<value>Probmethod_embedding</value>
</data>
<data name="Similarity method" xml:space="preserve">
<value>Similarity method</value>
</data>
<data name="Model" xml:space="preserve">
<value>Model</value>
</data>
<data name="Add datapoint" xml:space="preserve">
<value>Add datapoint</value>
</data>
<data name="Delete entity" xml:space="preserve">
<value>Delete entity</value>
</data>
<data name="Update entity" xml:space="preserve">
<value>Update entity</value>
</data>
<data name="Action" xml:space="preserve">
<value>Action</value>
</data>
<data name="Delete query" xml:space="preserve">
<value>Delete query</value>
</data>
<data name="Creating entity" xml:space="preserve">
<value>Creating entity</value>
</data>
<data name="Entity was created successfully" xml:space="preserve">
<value>Entity was created successfully</value>
</data>
<data name="Failed to create entity" xml:space="preserve">
<value>Failed to create entity</value>
</data>
<data name="Searchdomain was created successfully" xml:space="preserve">
<value>Searchdomain was created successfully</value>
</data>
<data name="Failed to create searchdomain" xml:space="preserve">
<value>Failed to create searchdomain</value>
</data>
<data name="Searchdomain cache was cleared successfully" xml:space="preserve">
<value>Searchdomain cache was cleared successfully</value>
</data>
<data name="Failed to clear searchdomain cache" xml:space="preserve">
<value>Failed to clear searchdomain cache</value>
</data>
<data name="Entity was deleted successfully" xml:space="preserve">
<value>Entity was deleted successfully</value>
</data>
<data name="Failed to delete entity" xml:space="preserve">
<value>Failed to delete entity</value>
</data>
<data name="Updating entity" xml:space="preserve">
<value>Updating entity</value>
</data>
<data name="Entity was updated successfully" xml:space="preserve">
<value>Entity was updated successfully</value>
</data>
<data name="Failed to update entity" xml:space="preserve">
<value>Failed to update entity</value>
</data>
<data name="Search query was deleted successfully" xml:space="preserve">
<value>Search query was deleted successfully</value>
</data>
<data name="Failed to delete search query" xml:space="preserve">
<value>Failed to delete search query</value>
</data>
<data name="Searchdomain was created successfully" xml:space="preserve">
<value>Searchdomain was created successfully</value>
</data>
<data name="Updating search query failed" xml:space="preserve">
<value>Updating search query failed</value>
</data>
<data name="Searchdomain was deleted successfully" xml:space="preserve">
<value>Searchdomain was deleted successfully</value>
</data>
<data name="Failed to delete searchdomain" xml:space="preserve">
<value>Failed to delete searchdomain</value>
</data>
<data name="Searchdomain was renamed successfully" xml:space="preserve">
<value>Searchdomain was renamed successfully</value>
</data>
<data name="Failed to rename searchdomain" xml:space="preserve">
<value>Failed to rename searchdomain</value>
</data>
<data name="Searchdomain settings were updated successfully" xml:space="preserve">
<value>Searchdomain settings were updated successfully</value>
</data>
<data name="Updating searchdomain settings failed" xml:space="preserve">
<value>Updating searchdomain settings failed</value>
</data>
<data name="Unable to fetch searchdomain config" xml:space="preserve">
<value>Unable to fetch searchdomain config</value>
</data>
<data name="Unable to fetch searchdomain cache utilization" xml:space="preserve">
<value>"Unable to fetch searchdomain cache utilization</value>
</data>
<data name="Details" xml:space="preserve">
<value>Details</value>
</data>
<data name="Remove attribute" xml:space="preserve">
<value>Remove attribute</value>
</data>
<data name="Remove" xml:space="preserve">
<value>Remove</value>
</data>
<data name="Close alert" xml:space="preserve">
<value>Close alert</value>
</data>
</root>

View File

@@ -5,6 +5,7 @@ using ElmahCore.Mvc.Logger;
using MySql.Data.MySqlClient;
using Server.Helper;
using Shared.Models;
using AdaptiveExpressions;
namespace Server;
@@ -19,13 +20,12 @@ public class Searchdomain
public Dictionary<string, DateTimedSearchResult> searchCache; // Key: query, Value: Search results for that query (with timestamp)
public List<Entity> entityCache;
public List<string> modelsInUse;
public Dictionary<string, Dictionary<string, float[]>> embeddingCache;
public int embeddingCacheMaxSize = 10000000;
public LRUCache<string, Dictionary<string, float[]>> embeddingCache;
private readonly MySqlConnection connection;
public SQLHelper helper;
private readonly ILogger _logger;
public Searchdomain(string searchdomain, string connectionString, AIProvider aIProvider, Dictionary<string, Dictionary<string, float[]>> embeddingCache, ILogger logger, string provider = "sqlserver", bool runEmpty = false)
public Searchdomain(string searchdomain, string connectionString, AIProvider aIProvider, LRUCache<string, Dictionary<string, float[]>> embeddingCache, ILogger logger, string provider = "sqlserver", bool runEmpty = false)
{
_connectionString = connectionString;
_provider = provider.ToLower();
@@ -96,8 +96,16 @@ public class Searchdomain
string probmethodString = datapointReader.GetString(3);
string similarityMethodString = datapointReader.GetString(4);
string hash = datapointReader.GetString(5);
ProbMethod probmethod = new(probmethodString, _logger);
SimilarityMethod similarityMethod = new(similarityMethodString, _logger);
ProbMethodEnum probmethodEnum = (ProbMethodEnum)Enum.Parse(
typeof(ProbMethodEnum),
probmethodString
);
SimilarityMethodEnum similairtyMethodEnum = (SimilarityMethodEnum)Enum.Parse(
typeof(SimilarityMethodEnum),
similarityMethodString
);
ProbMethod probmethod = new(probmethodEnum, _logger);
SimilarityMethod similarityMethod = new(similairtyMethodEnum, _logger);
if (embedding_unassigned.TryGetValue(id, out Dictionary<string, float[]>? embeddings) && probmethod is not null)
{
embedding_unassigned.Remove(id);
@@ -151,7 +159,6 @@ public class Searchdomain
}
entityReader.Close();
modelsInUse = GetModels(entityCache);
embeddingCache = []; // TODO remove this and implement proper remediation to improve performance
}
public List<(float, string)> Search(string query, int? topN = null)
@@ -162,33 +169,13 @@ public class Searchdomain
return [.. cachedResult.Results.Select(r => (r.Score, r.Name))];
}
if (!embeddingCache.TryGetValue(query, out Dictionary<string, float[]>? queryEmbeddings))
{
queryEmbeddings = Datapoint.GenerateEmbeddings(query, modelsInUse, aIProvider);
if (embeddingCache.Count < embeddingCacheMaxSize) // TODO add better way of managing cache limit hits
{ // Idea: Add access count to each entry. On limit hit, sort the entries by access count and remove the bottom 10% of entries
embeddingCache.Add(query, queryEmbeddings);
}
} // TODO implement proper cache remediation for embeddingCache here
Dictionary<string, float[]> queryEmbeddings = GetQueryEmbeddings(query);
List<(float, string)> result = [];
foreach (Entity entity in entityCache)
{
List<(string, float)> datapointProbs = [];
foreach (Datapoint datapoint in entity.datapoints)
{
SimilarityMethod similarityMethod = datapoint.similarityMethod;
List<(string, float)> list = [];
foreach ((string, float[]) embedding in datapoint.embeddings)
{
string key = embedding.Item1;
float value = similarityMethod.method(queryEmbeddings[embedding.Item1], embedding.Item2);
list.Add((key, value));
}
datapointProbs.Add((datapoint.name, datapoint.probMethod.method(list)));
}
result.Add((entity.probMethod(datapointProbs), entity.name));
result.Add((EvaluateEntityAgainstQueryEmbeddings(entity, queryEmbeddings), entity.name));
}
IEnumerable<(float, string)> sortedResults = result.OrderByDescending(s => s.Item1);
if (topN is not null)
@@ -204,6 +191,49 @@ public class Searchdomain
return results;
}
public Dictionary<string, float[]> GetQueryEmbeddings(string query)
{
bool hasQuery = embeddingCache.TryGet(query, out Dictionary<string, float[]> queryEmbeddings);
bool allModelsInQuery = queryEmbeddings is not null && modelsInUse.All(model => queryEmbeddings.ContainsKey(model));
if (!(hasQuery && allModelsInQuery) || queryEmbeddings is null)
{
queryEmbeddings = Datapoint.GenerateEmbeddings(query, modelsInUse, aIProvider, embeddingCache);
if (!embeddingCache.TryGet(query, out var embeddingCacheForCurrentQuery))
{
embeddingCache.Set(query, queryEmbeddings);
}
else // embeddingCache already has an entry for this query, so the missing model-embedding pairs have to be filled in
{
foreach (KeyValuePair<string, float[]> kvp in queryEmbeddings) // kvp.Key = model, kvp.Value = embedding
{
if (!embeddingCache.TryGet(kvp.Key, out var _))
{
embeddingCacheForCurrentQuery[kvp.Key] = kvp.Value;
}
}
}
}
return queryEmbeddings;
}
private static float EvaluateEntityAgainstQueryEmbeddings(Entity entity, Dictionary<string, float[]> queryEmbeddings)
{
List<(string, float)> datapointProbs = [];
foreach (Datapoint datapoint in entity.datapoints)
{
SimilarityMethod similarityMethod = datapoint.similarityMethod;
List<(string, float)> list = [];
foreach ((string, float[]) embedding in datapoint.embeddings)
{
string key = embedding.Item1;
float value = similarityMethod.method(queryEmbeddings[embedding.Item1], embedding.Item2);
list.Add((key, value));
}
datapointProbs.Add((datapoint.name, datapoint.probMethod.method(list)));
}
return entity.probMethod(datapointProbs);
}
public static List<string> GetModels(List<Entity> entities)
{
List<string> result = [];
@@ -250,6 +280,53 @@ public class Searchdomain
return JsonSerializer.Deserialize<SearchdomainSettings>(settingsString);
}
public void ReconciliateOrInvalidateCacheForNewOrUpdatedEntity(Entity entity)
{
if (settings.CacheReconciliation)
{
foreach (KeyValuePair<string, DateTimedSearchResult> element in searchCache)
{
string query = element.Key;
DateTimedSearchResult searchResult = element.Value;
Dictionary<string, float[]> queryEmbeddings = GetQueryEmbeddings(query);
float evaluationResult = EvaluateEntityAgainstQueryEmbeddings(entity, queryEmbeddings);
searchResult.Results.RemoveAll(x => x.Name == entity.name); // If entity already exists in that results list: remove it.
ResultItem newItem = new(evaluationResult, entity.name);
int index = searchResult.Results.BinarySearch(
newItem,
Comparer<ResultItem>.Create((a, b) => b.Score.CompareTo(a.Score)) // Invert searching order
);
if (index < 0) // If not found, BinarySearch gives the bitwise complement
index = ~index;
searchResult.Results.Insert(index, newItem);
}
}
else
{
InvalidateSearchCache();
}
}
public void ReconciliateOrInvalidateCacheForDeletedEntity(Entity entity)
{
if (settings.CacheReconciliation)
{
foreach (KeyValuePair<string, DateTimedSearchResult> element in searchCache)
{
string query = element.Key;
DateTimedSearchResult searchResult = element.Value;
searchResult.Results.RemoveAll(x => x.Name == entity.name);
}
}
else
{
InvalidateSearchCache();
}
}
public void InvalidateSearchCache()
{
searchCache = [];

View File

@@ -3,6 +3,9 @@ using System.Data.Common;
using Server.Migrations;
using Server.Helper;
using Server.Exceptions;
using AdaptiveExpressions;
using Shared.Models;
using System.Text.Json;
namespace Server;
@@ -16,7 +19,8 @@ public class SearchdomainManager
private readonly string connectionString;
private MySqlConnection connection;
public SQLHelper helper;
public Dictionary<string, Dictionary<string, float[]>> embeddingCache;
public LRUCache<string, Dictionary<string, float[]>> embeddingCache;
public int EmbeddingCacheMaxCount;
public SearchdomainManager(ILogger<SearchdomainManager> logger, IConfiguration config, AIProvider aIProvider, DatabaseHelper databaseHelper)
{
@@ -24,7 +28,8 @@ public class SearchdomainManager
_config = config;
this.aIProvider = aIProvider;
_databaseHelper = databaseHelper;
embeddingCache = [];
EmbeddingCacheMaxCount = config.GetValue<int?>("Embeddingsearch:EmbeddingCacheMaxCount") ?? 1000000;
embeddingCache = new(EmbeddingCacheMaxCount);
connectionString = _config.GetSection("Embeddingsearch").GetConnectionString("SQL") ?? "";
connection = new MySqlConnection(connectionString);
connection.Open();
@@ -66,7 +71,7 @@ public class SearchdomainManager
{
var searchdomain = GetSearchdomain(searchdomainName);
searchdomain.UpdateEntityCache();
searchdomain.InvalidateSearchCache(); // TODO implement cache remediation (Suggestion: searchdomain-wide setting for cache remediation / invalidation - )
searchdomain.InvalidateSearchCache();
}
public List<string> ListSearchdomains()
@@ -84,6 +89,10 @@ public class SearchdomainManager
}
}
public int CreateSearchdomain(string searchdomain, SearchdomainSettings settings)
{
return CreateSearchdomain(searchdomain, JsonSerializer.Serialize(settings));
}
public int CreateSearchdomain(string searchdomain, string settings = "{}")
{
if (searchdomains.TryGetValue(searchdomain, out Searchdomain? value))

View File

@@ -7,6 +7,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="AdaptiveExpressions" Version="4.23.0" />
<PackageReference Include="ElmahCore" Version="2.1.2" />
<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
<PackageReference Include="Serilog.AspNetCore" Version="9.0.0" />

View File

@@ -1,16 +1,18 @@
using System.Numerics.Tensors;
using System.Text.Json;
using Shared.Models;
namespace Server;
public class SimilarityMethod
{
public SimilarityMethods.similarityMethodDelegate method;
public SimilarityMethodEnum similarityMethodEnum;
public string name;
public SimilarityMethod(string name, ILogger logger)
public SimilarityMethod(SimilarityMethodEnum similarityMethodEnum, ILogger logger)
{
this.name = name;
this.similarityMethodEnum = similarityMethodEnum;
this.name = similarityMethodEnum.ToString();
SimilarityMethods.similarityMethodDelegate? probMethod = SimilarityMethods.GetMethod(name);
if (probMethod is null)
{
@@ -21,14 +23,6 @@ public class SimilarityMethod
}
}
public enum SimilarityMethodEnum
{
Cosine,
Euclidian,
Manhattan,
Pearson
}
public static class SimilarityMethods
{
public delegate float similarityMethodProtoDelegate(float[] vector1, float[] vector2);

View File

@@ -1,5 +1,6 @@
@using Server.Models
@using System.Web
@using Shared.Models
@using Server.Services
@using Server
@@ -744,8 +745,8 @@
"datapoints": datapoints
}];
showToast("@T["Creating entity"]", "primary");
fetch(`/Entity/Index`, {
method: 'POST',
fetch(`/Entity`, {
method: 'PUT',
headers: {
'Content-Type': 'application/json'
},
@@ -777,8 +778,12 @@
const cacheReconciliation = document.getElementById('createSearchdomainWithCacheReconciliation').checked;
const settings = { CacheReconciliation: cacheReconciliation };
// Implement create logic here
fetch(`/Searchdomain/Create?searchdomain=${encodeURIComponent(name)}&settings=${JSON.stringify(settings)}`, {
method: 'GET'
fetch(`/Searchdomain?searchdomain=${encodeURIComponent(name)}`, {
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify(settings)
}).then(response => {
if (response.ok) {
showToast("@T["Searchdomain was created successfully"]", "success");
@@ -799,8 +804,8 @@
.getElementById('cacheClear')
.addEventListener('click', () => {
const domainKey = getSelectedDomainKey();
fetch(`/Searchdomain/ClearSearchCache?searchdomain=${encodeURIComponent(domains[domainKey])}`, {
method: 'GET'
fetch(`/Searchdomain/SearchCache/Clear?searchdomain=${encodeURIComponent(domains[domainKey])}`, {
method: 'POST'
}).then(response => {
if (response.ok) {
showToast("@T["Searchdomain cache was cleared successfully"]", "success");
@@ -822,8 +827,8 @@
.addEventListener('click', () => {
const domainKey = getSelectedDomainKey();
const entityName = document.getElementById('EntityConfirmDelete').getAttribute('data-name');
fetch(`/Entity/Delete?searchdomain=${encodeURIComponent(domains[domainKey])}&entityName=${entityName}`, {
method: 'GET'
fetch(`/Entity?searchdomain=${encodeURIComponent(domains[domainKey])}&entityName=${entityName}`, {
method: 'DELETE'
}).then(async response => {
let result = await response.json();
if (response.ok && result.Success) {
@@ -831,10 +836,11 @@
console.log('Entity deleted successfully');
selectDomain(getSelectedDomainKey());
} else {
showToast("@T["Failed to delete entity"]", "danger");
showToast("@Html.Raw(T["Failed to delete entity"])", "danger");
console.error('Failed to delete entity:', result.Message);
}
}).catch(error => {
showToast("@Html.Raw(T["Failed to delete entity"])", "danger");
console.error('Error deleting entity:', error);
});
});
@@ -868,8 +874,8 @@
"datapoints": datapoints
}];
showToast("@T["Updating entity"]", "primary");
fetch(`/Entity/Index`, {
method: 'POST',
fetch(`/Entity`, {
method: 'PUT',
headers: {
'Content-Type': 'application/json'
},
@@ -895,7 +901,7 @@
.addEventListener('click', () => {
let searchdomain = domains[getSelectedDomainKey()];
let query = document.getElementById('deleteQueryConfirmationModalName').textContent;
fetch(`/Searchdomain/Searches?searchdomain=${searchdomain}&query=${query}`, {
fetch(`/Searchdomain/Query?searchdomain=${searchdomain}&query=${query}`, {
method: 'DELETE',
headers: {
'Content-Type': 'application/json'
@@ -907,11 +913,11 @@
console.log('Search query was deleted successfully');
selectDomain(getSelectedDomainKey());
} else {
showToast("@T["Failed to delete search query"]", "danger");
showToast("@Html.Raw(T["Failed to delete search query"])", "danger");
console.error('Failed to delete search query:', result.Message);
}
}).catch(error => {
showToast("@T["Failed to delete search query"]", "danger");
showToast("@Html.Raw(T["Failed to delete search query"])", "danger");
console.error('Failed to delete search query:', error);
});
});
@@ -922,7 +928,7 @@
let query = document.getElementById('queryUpdateQueryName').textContent;
let data = getQueryUpdateTableData();
console.log()
fetch(`/Searchdomain/Searches?searchdomain=${searchdomain}&query=${query}`, {
fetch(`/Searchdomain/Query?searchdomain=${searchdomain}&query=${query}`, {
method: 'PATCH',
headers: {
'Content-Type': 'application/json'
@@ -947,8 +953,8 @@
function deleteSearchdomain(domainKey) {
// Implement delete logic here
fetch(`/Searchdomain/Delete?searchdomain=${encodeURI(domains[domainKey])}`, {
method: 'GET'
fetch(`/Searchdomain?searchdomain=${encodeURI(domains[domainKey])}`, {
method: 'DELETE'
}).then(async response => {
var result = await response.json();;
if (response.ok && result.Success === true) {
@@ -958,19 +964,19 @@
domainItem.remove();
console.log('Searchdomain deleted successfully');
} else {
showToast("@T["Failed to delete searchdomain"]", "danger");
showToast("@Html.Raw(T["Failed to delete searchdomain"])", "danger");
console.error('Failed to delete searchdomain:', result.Message);
}
}).catch(error => {
showToast("@T["Failed to delete searchdomain"]", "danger");
showToast("@Html.Raw(T["Failed to delete searchdomain"])", "danger");
console.error('Error deleting searchdomain:', error);
});
}
function renameSearchdomain(domainKey, newName) {
// Implement rename logic here
fetch(`/Searchdomain/Update?searchdomain=${encodeURI(domains[domainKey])}&newName=${newName}`, {
method: 'GET'
fetch(`/Searchdomain?searchdomain=${encodeURI(domains[domainKey])}&newName=${newName}`, {
method: 'PUT'
}).then(async response => {
var result = await response.json();
if (response.ok && result.Success === true) {
@@ -994,8 +1000,8 @@
function updateSearchdomainConfig(domainKey, newSettings) {
// Implement update logic here
fetch(`/Searchdomain/UpdateSettings?searchdomain=${encodeURIComponent(domains[domainKey])}`, {
method: 'POST',
fetch(`/Searchdomain/Settings?searchdomain=${encodeURIComponent(domains[domainKey])}`, {
method: 'PUT',
headers: {
'Content-Type': 'application/json'
},
@@ -1020,17 +1026,17 @@
}
function getSearchdomainConfig(domainKey) {
return fetch(`/Searchdomain/GetSettings?searchdomain=${encodeURIComponent(domains[domainKey])}`)
return fetch(`/Searchdomain/Settings?searchdomain=${encodeURIComponent(domains[domainKey])}`)
.then(r => r.json());
}
function getSearchdomainCacheUtilization(domainKey) {
return fetch(`/Searchdomain/GetSearchCacheSize?searchdomain=${encodeURIComponent(domains[domainKey])}`)
return fetch(`/Searchdomain/SearchCache/Size?searchdomain=${encodeURIComponent(domains[domainKey])}`)
.then(r => r.json());
}
function getSearchdomainDatabaseUtilization(domainKey) {
return fetch(`/Searchdomain/GetDatabaseSize?searchdomain=${encodeURIComponent(domains[domainKey])}`)
return fetch(`/Searchdomain/Database/Size?searchdomain=${encodeURIComponent(domains[domainKey])}`)
.then(r => r.json());
}
@@ -1052,7 +1058,7 @@
let databaseUtilizationPromise = getSearchdomainDatabaseUtilization(getSelectedDomainKey());
/* ---------- ENTITIES ---------- */
let entitiesUrl = `/Entity/List?searchdomain=${encodeURIComponent(domainName)}&returnEmbeddings=false&returnModels=true`;
let entitiesUrl = `/Entities?searchdomain=${encodeURIComponent(domainName)}&returnEmbeddings=false&returnModels=true`;
let entitiesCard = document.querySelector("#entitiesTable").parentElement;
clearEntitiesTable();
showThrobber(entitiesCard);
@@ -1071,7 +1077,7 @@
});
/* ---------- QUERIES ---------- */
let queriesUrl = `/Searchdomain/GetSearches?searchdomain=${encodeURIComponent(domainName)}`;
let queriesUrl = `/Searchdomain/Queries?searchdomain=${encodeURIComponent(domainName)}`;
let queriesCard = document.querySelector("#queriesTable").parentElement;
clearQueriesTable();
showThrobber(queriesCard);
@@ -1091,7 +1097,6 @@
searchdomainConfigPromise.then(searchdomainConfig => {
if (searchdomainConfig != null && searchdomainConfig.Settings != null)
{
console.log(searchdomainConfig);
configElementCachereconciliation.checked = searchdomainConfig.Settings.CacheReconciliation;
configElementCachereconciliation.disabled = false;
} else {
@@ -1197,7 +1202,7 @@
var deleteButton = document.createElement('button');
deleteButton.className = 'btn btn-danger btn-sm';
deleteButton.textContent = '@T["Delete"]';
deleteButton.textContent = '@Html.Raw(T["Delete"])';
deleteButton.setAttribute("data-index", entities.findIndex(en => en == entity));
deleteButton.addEventListener('click', () => {
const modal = new bootstrap.Modal(
@@ -1255,7 +1260,7 @@
const btnDelete = document.createElement('button');
btnDelete.className = 'btn btn-sm btn-danger';
btnDelete.textContent = '@T["Delete"]';
btnDelete.textContent = '@Html.Raw(T["Delete"])';
btnDelete.addEventListener('click', () => {
const modal = new bootstrap.Modal(
document.getElementById('deleteQueryModal')
@@ -1296,7 +1301,7 @@
function showEntityDetails(entity) {
// Title
document.getElementById('entityDetailsTitle').innerText = entity.Name;
document.getElementById('entityDetailsTitle').innerText = '@T["Details"] - ' + entity.Name;
// Attributes
const attrBody = document.getElementById('entityAttributesBody');
@@ -1442,7 +1447,7 @@
const tdAction = document.createElement('td');
const deleteButton = document.createElement('button');
deleteButton.classList.add('btn', 'btn-danger', 'btn-sm');
deleteButton.innerText = '@T["Delete"]';
deleteButton.innerText = '@Html.Raw(T["Delete"])';
deleteButton.onclick = function() {
row.remove();
};
@@ -1541,7 +1546,7 @@
var tdKey = document.createElement('td');
var keyInput = document.createElement('input');
keyInput.classList.add('form-control');
keyInput.ariaLabel = '@T["Key"]';
keyInput.ariaLabel = '@Html.Raw(T["Key"])';
keyInput.value = key;
tdKey.append(keyInput);
var tdValue = document.createElement('td');

View File

@@ -1,6 +0,0 @@
@{
ViewData["Title"] = "Privacy Policy";
}
<h1>@ViewData["Title"]</h1>
<p>Use this page to detail your site's privacy policy.</p>

View File

@@ -54,7 +54,7 @@
<footer class="border-top footer text-muted">
<div class="container">
&copy; 2025 - embeddingsearch - <a asp-area="" asp-controller="Home" asp-action="Privacy">Privacy</a>
&copy; 2025 - embeddingsearch
</div>
</footer>
<script src="~/lib/jquery/dist/jquery.min.js"></script>

View File

@@ -24,6 +24,7 @@
"172.17.0.1"
]
},
"EmbeddingCacheMaxCount": 5,
"AiProviders": {
"ollama": {
"handler": "ollama",
@@ -35,6 +36,15 @@
"ApiKey": "Some API key here"
}
},
"SimpleAuth": {
"Users": [
{
"Username": "admin",
"Password": "UnsafePractice.67",
"Roles": ["Admin"]
}
]
},
"ApiKeys": ["Some UUID here", "Another UUID here"],
"UseHttpsRedirection": true
}

View File

@@ -0,0 +1,12 @@
using System.Text.Json.Serialization;
namespace Shared.Models;
public class SuccesMessageBaseModel
{
[JsonPropertyName("Success")]
public required bool Success { get; set; }
[JsonPropertyName("Message")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? Message { get; set; }
}

View File

@@ -3,14 +3,10 @@ using System.Text.Json.Serialization;
namespace Shared.Models;
public class EntityQueryResults
public class EntityQueryResults : SuccesMessageBaseModel
{
[JsonPropertyName("Results")]
public required List<EntityQueryResult> Results { get; set; }
[JsonPropertyName("Success")]
public required bool Success { get; set; }
[JsonPropertyName("Message")]
public string? Message { get; set; }
}
public class EntityQueryResult
@@ -19,20 +15,19 @@ public class EntityQueryResult
public required string Name { get; set; }
[JsonPropertyName("Value")]
public float Value { get; set; }
[JsonPropertyName("Attributes")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public Dictionary<string, string>? Attributes { get; set; }
}
public class EntityIndexResult
{
[JsonPropertyName("Success")]
public required bool Success { get; set; }
[JsonPropertyName("Message")]
public string? Message { get; set; }
}
public class EntityIndexResult : SuccesMessageBaseModel {}
public class EntityListResults
{
[JsonPropertyName("Results")]
public required List<EntityListResult> Results { get; set; }
[JsonPropertyName("Message")]
public string? Message { get; set; }
[JsonPropertyName("Success")]
public required bool Success { get; set; }
}
@@ -77,11 +72,5 @@ public class EmbeddingResult
public required float[] Embeddings { get; set; }
}
public class EntityDeleteResults
{
[JsonPropertyName("Success")]
public required bool Success { get; set; }
[JsonPropertyName("Message")]
public string? Message { get; set; }
}
public class EntityDeleteResults : SuccesMessageBaseModel {}

View File

@@ -3,7 +3,7 @@ namespace Shared.Models;
public class JSONEntity
{
public required string Name { get; set; }
public required string Probmethod { get; set; }
public required ProbMethodEnum Probmethod { get; set; }
public required string Searchdomain { get; set; }
public required Dictionary<string, string> Attributes { get; set; }
public required JSONDatapoint[] Datapoints { get; set; }
@@ -13,7 +13,27 @@ public class JSONDatapoint
{
public required string Name { get; set; }
public required string? Text { get; set; }
public required string Probmethod_embedding { get; set; }
public required string SimilarityMethod { get; set; }
public required ProbMethodEnum Probmethod_embedding { get; set; }
public required SimilarityMethodEnum SimilarityMethod { get; set; }
public required string[] Model { get; set; }
}
public enum ProbMethodEnum
{
Mean,
HarmonicMean,
QuadraticMean,
GeometricMean,
EVEWAvg,
HVEWAvg,
LVEWAvg,
DictionaryWeightedAverage
}
public enum SimilarityMethodEnum
{
Cosine,
Euclidian,
Manhattan,
Pearson
}

View File

@@ -11,109 +11,46 @@ public class SearchdomainListResults
public string? Message { get; set; }
}
public class SearchdomainCreateResults
public class SearchdomainCreateResults : SuccesMessageBaseModel
{
[JsonPropertyName("Success")]
public required bool Success { get; set; }
[JsonPropertyName("Message")]
public string? Message { get; set; }
[JsonPropertyName("Id")]
public int? Id { get; set; }
}
public class SearchdomainUpdateResults
public class SearchdomainUpdateResults : SuccesMessageBaseModel {}
public class SearchdomainDeleteResults : SuccesMessageBaseModel
{
[JsonPropertyName("Success")]
public required bool Success { get; set; }
[JsonPropertyName("Message")]
public string? Message { get; set; }
}
public class SearchdomainDeleteResults
{
[JsonPropertyName("Success")]
public required bool Success { get; set; }
[JsonPropertyName("Message")]
public string? Message { get; set; }
[JsonPropertyName("DeletedEntities")]
public required int DeletedEntities { get; set; }
}
public class SearchdomainSearchesResults
public class SearchdomainSearchesResults : SuccesMessageBaseModel
{
[JsonPropertyName("Success")]
public required bool Success { get; set; }
[JsonPropertyName("Message")]
public string? Message { get; set; }
[JsonPropertyName("Searches")]
public required Dictionary<string, DateTimedSearchResult> Searches { get; set; }
}
public class SearchdomainDeleteSearchResult
public class SearchdomainDeleteSearchResult : SuccesMessageBaseModel {}
public class SearchdomainUpdateSearchResult : SuccesMessageBaseModel {}
public class SearchdomainSettingsResults : SuccesMessageBaseModel
{
[JsonPropertyName("Success")]
public required bool Success { get; set; }
[JsonPropertyName("Message")]
public string? Message { get; set; }
}
public class SearchdomainUpdateSearchResult
{
[JsonPropertyName("Success")]
public required bool Success { get; set; }
[JsonPropertyName("Message")]
public string? Message { get; set; }
}
public class SearchdomainSettingsResults
{
[JsonPropertyName("Success")]
public required bool Success { get; set; }
[JsonPropertyName("Message")]
public string? Message { get; set; }
[JsonPropertyName("Settings")]
public required SearchdomainSettings? Settings { get; set; }
}
public class SearchdomainSearchCacheSizeResults
public class SearchdomainSearchCacheSizeResults : SuccesMessageBaseModel
{
[JsonPropertyName("Success")]
public required bool Success { get; set; }
[JsonPropertyName("Message")]
public string? Message { get; set; }
[JsonPropertyName("SearchCacheSizeBytes")]
public required long? SearchCacheSizeBytes { get; set; }
}
public class SearchdomainInvalidateCacheResults
public class SearchdomainInvalidateCacheResults : SuccesMessageBaseModel {}
public class SearchdomainGetDatabaseSizeResult : SuccesMessageBaseModel
{
[JsonPropertyName("Success")]
public required bool Success { get; set; }
[JsonPropertyName("Message")]
public string? Message { get; set; }
}
public class SearchdomainGetDatabaseSizeResult
{
[JsonPropertyName("Success")]
public required bool Success { get; set; }
[JsonPropertyName("Message")]
public string? Message { get; set; }
[JsonPropertyName("SearchdomainDatabaseSizeBytes")]
public required long? SearchdomainDatabaseSizeBytes { get; set; }
}

View File

@@ -2,14 +2,8 @@ using System.Text.Json.Serialization;
namespace Shared.Models;
public class ServerGetModelsResult
public class ServerGetModelsResult : SuccesMessageBaseModel
{
[JsonPropertyName("Success")]
public required bool Success { get; set; }
[JsonPropertyName("Message")]
public string? Message { get; set; }
[JsonPropertyName("Models")]
public string[]? Models { get; set; }
}