Compare commits
28 Commits
41-create-
...
d647bedb33
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d647bedb33 | ||
| fe6bbfe9e5 | |||
|
|
6f7afca195 | ||
| 3fa71a8d8b | |||
|
|
8921121078 | ||
| baf76685b7 | |||
|
|
4030e4a824 | ||
| 7b4a3bd2c8 | |||
|
|
5eabb0d924 | ||
| 40424053da | |||
| f3a4665153 | |||
| a358eaea86 | |||
| 665a392b5a | |||
| 26d0561c3b | |||
| cc93a76546 | |||
|
|
7298593341 | ||
| 25723cb7a4 | |||
|
|
84d83206cb | ||
| b6e01a3f66 | |||
|
|
e4cfcb1030 | ||
| 6d1cffe2db | |||
|
|
dd0019b1c1 | ||
| 5877ebaff2 | |||
| 040d4f916a | |||
|
|
57beddd70f | ||
| 8416d7f404 | |||
| 16f08aa8a7 | |||
|
|
cce42d8ec3 |
@@ -36,7 +36,7 @@ public class Client
|
||||
|
||||
public async Task<SearchdomainListResults> SearchdomainListAsync()
|
||||
{
|
||||
return await GetUrlAndProcessJson<SearchdomainListResults>(GetUrl($"{baseUri}/Searchdomain", "List", apiKey, []));
|
||||
return await GetUrlAndProcessJson<SearchdomainListResults>(GetUrl($"{baseUri}", "Searchdomains", apiKey, []));
|
||||
}
|
||||
|
||||
public async Task<SearchdomainDeleteResults> SearchdomainDeleteAsync()
|
||||
@@ -46,7 +46,7 @@ public class Client
|
||||
|
||||
public async Task<SearchdomainDeleteResults> SearchdomainDeleteAsync(string searchdomain)
|
||||
{
|
||||
return await GetUrlAndProcessJson<SearchdomainDeleteResults>(GetUrl($"{baseUri}/Searchdomain", "Delete", apiKey, new Dictionary<string, string>()
|
||||
return await DeleteUrlAndProcessJson<SearchdomainDeleteResults>(GetUrl($"{baseUri}", "Searchdomain", apiKey, new Dictionary<string, string>()
|
||||
{
|
||||
{"searchdomain", searchdomain}
|
||||
}));
|
||||
@@ -57,12 +57,12 @@ public class Client
|
||||
return await SearchdomainCreateAsync(searchdomain);
|
||||
}
|
||||
|
||||
public async Task<SearchdomainCreateResults> SearchdomainCreateAsync(string searchdomain)
|
||||
public async Task<SearchdomainCreateResults> SearchdomainCreateAsync(string searchdomain, SearchdomainSettings searchdomainSettings = new())
|
||||
{
|
||||
return await GetUrlAndProcessJson<SearchdomainCreateResults>(GetUrl($"{baseUri}/Searchdomain", "Create", apiKey, new Dictionary<string, string>()
|
||||
return await PostUrlAndProcessJson<SearchdomainCreateResults>(GetUrl($"{baseUri}", "Searchdomain", apiKey, new Dictionary<string, string>()
|
||||
{
|
||||
{"searchdomain", searchdomain}
|
||||
}));
|
||||
}), new StringContent(JsonSerializer.Serialize(searchdomainSettings), Encoding.UTF8, "application/json"));
|
||||
}
|
||||
|
||||
public async Task<SearchdomainUpdateResults> SearchdomainUpdateAsync(string newName, string settings = "{}")
|
||||
@@ -72,14 +72,18 @@ public class Client
|
||||
return updateResults;
|
||||
}
|
||||
|
||||
public async Task<SearchdomainUpdateResults> SearchdomainUpdateAsync(string searchdomain, string newName, SearchdomainSettings settings = new())
|
||||
{
|
||||
return await SearchdomainUpdateAsync(searchdomain, newName, JsonSerializer.Serialize(settings));
|
||||
}
|
||||
|
||||
public async Task<SearchdomainUpdateResults> SearchdomainUpdateAsync(string searchdomain, string newName, string settings = "{}")
|
||||
{
|
||||
return await GetUrlAndProcessJson<SearchdomainUpdateResults>(GetUrl($"{baseUri}/Searchdomain", "Update", apiKey, new Dictionary<string, string>()
|
||||
return await PutUrlAndProcessJson<SearchdomainUpdateResults>(GetUrl($"{baseUri}", "Searchdomain", apiKey, new Dictionary<string, string>()
|
||||
{
|
||||
{"searchdomain", searchdomain},
|
||||
{"newName", newName},
|
||||
{"settings", settings}
|
||||
}));
|
||||
{"newName", newName}
|
||||
}), new StringContent(settings, Encoding.UTF8, "application/json"));
|
||||
}
|
||||
|
||||
public async Task<EntityQueryResults> EntityQueryAsync(string query)
|
||||
@@ -89,11 +93,11 @@ public class Client
|
||||
|
||||
public async Task<EntityQueryResults> EntityQueryAsync(string searchdomain, string query)
|
||||
{
|
||||
return await GetUrlAndProcessJson<EntityQueryResults>(GetUrl($"{baseUri}/Entity", "Query", apiKey, new Dictionary<string, string>()
|
||||
return await PostUrlAndProcessJson<EntityQueryResults>(GetUrl($"{baseUri}/Searchdomain", "Query", apiKey, new Dictionary<string, string>()
|
||||
{
|
||||
{"searchdomain", searchdomain},
|
||||
{"query", query}
|
||||
}));
|
||||
}), null);
|
||||
}
|
||||
|
||||
public async Task<EntityIndexResult> EntityIndexAsync(List<JSONEntity> jsonEntity)
|
||||
@@ -104,7 +108,7 @@ public class Client
|
||||
public async Task<EntityIndexResult> EntityIndexAsync(string jsonEntity)
|
||||
{
|
||||
var content = new StringContent(jsonEntity, Encoding.UTF8, "application/json");
|
||||
return await PostUrlAndProcessJson<EntityIndexResult>(GetUrl($"{baseUri}/Entity", "Index", apiKey, []), content);//new FormUrlEncodedContent(values));
|
||||
return await PutUrlAndProcessJson<EntityIndexResult>(GetUrl($"{baseUri}", "Entity", apiKey, []), content);
|
||||
}
|
||||
|
||||
public async Task<EntityListResults> EntityListAsync(bool returnEmbeddings = false)
|
||||
@@ -114,7 +118,7 @@ public class Client
|
||||
|
||||
public async Task<EntityListResults> EntityListAsync(string searchdomain, bool returnEmbeddings = false)
|
||||
{
|
||||
var url = $"{baseUri}/Entity/List?apiKey={HttpUtility.UrlEncode(apiKey)}&searchdomain={HttpUtility.UrlEncode(searchdomain)}&returnEmbeddings={HttpUtility.UrlEncode(returnEmbeddings.ToString())}";
|
||||
var url = $"{baseUri}/Entities?apiKey={HttpUtility.UrlEncode(apiKey)}&searchdomain={HttpUtility.UrlEncode(searchdomain)}&returnEmbeddings={HttpUtility.UrlEncode(returnEmbeddings.ToString())}";
|
||||
return await GetUrlAndProcessJson<EntityListResults>(url);
|
||||
}
|
||||
|
||||
@@ -125,8 +129,8 @@ public class Client
|
||||
|
||||
public async Task<EntityDeleteResults> EntityDeleteAsync(string searchdomain, string entityName)
|
||||
{
|
||||
var url = $"{baseUri}/Entity/Delete?apiKey={HttpUtility.UrlEncode(apiKey)}&searchdomain={HttpUtility.UrlEncode(searchdomain)}&entity={HttpUtility.UrlEncode(entityName)}";
|
||||
return await GetUrlAndProcessJson<EntityDeleteResults>(url);
|
||||
var url = $"{baseUri}/Entity?apiKey={HttpUtility.UrlEncode(apiKey)}&searchdomain={HttpUtility.UrlEncode(searchdomain)}&entity={HttpUtility.UrlEncode(entityName)}";
|
||||
return await DeleteUrlAndProcessJson<EntityDeleteResults>(url);
|
||||
}
|
||||
|
||||
private static async Task<T> GetUrlAndProcessJson<T>(string url)
|
||||
@@ -138,7 +142,8 @@ public class Client
|
||||
?? throw new Exception($"Failed to deserialize JSON to type {typeof(T).Name}");
|
||||
return result;
|
||||
}
|
||||
private static async Task<T> PostUrlAndProcessJson<T>(string url, HttpContent content)
|
||||
|
||||
private static async Task<T> PostUrlAndProcessJson<T>(string url, HttpContent? content)
|
||||
{
|
||||
using var client = new HttpClient();
|
||||
var response = await client.PostAsync(url, content);
|
||||
@@ -148,6 +153,26 @@ public class Client
|
||||
return result;
|
||||
}
|
||||
|
||||
private static async Task<T> PutUrlAndProcessJson<T>(string url, HttpContent content)
|
||||
{
|
||||
using var client = new HttpClient();
|
||||
var response = await client.PutAsync(url, content);
|
||||
string responseContent = await response.Content.ReadAsStringAsync();
|
||||
var result = JsonSerializer.Deserialize<T>(responseContent)
|
||||
?? throw new Exception($"Failed to deserialize JSON to type {typeof(T).Name}");
|
||||
return result;
|
||||
}
|
||||
|
||||
private static async Task<T> DeleteUrlAndProcessJson<T>(string url)
|
||||
{
|
||||
using var client = new HttpClient();
|
||||
var response = await client.DeleteAsync(url);
|
||||
string responseContent = await response.Content.ReadAsStringAsync();
|
||||
var result = JsonSerializer.Deserialize<T>(responseContent)
|
||||
?? throw new Exception($"Failed to deserialize JSON to type {typeof(T).Name}");
|
||||
return result;
|
||||
}
|
||||
|
||||
public static string GetUrl(string baseUri, string endpoint, string apiKey, Dictionary<string, string> parameters)
|
||||
{
|
||||
var uriBuilder = new UriBuilder($"{baseUri}/{endpoint}");
|
||||
|
||||
@@ -6,6 +6,7 @@ using Server.Models;
|
||||
|
||||
namespace Server.Controllers;
|
||||
|
||||
[ApiExplorerSettings(IgnoreApi = true)]
|
||||
[Route("[Controller]")]
|
||||
public class AccountController : Controller
|
||||
{
|
||||
|
||||
@@ -24,32 +24,7 @@ public class EntityController : ControllerBase
|
||||
_databaseHelper = databaseHelper;
|
||||
}
|
||||
|
||||
[HttpGet("Query")]
|
||||
public ActionResult<EntityQueryResults> Query(string searchdomain, string query, int? topN)
|
||||
{
|
||||
Searchdomain searchdomain_;
|
||||
try
|
||||
{
|
||||
searchdomain_ = _domainManager.GetSearchdomain(searchdomain);
|
||||
} catch (SearchdomainNotFoundException)
|
||||
{
|
||||
_logger.LogError("Unable to retrieve the searchdomain {searchdomain} - it likely does not exist yet", [searchdomain]);
|
||||
return Ok(new EntityQueryResults() {Results = [], Success = false, Message = "Searchdomain not found" });
|
||||
} catch (Exception ex)
|
||||
{
|
||||
_logger.LogError("Unable to retrieve the searchdomain {searchdomain} - {ex.Message} - {ex.StackTrace}", [searchdomain, ex.Message, ex.StackTrace]);
|
||||
return Ok(new EntityQueryResults() {Results = [], Success = false, Message = "Unable to retrieve the searchdomain - it likely exists, but some other error happened." });
|
||||
}
|
||||
List<(float, string)> results = searchdomain_.Search(query, topN);
|
||||
List<EntityQueryResult> queryResults = [.. results.Select(r => new EntityQueryResult
|
||||
{
|
||||
Name = r.Item2,
|
||||
Value = r.Item1
|
||||
})];
|
||||
return Ok(new EntityQueryResults(){Results = queryResults, Success = true });
|
||||
}
|
||||
|
||||
[HttpPost("Index")]
|
||||
[HttpPut]
|
||||
public ActionResult<EntityIndexResult> Index([FromBody] List<JSONEntity>? jsonEntities)
|
||||
{
|
||||
try
|
||||
@@ -69,7 +44,6 @@ public class EntityController : ControllerBase
|
||||
&& !invalidatedSearchdomains.Contains(jsonEntitySearchdomainName))
|
||||
{
|
||||
invalidatedSearchdomains.Add(jsonEntitySearchdomainName);
|
||||
_domainManager.InvalidateSearchdomainCache(jsonEntitySearchdomainName);
|
||||
}
|
||||
}
|
||||
return Ok(new EntityIndexResult() { Success = true });
|
||||
@@ -88,27 +62,16 @@ public class EntityController : ControllerBase
|
||||
|
||||
}
|
||||
|
||||
[HttpGet("List")]
|
||||
[HttpGet("/Entities")]
|
||||
public ActionResult<EntityListResults> List(string searchdomain, bool returnModels = false, bool returnEmbeddings = false)
|
||||
{
|
||||
if (returnEmbeddings && !returnModels)
|
||||
{
|
||||
_logger.LogError("Invalid request for {searchdomain} - embeddings return requested but without models - not possible!", [searchdomain]);
|
||||
return Ok(new EntityQueryResults() {Results = [], Success = false, Message = "Invalid request" });
|
||||
}
|
||||
Searchdomain searchdomain_;
|
||||
try
|
||||
{
|
||||
searchdomain_ = _domainManager.GetSearchdomain(searchdomain);
|
||||
} catch (SearchdomainNotFoundException)
|
||||
{
|
||||
_logger.LogError("Unable to retrieve the searchdomain {searchdomain} - it likely does not exist yet", [searchdomain]);
|
||||
return Ok(new EntityQueryResults() {Results = [], Success = false, Message = "Searchdomain not found" });
|
||||
} catch (Exception ex)
|
||||
{
|
||||
_logger.LogError("Unable to retrieve the searchdomain {searchdomain} - {ex.Message} - {ex.StackTrace}", [searchdomain, ex.Message, ex.StackTrace]);
|
||||
return Ok(new EntityQueryResults() {Results = [], Success = false, Message = "Unable to retrieve the searchdomain - it likely exists, but some other error happened." });
|
||||
return BadRequest(new EntityListResults() {Results = [], Success = false, Message = "Invalid request" });
|
||||
}
|
||||
(Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomain, _logger);
|
||||
if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new SearchdomainUpdateResults(){Success = false, Message = message});
|
||||
EntityListResults entityListResults = new() {Results = [], Success = true};
|
||||
foreach (Entity entity in searchdomain_.entityCache)
|
||||
{
|
||||
@@ -146,22 +109,11 @@ public class EntityController : ControllerBase
|
||||
return Ok(entityListResults);
|
||||
}
|
||||
|
||||
[HttpGet("Delete")]
|
||||
[HttpDelete]
|
||||
public ActionResult<EntityDeleteResults> Delete(string searchdomain, string entityName)
|
||||
{
|
||||
Searchdomain searchdomain_;
|
||||
try
|
||||
{
|
||||
searchdomain_ = _domainManager.GetSearchdomain(searchdomain);
|
||||
} catch (SearchdomainNotFoundException)
|
||||
{
|
||||
_logger.LogError("Unable to retrieve the searchdomain {searchdomain} - it likely does not exist yet", [searchdomain]);
|
||||
return Ok(new EntityQueryResults() {Results = [], Success = false, Message = "Searchdomain not found" });
|
||||
} catch (Exception ex)
|
||||
{
|
||||
_logger.LogError("Unable to retrieve the searchdomain {searchdomain} - {ex.Message} - {ex.StackTrace}", [searchdomain, ex.Message, ex.StackTrace]);
|
||||
return Ok(new EntityQueryResults() {Results = [], Success = false, Message = "Unable to retrieve the searchdomain - it likely exists, but some other error happened." });
|
||||
}
|
||||
(Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomain, _logger);
|
||||
if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new SearchdomainUpdateResults(){Success = false, Message = message});
|
||||
|
||||
Entity? entity_ = SearchdomainHelper.CacheGetEntity(searchdomain_.entityCache, entityName);
|
||||
if (entity_ is null)
|
||||
@@ -169,6 +121,7 @@ public class EntityController : ControllerBase
|
||||
_logger.LogError("Unable to delete the entity {entityName} in {searchdomain} - it was not found under the specified name", [entityName, searchdomain]);
|
||||
return Ok(new EntityDeleteResults() {Success = false, Message = "Entity not found"});
|
||||
}
|
||||
searchdomain_.ReconciliateOrInvalidateCacheForDeletedEntity(entity_);
|
||||
_databaseHelper.RemoveEntity([], _domainManager.helper, entityName, searchdomain);
|
||||
searchdomain_.entityCache.RemoveAll(entity => entity.name == entityName);
|
||||
return Ok(new EntityDeleteResults() {Success = true});
|
||||
|
||||
@@ -7,7 +7,7 @@ using Server.Exceptions;
|
||||
using Server.Models;
|
||||
namespace Server.Controllers;
|
||||
|
||||
[ApiController]
|
||||
[ApiExplorerSettings(IgnoreApi = true)]
|
||||
[Route("/")]
|
||||
public class HomeController : Controller
|
||||
{
|
||||
|
||||
@@ -23,7 +23,7 @@ public class SearchdomainController : ControllerBase
|
||||
_domainManager = domainManager;
|
||||
}
|
||||
|
||||
[HttpGet("List")]
|
||||
[HttpGet("/Searchdomains")]
|
||||
public ActionResult<SearchdomainListResults> List()
|
||||
{
|
||||
List<string> results;
|
||||
@@ -40,8 +40,8 @@ public class SearchdomainController : ControllerBase
|
||||
return Ok(searchdomainListResults);
|
||||
}
|
||||
|
||||
[HttpGet("Create")]
|
||||
public ActionResult<SearchdomainCreateResults> Create(string searchdomain, string settings = "{}")
|
||||
[HttpPost]
|
||||
public ActionResult<SearchdomainCreateResults> Create(string searchdomain, [FromBody]SearchdomainSettings settings = new())
|
||||
{
|
||||
try
|
||||
{
|
||||
@@ -54,7 +54,7 @@ public class SearchdomainController : ControllerBase
|
||||
}
|
||||
}
|
||||
|
||||
[HttpGet("Delete")]
|
||||
[HttpDelete]
|
||||
public ActionResult<SearchdomainDeleteResults> Delete(string searchdomain)
|
||||
{
|
||||
bool success;
|
||||
@@ -84,12 +84,21 @@ public class SearchdomainController : ControllerBase
|
||||
return Ok(new SearchdomainDeleteResults(){Success = success, DeletedEntities = deletedEntries, Message = message});
|
||||
}
|
||||
|
||||
[HttpGet("Update")]
|
||||
public ActionResult<SearchdomainUpdateResults> Update(string searchdomain, string newName, string settings = "{}")
|
||||
[HttpPut]
|
||||
public ActionResult<SearchdomainUpdateResults> Update(string searchdomain, string newName, [FromBody]string? settings = "{}")
|
||||
{
|
||||
try
|
||||
(Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomain, _logger);
|
||||
if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new SearchdomainUpdateResults(){Success = false, Message = message});
|
||||
if (settings is null)
|
||||
{
|
||||
Dictionary<string, dynamic> parameters = new()
|
||||
{
|
||||
{"name", newName},
|
||||
{"id", searchdomain_.id}
|
||||
};
|
||||
searchdomain_.helper.ExecuteSQLNonQuery("UPDATE searchdomain set name = @name WHERE id = @id", parameters);
|
||||
} else
|
||||
{
|
||||
Searchdomain searchdomain_ = _domainManager.GetSearchdomain(searchdomain);
|
||||
Dictionary<string, dynamic> parameters = new()
|
||||
{
|
||||
{"name", newName},
|
||||
@@ -97,84 +106,55 @@ public class SearchdomainController : ControllerBase
|
||||
{"id", searchdomain_.id}
|
||||
};
|
||||
searchdomain_.helper.ExecuteSQLNonQuery("UPDATE searchdomain set name = @name, settings = @settings WHERE id = @id", parameters);
|
||||
} catch (SearchdomainNotFoundException)
|
||||
{
|
||||
_logger.LogError("Unable to update searchdomain {searchdomain} - not found", [searchdomain]);
|
||||
return Ok(new SearchdomainUpdateResults() { Success = false, Message = $"Unable to update searchdomain {searchdomain} - not found" });
|
||||
} catch (Exception ex)
|
||||
{
|
||||
_logger.LogError("Unable to update searchdomain {searchdomain} - Exception: {ex.Message} - {ex.StackTrace}", [searchdomain, ex.Message, ex.StackTrace]);
|
||||
return Ok(new SearchdomainUpdateResults() { Success = false, Message = $"Unable to update searchdomain {searchdomain}" });
|
||||
}
|
||||
return Ok(new SearchdomainUpdateResults(){Success = true});
|
||||
}
|
||||
|
||||
[HttpPost("UpdateSettings")]
|
||||
[HttpPost("Query")]
|
||||
public ActionResult<EntityQueryResults> Query(string searchdomain, string query, int? topN, bool returnAttributes = false)
|
||||
{
|
||||
(Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomain, _logger);
|
||||
if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new SearchdomainUpdateResults(){Success = false, Message = message});
|
||||
List<(float, string)> results = searchdomain_.Search(query, topN);
|
||||
List<EntityQueryResult> queryResults = [.. results.Select(r => new EntityQueryResult
|
||||
{
|
||||
Name = r.Item2,
|
||||
Value = r.Item1,
|
||||
Attributes = returnAttributes ? (searchdomain_.entityCache.FirstOrDefault(x => x.name == r.Item2)?.attributes ?? null) : null
|
||||
})];
|
||||
return Ok(new EntityQueryResults(){Results = queryResults, Success = true });
|
||||
}
|
||||
|
||||
[HttpPut("Settings")]
|
||||
public ActionResult<SearchdomainUpdateResults> UpdateSettings(string searchdomain, [FromBody] SearchdomainSettings request)
|
||||
{
|
||||
try
|
||||
(Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomain, _logger);
|
||||
if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new SearchdomainUpdateResults(){Success = false, Message = message});
|
||||
Dictionary<string, dynamic> parameters = new()
|
||||
{
|
||||
Searchdomain searchdomain_ = _domainManager.GetSearchdomain(searchdomain);
|
||||
Dictionary<string, dynamic> parameters = new()
|
||||
{
|
||||
{"settings", JsonSerializer.Serialize(request)},
|
||||
{"id", searchdomain_.id}
|
||||
};
|
||||
searchdomain_.helper.ExecuteSQLNonQuery("UPDATE searchdomain set settings = @settings WHERE id = @id", parameters);
|
||||
searchdomain_.settings = request;
|
||||
} catch (SearchdomainNotFoundException)
|
||||
{
|
||||
_logger.LogError("Unable to update settings for searchdomain {searchdomain} - not found", [searchdomain]);
|
||||
return Ok(new SearchdomainUpdateResults() { Success = false, Message = $"Unable to update settings for searchdomain {searchdomain} - not found" });
|
||||
} catch (Exception ex)
|
||||
{
|
||||
_logger.LogError("Unable to update settings for searchdomain {searchdomain} - Exception: {ex.Message} - {ex.StackTrace}", [searchdomain, ex.Message, ex.StackTrace]);
|
||||
return Ok(new SearchdomainUpdateResults() { Success = false, Message = $"Unable to update settings for searchdomain {searchdomain}" });
|
||||
}
|
||||
{"settings", JsonSerializer.Serialize(request)},
|
||||
{"id", searchdomain_.id}
|
||||
};
|
||||
searchdomain_.helper.ExecuteSQLNonQuery("UPDATE searchdomain set settings = @settings WHERE id = @id", parameters);
|
||||
searchdomain_.settings = request;
|
||||
return Ok(new SearchdomainUpdateResults(){Success = true});
|
||||
}
|
||||
|
||||
[HttpGet("GetSearches")]
|
||||
public ActionResult<SearchdomainSearchesResults> GetSearches(string searchdomain)
|
||||
[HttpGet("Queries")]
|
||||
public ActionResult<SearchdomainSearchesResults> GetQueries(string searchdomain)
|
||||
{
|
||||
Searchdomain searchdomain_;
|
||||
try
|
||||
{
|
||||
searchdomain_ = _domainManager.GetSearchdomain(searchdomain);
|
||||
}
|
||||
catch (SearchdomainNotFoundException)
|
||||
{
|
||||
_logger.LogError("Unable to retrieve the searchdomain {searchdomain} - it likely does not exist yet", [searchdomain]);
|
||||
return Ok(new SearchdomainSearchesResults() { Searches = [], Success = false, Message = "Searchdomain not found" });
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError("Unable to retrieve the searchdomain {searchdomain} - {ex.Message} - {ex.StackTrace}", [searchdomain, ex.Message, ex.StackTrace]);
|
||||
return Ok(new SearchdomainSearchesResults() { Searches = [], Success = false, Message = ex.Message });
|
||||
}
|
||||
(Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomain, _logger);
|
||||
if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new SearchdomainUpdateResults(){Success = false, Message = message});
|
||||
Dictionary<string, DateTimedSearchResult> searchCache = searchdomain_.searchCache;
|
||||
|
||||
return Ok(new SearchdomainSearchesResults() { Searches = searchCache, Success = true });
|
||||
}
|
||||
|
||||
[HttpDelete("Searches")]
|
||||
public ActionResult<SearchdomainDeleteSearchResult> DeleteSearch(string searchdomain, string query)
|
||||
[HttpDelete("Query")]
|
||||
public ActionResult<SearchdomainDeleteSearchResult> DeleteQuery(string searchdomain, string query)
|
||||
{
|
||||
Searchdomain searchdomain_;
|
||||
try
|
||||
{
|
||||
searchdomain_ = _domainManager.GetSearchdomain(searchdomain);
|
||||
}
|
||||
catch (SearchdomainNotFoundException)
|
||||
{
|
||||
_logger.LogError("Unable to retrieve the searchdomain {searchdomain} - it likely does not exist yet", [searchdomain]);
|
||||
return Ok(new SearchdomainDeleteSearchResult() { Success = false, Message = "Searchdomain not found" });
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError("Unable to retrieve the searchdomain {searchdomain} - {ex.Message} - {ex.StackTrace}", [searchdomain, ex.Message, ex.StackTrace]);
|
||||
return Ok(new SearchdomainDeleteSearchResult() { Success = false, Message = ex.Message });
|
||||
}
|
||||
(Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomain, _logger);
|
||||
if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new SearchdomainUpdateResults(){Success = false, Message = message});
|
||||
Dictionary<string, DateTimedSearchResult> searchCache = searchdomain_.searchCache;
|
||||
bool containsKey = searchCache.ContainsKey(query);
|
||||
if (containsKey)
|
||||
@@ -185,24 +165,11 @@ public class SearchdomainController : ControllerBase
|
||||
return Ok(new SearchdomainDeleteSearchResult() {Success = false, Message = "Query not found in search cache"});
|
||||
}
|
||||
|
||||
[HttpPatch("Searches")]
|
||||
public ActionResult<SearchdomainUpdateSearchResult> UpdateSearch(string searchdomain, string query, [FromBody]List<ResultItem> results)
|
||||
[HttpPatch("Query")]
|
||||
public ActionResult<SearchdomainUpdateSearchResult> UpdateQuery(string searchdomain, string query, [FromBody]List<ResultItem> results)
|
||||
{
|
||||
Searchdomain searchdomain_;
|
||||
try
|
||||
{
|
||||
searchdomain_ = _domainManager.GetSearchdomain(searchdomain);
|
||||
}
|
||||
catch (SearchdomainNotFoundException)
|
||||
{
|
||||
_logger.LogError("Unable to retrieve the searchdomain {searchdomain} - it likely does not exist yet", [searchdomain]);
|
||||
return Ok(new SearchdomainUpdateSearchResult() { Success = false, Message = "Searchdomain not found" });
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError("Unable to retrieve the searchdomain {searchdomain} - {ex.Message} - {ex.StackTrace}", [searchdomain, ex.Message, ex.StackTrace]);
|
||||
return Ok(new SearchdomainUpdateSearchResult() { Success = false, Message = ex.Message });
|
||||
}
|
||||
(Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomain, _logger);
|
||||
if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new SearchdomainUpdateResults(){Success = false, Message = message});
|
||||
Dictionary<string, DateTimedSearchResult> searchCache = searchdomain_.searchCache;
|
||||
bool containsKey = searchCache.ContainsKey(query);
|
||||
if (containsKey)
|
||||
@@ -215,46 +182,20 @@ public class SearchdomainController : ControllerBase
|
||||
return Ok(new SearchdomainUpdateSearchResult() {Success = false, Message = "Query not found in search cache"});
|
||||
}
|
||||
|
||||
[HttpGet("GetSettings")]
|
||||
[HttpGet("Settings")]
|
||||
public ActionResult<SearchdomainSettingsResults> GetSettings(string searchdomain)
|
||||
{
|
||||
Searchdomain searchdomain_;
|
||||
try
|
||||
{
|
||||
searchdomain_ = _domainManager.GetSearchdomain(searchdomain);
|
||||
}
|
||||
catch (SearchdomainNotFoundException)
|
||||
{
|
||||
_logger.LogError("Unable to retrieve the searchdomain {searchdomain} - it likely does not exist yet", [searchdomain]);
|
||||
return Ok(new SearchdomainSettingsResults() { Settings = null, Success = false, Message = "Searchdomain not found" });
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError("Unable to retrieve the searchdomain {searchdomain} - {ex.Message} - {ex.StackTrace}", [searchdomain, ex.Message, ex.StackTrace]);
|
||||
return Ok(new SearchdomainSettingsResults() { Settings = null, Success = false, Message = ex.Message });
|
||||
}
|
||||
(Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomain, _logger);
|
||||
if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new SearchdomainUpdateResults(){Success = false, Message = message});
|
||||
SearchdomainSettings settings = searchdomain_.settings;
|
||||
return Ok(new SearchdomainSettingsResults() { Settings = settings, Success = true });
|
||||
}
|
||||
|
||||
[HttpGet("GetSearchCacheSize")]
|
||||
[HttpGet("SearchCache/Size")]
|
||||
public ActionResult<SearchdomainSearchCacheSizeResults> GetSearchCacheSize(string searchdomain)
|
||||
{
|
||||
Searchdomain searchdomain_;
|
||||
try
|
||||
{
|
||||
searchdomain_ = _domainManager.GetSearchdomain(searchdomain);
|
||||
}
|
||||
catch (SearchdomainNotFoundException)
|
||||
{
|
||||
_logger.LogError("Unable to retrieve the searchdomain {searchdomain} - it likely does not exist yet", [searchdomain]);
|
||||
return Ok(new SearchdomainSearchCacheSizeResults() { SearchCacheSizeBytes = null, Success = false, Message = "Searchdomain not found" });
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError("Unable to retrieve the searchdomain {searchdomain} - {ex.Message} - {ex.StackTrace}", [searchdomain, ex.Message, ex.StackTrace]);
|
||||
return Ok(new SearchdomainSearchCacheSizeResults() { SearchCacheSizeBytes = null, Success = false, Message = ex.Message });
|
||||
}
|
||||
(Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomain, _logger);
|
||||
if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new SearchdomainUpdateResults(){Success = false, Message = message});
|
||||
Dictionary<string, DateTimedSearchResult> searchCache = searchdomain_.searchCache;
|
||||
long sizeInBytes = 0;
|
||||
foreach (var entry in searchCache)
|
||||
@@ -266,43 +207,20 @@ public class SearchdomainController : ControllerBase
|
||||
return Ok(new SearchdomainSearchCacheSizeResults() { SearchCacheSizeBytes = sizeInBytes, Success = true });
|
||||
}
|
||||
|
||||
[HttpGet("ClearSearchCache")]
|
||||
[HttpPost("SearchCache/Clear")]
|
||||
public ActionResult<SearchdomainInvalidateCacheResults> InvalidateSearchCache(string searchdomain)
|
||||
{
|
||||
try
|
||||
{
|
||||
Searchdomain searchdomain_ = _domainManager.GetSearchdomain(searchdomain);
|
||||
searchdomain_.InvalidateSearchCache();
|
||||
} catch (SearchdomainNotFoundException)
|
||||
{
|
||||
_logger.LogError("Unable to invalidate search cache for searchdomain {searchdomain} - not found", [searchdomain]);
|
||||
return Ok(new SearchdomainInvalidateCacheResults() { Success = false, Message = $"Unable to invalidate search cache for searchdomain {searchdomain} - not found" });
|
||||
} catch (Exception ex)
|
||||
{
|
||||
_logger.LogError("Unable to invalidate search cache for searchdomain {searchdomain} - Exception: {ex.Message} - {ex.StackTrace}", [searchdomain, ex.Message, ex.StackTrace]);
|
||||
return Ok(new SearchdomainInvalidateCacheResults() { Success = false, Message = $"Unable to invalidate search cache for searchdomain {searchdomain}" });
|
||||
}
|
||||
(Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomain, _logger);
|
||||
if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new SearchdomainUpdateResults(){Success = false, Message = message});
|
||||
searchdomain_.InvalidateSearchCache();
|
||||
return Ok(new SearchdomainInvalidateCacheResults(){Success = true});
|
||||
}
|
||||
|
||||
[HttpGet("GetDatabaseSize")]
|
||||
[HttpGet("Database/Size")]
|
||||
public ActionResult<SearchdomainGetDatabaseSizeResult> GetDatabaseSize(string searchdomain)
|
||||
{
|
||||
Searchdomain searchdomain_;
|
||||
try
|
||||
{
|
||||
searchdomain_ = _domainManager.GetSearchdomain(searchdomain);
|
||||
}
|
||||
catch (SearchdomainNotFoundException)
|
||||
{
|
||||
_logger.LogError("Unable to retrieve the searchdomain {searchdomain} - it likely does not exist yet", [searchdomain]);
|
||||
return Ok(new SearchdomainGetDatabaseSizeResult() { SearchdomainDatabaseSizeBytes = null, Success = false, Message = "Searchdomain not found" });
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError("Unable to retrieve the searchdomain {searchdomain} - {ex.Message} - {ex.StackTrace}", [searchdomain, ex.Message, ex.StackTrace]);
|
||||
return Ok(new SearchdomainGetDatabaseSizeResult() { SearchdomainDatabaseSizeBytes = null, Success = false, Message = ex.Message });
|
||||
}
|
||||
(Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomain, _logger);
|
||||
if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new SearchdomainUpdateResults(){Success = false, Message = message});
|
||||
long sizeInBytes = DatabaseHelper.GetSearchdomainDatabaseSize(searchdomain_.helper, searchdomain);
|
||||
return Ok(new SearchdomainGetDatabaseSizeResult() { SearchdomainDatabaseSizeBytes = sizeInBytes, Success = true });
|
||||
}
|
||||
|
||||
@@ -22,7 +22,7 @@ public class ServerController : ControllerBase
|
||||
_aIProvider = aIProvider;
|
||||
}
|
||||
|
||||
[HttpGet("GetModels")]
|
||||
[HttpGet("Models")]
|
||||
public ActionResult<ServerGetModelsResult> GetModels()
|
||||
{
|
||||
try
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
using AdaptiveExpressions;
|
||||
using OllamaSharp;
|
||||
using OllamaSharp.Models;
|
||||
|
||||
@@ -27,74 +28,31 @@ public class Datapoint
|
||||
|
||||
public static Dictionary<string, float[]> GenerateEmbeddings(string content, List<string> models, AIProvider aIProvider)
|
||||
{
|
||||
return GenerateEmbeddings(content, models, aIProvider, []);
|
||||
return GenerateEmbeddings(content, models, aIProvider, new());
|
||||
}
|
||||
|
||||
public static Dictionary<string, float[]> GenerateEmbeddings(List<string> contents, string model, OllamaApiClient ollama, Dictionary<string, Dictionary<string, float[]>> embeddingCache)
|
||||
{
|
||||
Dictionary<string, float[]> retVal = [];
|
||||
|
||||
List<string> remainingContents = new List<string>(contents);
|
||||
for (int i = contents.Count - 1; i >= 0; i--) // Compare against cache and remove accordingly
|
||||
{
|
||||
string content = contents[i];
|
||||
if (embeddingCache.ContainsKey(model) && embeddingCache[model].ContainsKey(content))
|
||||
{
|
||||
retVal[content] = embeddingCache[model][content];
|
||||
remainingContents.RemoveAt(i);
|
||||
}
|
||||
}
|
||||
if (remainingContents.Count == 0)
|
||||
{
|
||||
return retVal;
|
||||
}
|
||||
|
||||
EmbedRequest request = new()
|
||||
{
|
||||
Model = model,
|
||||
Input = remainingContents
|
||||
};
|
||||
|
||||
EmbedResponse response = ollama.EmbedAsync(request).Result;
|
||||
for (int i = 0; i < response.Embeddings.Count; i++)
|
||||
{
|
||||
string content = remainingContents.ElementAt(i);
|
||||
float[] embeddings = response.Embeddings.ElementAt(i);
|
||||
retVal[content] = embeddings;
|
||||
if (!embeddingCache.ContainsKey(model))
|
||||
{
|
||||
embeddingCache[model] = [];
|
||||
}
|
||||
if (!embeddingCache[model].ContainsKey(content))
|
||||
{
|
||||
embeddingCache[model][content] = embeddings;
|
||||
}
|
||||
}
|
||||
|
||||
return retVal;
|
||||
}
|
||||
|
||||
public static Dictionary<string, float[]> GenerateEmbeddings(string content, List<string> models, AIProvider aIProvider, Dictionary<string, Dictionary<string, float[]>> embeddingCache)
|
||||
public static Dictionary<string, float[]> GenerateEmbeddings(string content, List<string> models, AIProvider aIProvider, LRUCache<string, Dictionary<string, float[]>> embeddingCache)
|
||||
{
|
||||
Dictionary<string, float[]> retVal = [];
|
||||
foreach (string model in models)
|
||||
{
|
||||
if (embeddingCache.ContainsKey(model) && embeddingCache[model].ContainsKey(content))
|
||||
bool embeddingCacheHasModel = embeddingCache.TryGet(model, out var embeddingCacheForModel);
|
||||
if (embeddingCacheHasModel && embeddingCacheForModel.ContainsKey(content))
|
||||
{
|
||||
retVal[model] = embeddingCache[model][content];
|
||||
retVal[model] = embeddingCacheForModel[content];
|
||||
continue;
|
||||
}
|
||||
var response = aIProvider.GenerateEmbeddings(model, [content]);
|
||||
if (response is not null)
|
||||
{
|
||||
retVal[model] = response;
|
||||
if (!embeddingCache.ContainsKey(model))
|
||||
if (!embeddingCacheHasModel)
|
||||
{
|
||||
embeddingCache[model] = [];
|
||||
embeddingCacheForModel = [];
|
||||
}
|
||||
if (!embeddingCache[model].ContainsKey(content))
|
||||
if (!embeddingCacheForModel.ContainsKey(content))
|
||||
{
|
||||
embeddingCache[model][content] = response;
|
||||
embeddingCacheForModel[content] = response;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
using Shared.Models;
|
||||
|
||||
namespace Server.Exceptions;
|
||||
|
||||
public class ProbMethodNotFoundException(string probMethod) : Exception($"Unknown probMethod name {probMethod}") { }
|
||||
public class ProbMethodNotFoundException(ProbMethodEnum probMethod) : Exception($"Unknown probMethod name {probMethod}") { }
|
||||
|
||||
public class SimilarityMethodNotFoundException(string similarityMethod) : Exception($"Unknown similarityMethod name \"{similarityMethod}\"") { }
|
||||
public class SimilarityMethodNotFoundException(SimilarityMethodEnum similarityMethod) : Exception($"Unknown similarityMethod name \"{similarityMethod}\"") { }
|
||||
|
||||
public class JSONPathSelectionException(string path, string testedContent) : Exception($"Unable to select tokens using JSONPath {path} for string: {testedContent}.") { }
|
||||
@@ -38,12 +38,12 @@ public class DatabaseHelper(ILogger<DatabaseHelper> logger)
|
||||
return helper.ExecuteSQLCommandGetInsertedID("INSERT INTO searchdomain (name, settings) VALUES (@name, @settings)", parameters);
|
||||
}
|
||||
|
||||
public static int DatabaseInsertEntity(SQLHelper helper, string name, string probmethod, int id_searchdomain)
|
||||
public static int DatabaseInsertEntity(SQLHelper helper, string name, ProbMethodEnum probmethod, int id_searchdomain)
|
||||
{
|
||||
Dictionary<string, dynamic> parameters = new()
|
||||
{
|
||||
{ "name", name },
|
||||
{ "probmethod", probmethod },
|
||||
{ "probmethod", probmethod.ToString() },
|
||||
{ "id_searchdomain", id_searchdomain }
|
||||
};
|
||||
return helper.ExecuteSQLCommandGetInsertedID("INSERT INTO entity (name, probmethod, id_searchdomain) VALUES (@name, @probmethod, @id_searchdomain)", parameters);
|
||||
@@ -60,13 +60,13 @@ public class DatabaseHelper(ILogger<DatabaseHelper> logger)
|
||||
return helper.ExecuteSQLCommandGetInsertedID("INSERT INTO attribute (attribute, value, id_entity) VALUES (@attribute, @value, @id_entity)", parameters);
|
||||
}
|
||||
|
||||
public static int DatabaseInsertDatapoint(SQLHelper helper, string name, string probmethod_embedding, string similarityMethod, string hash, int id_entity)
|
||||
public static int DatabaseInsertDatapoint(SQLHelper helper, string name, ProbMethodEnum probmethod_embedding, SimilarityMethodEnum similarityMethod, string hash, int id_entity)
|
||||
{
|
||||
Dictionary<string, dynamic> parameters = new()
|
||||
{
|
||||
{ "name", name },
|
||||
{ "probmethod_embedding", probmethod_embedding },
|
||||
{ "similaritymethod", similarityMethod },
|
||||
{ "probmethod_embedding", probmethod_embedding.ToString() },
|
||||
{ "similaritymethod", similarityMethod.ToString() },
|
||||
{ "hash", hash },
|
||||
{ "id_entity", id_entity }
|
||||
};
|
||||
|
||||
@@ -2,6 +2,7 @@ using System.Collections.Concurrent;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using AdaptiveExpressions;
|
||||
using Server.Exceptions;
|
||||
using Shared.Models;
|
||||
|
||||
@@ -46,7 +47,7 @@ public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelp
|
||||
|
||||
public List<Entity>? EntitiesFromJSON(SearchdomainManager searchdomainManager, ILogger logger, string json)
|
||||
{
|
||||
Dictionary<string, Dictionary<string, float[]>> embeddingCache = searchdomainManager.embeddingCache;
|
||||
LRUCache<string, Dictionary<string, float[]>> embeddingCache = searchdomainManager.embeddingCache;
|
||||
AIProvider aIProvider = searchdomainManager.aIProvider;
|
||||
SQLHelper helper = searchdomainManager.helper;
|
||||
|
||||
@@ -91,8 +92,9 @@ public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelp
|
||||
Searchdomain searchdomain = searchdomainManager.GetSearchdomain(jsonEntity.Searchdomain);
|
||||
List<Entity> entityCache = searchdomain.entityCache;
|
||||
AIProvider aIProvider = searchdomain.aIProvider;
|
||||
Dictionary<string, Dictionary<string, float[]>> embeddingCache = searchdomain.embeddingCache;
|
||||
LRUCache<string, Dictionary<string, float[]>> embeddingCache = searchdomain.embeddingCache;
|
||||
Entity? preexistingEntity = entityCache.FirstOrDefault(entity => entity.name == jsonEntity.Name);
|
||||
bool invalidateSearchCache = false;
|
||||
|
||||
if (preexistingEntity is not null)
|
||||
{
|
||||
@@ -147,8 +149,9 @@ public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelp
|
||||
}
|
||||
|
||||
// Datapoint
|
||||
foreach (Datapoint datapoint in preexistingEntity.datapoints.ToList())
|
||||
foreach (Datapoint datapoint_ in preexistingEntity.datapoints.ToList())
|
||||
{
|
||||
Datapoint datapoint = datapoint_; // To enable replacing the datapoint reference as foreach iterators cannot be overwritten
|
||||
bool newEntityHasDatapoint = jsonEntity.Datapoints.Any(x => x.Name == datapoint.name);
|
||||
if (!newEntityHasDatapoint)
|
||||
{
|
||||
@@ -161,6 +164,7 @@ public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelp
|
||||
helper.ExecuteSQLNonQuery("DELETE e FROM embedding e JOIN datapoint d ON e.id_datapoint=d.id WHERE d.name=@datapointName AND d.id_entity=@entityId", parameters);
|
||||
helper.ExecuteSQLNonQuery("DELETE FROM datapoint WHERE id_entity=@entityId AND name=@datapointName", parameters);
|
||||
preexistingEntity.datapoints.Remove(datapoint);
|
||||
invalidateSearchCache = true;
|
||||
} else
|
||||
{
|
||||
JSONDatapoint? newEntityDatapoint = jsonEntity.Datapoints.FirstOrDefault(x => x.Name == datapoint.name);
|
||||
@@ -177,22 +181,24 @@ public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelp
|
||||
preexistingEntity.datapoints.Remove(datapoint);
|
||||
Datapoint newDatapoint = DatabaseInsertDatapointWithEmbeddings(helper, searchdomain, newEntityDatapoint, (int)preexistingEntityID);
|
||||
preexistingEntity.datapoints.Add(newDatapoint);
|
||||
|
||||
datapoint = newDatapoint;
|
||||
invalidateSearchCache = true;
|
||||
}
|
||||
if (newEntityDatapoint is not null && (newEntityDatapoint.Probmethod_embedding != datapoint.probMethod.name || newEntityDatapoint.SimilarityMethod != datapoint.similarityMethod.name))
|
||||
if (newEntityDatapoint is not null && (newEntityDatapoint.Probmethod_embedding != datapoint.probMethod.probMethodEnum || newEntityDatapoint.SimilarityMethod != datapoint.similarityMethod.similarityMethodEnum))
|
||||
{
|
||||
// Datapoint - Updated (probmethod or similaritymethod)
|
||||
Dictionary<string, dynamic> parameters = new()
|
||||
{
|
||||
{ "probmethod", newEntityDatapoint.Probmethod_embedding },
|
||||
{ "similaritymethod", newEntityDatapoint.SimilarityMethod },
|
||||
{ "probmethod", newEntityDatapoint.Probmethod_embedding.ToString() },
|
||||
{ "similaritymethod", newEntityDatapoint.SimilarityMethod.ToString() },
|
||||
{ "datapointName", datapoint.name },
|
||||
{ "entityId", preexistingEntityID}
|
||||
};
|
||||
helper.ExecuteSQLNonQuery("UPDATE datapoint SET probmethod_embedding=@probmethod, similaritymethod=@similaritymethod WHERE id_entity=@entityId AND name=@datapointName", parameters);
|
||||
Datapoint preexistingDatapoint = preexistingEntity.datapoints.First(x => x == datapoint); // The for loop is a copy. This retrieves the original such that it can be updated.
|
||||
preexistingDatapoint.probMethod = datapoint.probMethod;
|
||||
preexistingDatapoint.similarityMethod = datapoint.similarityMethod;
|
||||
preexistingDatapoint.probMethod = new(newEntityDatapoint.Probmethod_embedding, _logger);
|
||||
preexistingDatapoint.similarityMethod = new(newEntityDatapoint.SimilarityMethod, _logger);
|
||||
invalidateSearchCache = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -204,10 +210,14 @@ public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelp
|
||||
// Datapoint - New
|
||||
Datapoint datapoint = DatabaseInsertDatapointWithEmbeddings(helper, searchdomain, jsonDatapoint, (int)preexistingEntityID);
|
||||
preexistingEntity.datapoints.Add(datapoint);
|
||||
invalidateSearchCache = true;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if (invalidateSearchCache)
|
||||
{
|
||||
searchdomain.ReconciliateOrInvalidateCacheForNewOrUpdatedEntity(preexistingEntity);
|
||||
}
|
||||
return preexistingEntity;
|
||||
}
|
||||
else
|
||||
@@ -227,11 +237,12 @@ public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelp
|
||||
}
|
||||
|
||||
var probMethod = Probmethods.GetMethod(jsonEntity.Probmethod) ?? throw new ProbMethodNotFoundException(jsonEntity.Probmethod);
|
||||
Entity entity = new(jsonEntity.Attributes, probMethod, jsonEntity.Probmethod, datapoints, jsonEntity.Name)
|
||||
Entity entity = new(jsonEntity.Attributes, probMethod, jsonEntity.Probmethod.ToString(), datapoints, jsonEntity.Name)
|
||||
{
|
||||
id = id_entity
|
||||
};
|
||||
entityCache.Add(entity);
|
||||
searchdomain.ReconciliateOrInvalidateCacheForNewOrUpdatedEntity(entity);
|
||||
return entity;
|
||||
}
|
||||
}
|
||||
@@ -261,7 +272,7 @@ public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelp
|
||||
throw new Exception("jsonDatapoint.Text must not be null at this point");
|
||||
}
|
||||
using SQLHelper helper = searchdomain.helper.DuplicateConnection();
|
||||
Dictionary<string, Dictionary<string, float[]>> embeddingCache = searchdomain.embeddingCache;
|
||||
LRUCache<string, Dictionary<string, float[]>> embeddingCache = searchdomain.embeddingCache;
|
||||
hash ??= Convert.ToBase64String(SHA256.HashData(Encoding.UTF8.GetBytes(jsonDatapoint.Text)));
|
||||
DatabaseHelper.DatabaseInsertDatapoint(helper, jsonDatapoint.Name, jsonDatapoint.Probmethod_embedding, jsonDatapoint.SimilarityMethod, hash, entityId);
|
||||
Dictionary<string, float[]> embeddings = Datapoint.GenerateEmbeddings(jsonDatapoint.Text, [.. jsonDatapoint.Model], searchdomain.aIProvider, embeddingCache);
|
||||
@@ -269,4 +280,21 @@ public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelp
|
||||
var similarityMethod = new SimilarityMethod(jsonDatapoint.SimilarityMethod, logger) ?? throw new SimilarityMethodNotFoundException(jsonDatapoint.SimilarityMethod);
|
||||
return new Datapoint(jsonDatapoint.Name, probMethod_embedding, similarityMethod, hash, [.. embeddings.Select(kv => (kv.Key, kv.Value))]);
|
||||
}
|
||||
|
||||
public static (Searchdomain?, int?, string?) TryGetSearchdomain(SearchdomainManager searchdomainManager, string searchdomain, ILogger logger)
|
||||
{
|
||||
try
|
||||
{
|
||||
Searchdomain searchdomain_ = searchdomainManager.GetSearchdomain(searchdomain);
|
||||
return (searchdomain_, null, null);
|
||||
} catch (SearchdomainNotFoundException)
|
||||
{
|
||||
logger.LogError("Unable to update searchdomain {searchdomain} - not found", [searchdomain]);
|
||||
return (null, 500, $"Unable to update searchdomain {searchdomain} - not found");
|
||||
} catch (Exception ex)
|
||||
{
|
||||
logger.LogError("Unable to update searchdomain {searchdomain} - Exception: {ex.Message} - {ex.StackTrace}", [searchdomain, ex.Message, ex.StackTrace]);
|
||||
return (null, 404, $"Unable to update searchdomain {searchdomain}");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,37 +1,29 @@
|
||||
using System.Text.Json;
|
||||
using Server.Exceptions;
|
||||
using Shared.Models;
|
||||
|
||||
namespace Server;
|
||||
|
||||
public class ProbMethod
|
||||
{
|
||||
public Probmethods.probMethodDelegate method;
|
||||
public ProbMethodEnum probMethodEnum;
|
||||
public string name;
|
||||
|
||||
public ProbMethod(string name, ILogger logger)
|
||||
public ProbMethod(ProbMethodEnum probMethodEnum, ILogger logger)
|
||||
{
|
||||
this.name = name;
|
||||
this.probMethodEnum = probMethodEnum;
|
||||
this.name = probMethodEnum.ToString();
|
||||
Probmethods.probMethodDelegate? probMethod = Probmethods.GetMethod(name);
|
||||
if (probMethod is null)
|
||||
{
|
||||
logger.LogError("Unable to retrieve probMethod {name}", [name]);
|
||||
throw new ProbMethodNotFoundException(name);
|
||||
throw new ProbMethodNotFoundException(probMethodEnum);
|
||||
}
|
||||
method = probMethod;
|
||||
}
|
||||
}
|
||||
|
||||
public enum ProbMethodEnum
|
||||
{
|
||||
Mean,
|
||||
HarmonicMean,
|
||||
QuadraticMean,
|
||||
GeometricMean,
|
||||
EVEWAvg,
|
||||
HVEWAvg,
|
||||
LVEWAvg,
|
||||
DictionaryWeightedAverage
|
||||
}
|
||||
|
||||
public static class Probmethods
|
||||
{
|
||||
@@ -54,6 +46,11 @@ public static class Probmethods
|
||||
};
|
||||
}
|
||||
|
||||
public static probMethodDelegate? GetMethod(ProbMethodEnum probMethodEnum)
|
||||
{
|
||||
return GetMethod(probMethodEnum.ToString());
|
||||
}
|
||||
|
||||
public static probMethodDelegate? GetMethod(string name)
|
||||
{
|
||||
string methodName = name;
|
||||
|
||||
@@ -8,12 +8,19 @@ using Server.HealthChecks;
|
||||
using Server.Helper;
|
||||
using Server.Models;
|
||||
using Server.Services;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
var builder = WebApplication.CreateBuilder(args);
|
||||
|
||||
// Add services to the container.
|
||||
|
||||
builder.Services.AddControllersWithViews();
|
||||
builder.Services.AddControllersWithViews()
|
||||
.AddJsonOptions(options =>
|
||||
{
|
||||
options.JsonSerializerOptions.Converters.Add(
|
||||
new JsonStringEnumConverter()
|
||||
);
|
||||
});
|
||||
|
||||
// Add Localization
|
||||
builder.Services.AddLocalization(options => options.ResourcesPath = "Resources");
|
||||
|
||||
@@ -24,4 +24,223 @@
|
||||
<data name="IrreversibleActionWarning" xml:space="preserve">
|
||||
<value>Diese Aktion kann nicht rückgängig gemacht werden.</value>
|
||||
</data>
|
||||
<data name="Searchdomain selection" xml:space="preserve">
|
||||
<value>Searchdomain Auswahl</value>
|
||||
</data>
|
||||
<data name="Create" xml:space="preserve">
|
||||
<value>Erstellen</value>
|
||||
</data>
|
||||
<data name="Searchdomain information and settings" xml:space="preserve">
|
||||
<value>Searchdomain Informationen und Einstellungen</value>
|
||||
</data>
|
||||
<data name="Actions" xml:space="preserve">
|
||||
<value>Aktionen</value>
|
||||
</data>
|
||||
<data name="Rename" xml:space="preserve">
|
||||
<value>Umbenennen</value>
|
||||
</data>
|
||||
<data name="Delete" xml:space="preserve">
|
||||
<value>Löschen</value>
|
||||
</data>
|
||||
<data name="Settings" xml:space="preserve">
|
||||
<value>Einstellungen</value>
|
||||
</data>
|
||||
<data name="Cache reconciliation" xml:space="preserve">
|
||||
<value>Cache Abgleich</value>
|
||||
</data>
|
||||
<data name="Update" xml:space="preserve">
|
||||
<value>Anpassen</value>
|
||||
</data>
|
||||
<data name="Search cache" xml:space="preserve">
|
||||
<value>Such-Cache</value>
|
||||
</data>
|
||||
<data name="Search cache utilization" xml:space="preserve">
|
||||
<value>Such-Cache Speicherauslastung</value>
|
||||
</data>
|
||||
<data name="Clear" xml:space="preserve">
|
||||
<value>Leeren</value>
|
||||
</data>
|
||||
<data name="Database size" xml:space="preserve">
|
||||
<value>Größe in der Datenbank</value>
|
||||
</data>
|
||||
<data name="Add new entity" xml:space="preserve">
|
||||
<value>Neue Entity erstellen</value>
|
||||
</data>
|
||||
<data name="Entity Details" xml:space="preserve">
|
||||
<value>Entity Details</value>
|
||||
</data>
|
||||
<data name="Attributes" xml:space="preserve">
|
||||
<value>Attribute</value>
|
||||
</data>
|
||||
<data name="Key" xml:space="preserve">
|
||||
<value>Schlüssel</value>
|
||||
</data>
|
||||
<data name="Value" xml:space="preserve">
|
||||
<value>Wert</value>
|
||||
</data>
|
||||
<data name="Datapoints" xml:space="preserve">
|
||||
<value>Datapoints</value>
|
||||
</data>
|
||||
<data name="Name" xml:space="preserve">
|
||||
<value>Name</value>
|
||||
</data>
|
||||
<data name="ProbMethod" xml:space="preserve">
|
||||
<value>ProbMethod</value>
|
||||
</data>
|
||||
<data name="SimilarityMethod" xml:space="preserve">
|
||||
<value>SimilarityMethod</value>
|
||||
</data>
|
||||
<data name="Close" xml:space="preserve">
|
||||
<value>Schließen</value>
|
||||
</data>
|
||||
<data name="Query Details" xml:space="preserve">
|
||||
<value>Suchanfrage Details</value>
|
||||
</data>
|
||||
<data name="Access times" xml:space="preserve">
|
||||
<value>Zugriffszeiten</value>
|
||||
</data>
|
||||
<data name="Results" xml:space="preserve">
|
||||
<value>Ergebnisse</value>
|
||||
</data>
|
||||
<data name="Score" xml:space="preserve">
|
||||
<value>Bewertung</value>
|
||||
</data>
|
||||
<data name="Query Update" xml:space="preserve">
|
||||
<value>Suchanfrage anpassen</value>
|
||||
</data>
|
||||
<data name="Rename searchdomain" xml:space="preserve">
|
||||
<value>Searchdomain umbenennen</value>
|
||||
</data>
|
||||
<data name="Delete searchdomain" xml:space="preserve">
|
||||
<value>Searchdomain löschen</value>
|
||||
</data>
|
||||
<data name="Create searchdomain" xml:space="preserve">
|
||||
<value>Searchdomain anlegen</value>
|
||||
</data>
|
||||
<data name="Searchdomain name" xml:space="preserve">
|
||||
<value>Searchdomain Name</value>
|
||||
</data>
|
||||
<data name="Enable cache reconciliation" xml:space="preserve">
|
||||
<value>Cache Abgleich verwenden</value>
|
||||
</data>
|
||||
<data name="Create entity" xml:space="preserve">
|
||||
<value>Entity erstellen</value>
|
||||
</data>
|
||||
<data name="Entity name" xml:space="preserve">
|
||||
<value>Entity Name</value>
|
||||
</data>
|
||||
<data name="Probmethod" xml:space="preserve">
|
||||
<value>Probmethod</value>
|
||||
</data>
|
||||
<data name="Add attribute" xml:space="preserve">
|
||||
<value>Attribut hinzufügen</value>
|
||||
</data>
|
||||
<data name="Probmethod_embedding" xml:space="preserve">
|
||||
<value>Probmethod_embedding</value>
|
||||
</data>
|
||||
<data name="Similarity method" xml:space="preserve">
|
||||
<value>Similarity method</value>
|
||||
</data>
|
||||
<data name="Model" xml:space="preserve">
|
||||
<value>Modell</value>
|
||||
</data>
|
||||
<data name="Add datapoint" xml:space="preserve">
|
||||
<value>Datapoint hinzufügen</value>
|
||||
</data>
|
||||
<data name="Delete entity" xml:space="preserve">
|
||||
<value>Entity löschen</value>
|
||||
</data>
|
||||
<data name="Update entity" xml:space="preserve">
|
||||
<value>Entity anpassen</value>
|
||||
</data>
|
||||
<data name="Action" xml:space="preserve">
|
||||
<value>Aktion</value>
|
||||
</data>
|
||||
<data name="Delete query" xml:space="preserve">
|
||||
<value>Suchanfrage löschen</value>
|
||||
</data>
|
||||
<data name="Creating entity" xml:space="preserve">
|
||||
<value>Erstelle Entity</value>
|
||||
</data>
|
||||
<data name="Entity was created successfully" xml:space="preserve">
|
||||
<value>Entity wurde erfolgreich erstellt</value>
|
||||
</data>
|
||||
<data name="Failed to create entity" xml:space="preserve">
|
||||
<value>Entity konnte nicht erstellt werden</value>
|
||||
</data>
|
||||
<data name="Searchdomain was created successfully" xml:space="preserve">
|
||||
<value>Searchdomain wurde erfolgreich erstellt</value>
|
||||
</data>
|
||||
<data name="Failed to create searchdomain" xml:space="preserve">
|
||||
<value>Searchdomain konnte nicht erstellt werden</value>
|
||||
</data>
|
||||
<data name="Searchdomain cache was cleared successfully" xml:space="preserve">
|
||||
<value>Searchdomain Cache wurde erfolgreich geleert</value>
|
||||
</data>
|
||||
<data name="Failed to clear searchdomain cache" xml:space="preserve">
|
||||
<value>Searchdomain Cache konnte nicht geleert werden</value>
|
||||
</data>
|
||||
<data name="Entity was deleted successfully" xml:space="preserve">
|
||||
<value>Entity wurde erfolgreich gelöscht</value>
|
||||
</data>
|
||||
<data name="Failed to delete entity" xml:space="preserve">
|
||||
<value>Entity konnte nicht gelöscht werden</value>
|
||||
</data>
|
||||
<data name="Updating entity" xml:space="preserve">
|
||||
<value>Entity wird angepasst</value>
|
||||
</data>
|
||||
<data name="Entity was updated successfully" xml:space="preserve">
|
||||
<value>Entity wurde erfolgreich angepasst</value>
|
||||
</data>
|
||||
<data name="Failed to update entity" xml:space="preserve">
|
||||
<value>Entity konnte nicht angepasst werden</value>
|
||||
</data>
|
||||
<data name="Search query was deleted successfully" xml:space="preserve">
|
||||
<value>Suchanfrage wurde erfolgreich gelöscht</value>
|
||||
</data>
|
||||
<data name="Failed to delete search query" xml:space="preserve">
|
||||
<value>Suchanfrage konnte nicht gelöscht werden</value>
|
||||
</data>
|
||||
<data name="Searchdomain was created successfully" xml:space="preserve">
|
||||
<value>Searchdomain wurde erfolgreich erstellt</value>
|
||||
</data>
|
||||
<data name="Updating search query failed" xml:space="preserve">
|
||||
<value>Suchanfrage konnte nicht angepasst werden</value>
|
||||
</data>
|
||||
<data name="Searchdomain was deleted successfully" xml:space="preserve">
|
||||
<value>Searchdomain wurde erfolgreich gelöscht</value>
|
||||
</data>
|
||||
<data name="Failed to delete searchdomain" xml:space="preserve">
|
||||
<value>Konnte Searchdomain nicht löschen</value>
|
||||
</data>
|
||||
<data name="Searchdomain was renamed successfully" xml:space="preserve">
|
||||
<value>Searchdomain wurde erfolgreich umbenannt</value>
|
||||
</data>
|
||||
<data name="Failed to rename searchdomain" xml:space="preserve">
|
||||
<value>Searchdomain konnte nicht umbenannt werden</value>
|
||||
</data>
|
||||
<data name="Searchdomain settings were updated successfully" xml:space="preserve">
|
||||
<value>Searchdomain Einstellungen wurden erfolgreich angepasst</value>
|
||||
</data>
|
||||
<data name="Updating searchdomain settings failed" xml:space="preserve">
|
||||
<value>Searchdomain Einstellungen konnten nicht angepasst werden</value>
|
||||
</data>
|
||||
<data name="Unable to fetch searchdomain config" xml:space="preserve">
|
||||
<value>Searchdomain Einstellungen konnten nicht abgerufen werden</value>
|
||||
</data>
|
||||
<data name="Unable to fetch searchdomain cache utilization" xml:space="preserve">
|
||||
<value>Searchdomain Cache-Auslastung konnte nicht abgerufen werden</value>
|
||||
</data>
|
||||
<data name="Details" xml:space="preserve">
|
||||
<value>Details</value>
|
||||
</data>
|
||||
<data name="Remove attribute" xml:space="preserve">
|
||||
<value>Attribut entfernen</value>
|
||||
</data>
|
||||
<data name="Remove" xml:space="preserve">
|
||||
<value>Entfernen</value>
|
||||
</data>
|
||||
<data name="Close alert" xml:space="preserve">
|
||||
<value>Benachrichtigung schließen</value>
|
||||
</data>
|
||||
</root>
|
||||
@@ -24,4 +24,223 @@
|
||||
<data name="IrreversibleActionWarning" xml:space="preserve">
|
||||
<value>This action cannot be undone.</value>
|
||||
</data>
|
||||
<data name="Searchdomain selection" xml:space="preserve">
|
||||
<value>Searchdomain selection</value>
|
||||
</data>
|
||||
<data name="Create" xml:space="preserve">
|
||||
<value>Create</value>
|
||||
</data>
|
||||
<data name="Searchdomain information and settings" xml:space="preserve">
|
||||
<value>Searchdomain information and settings</value>
|
||||
</data>
|
||||
<data name="Actions" xml:space="preserve">
|
||||
<value>Actions</value>
|
||||
</data>
|
||||
<data name="Rename" xml:space="preserve">
|
||||
<value>Rename</value>
|
||||
</data>
|
||||
<data name="Delete" xml:space="preserve">
|
||||
<value>Delete</value>
|
||||
</data>
|
||||
<data name="Settings" xml:space="preserve">
|
||||
<value>Settings</value>
|
||||
</data>
|
||||
<data name="Cache reconciliation" xml:space="preserve">
|
||||
<value>Cache reconciliation</value>
|
||||
</data>
|
||||
<data name="Update" xml:space="preserve">
|
||||
<value>Update</value>
|
||||
</data>
|
||||
<data name="Search cache" xml:space="preserve">
|
||||
<value>Search cache</value>
|
||||
</data>
|
||||
<data name="Search cache utilization" xml:space="preserve">
|
||||
<value>Search cache utilization</value>
|
||||
</data>
|
||||
<data name="Clear" xml:space="preserve">
|
||||
<value>Clear</value>
|
||||
</data>
|
||||
<data name="Database size" xml:space="preserve">
|
||||
<value>Database size</value>
|
||||
</data>
|
||||
<data name="Add new entity" xml:space="preserve">
|
||||
<value>Add new entity</value>
|
||||
</data>
|
||||
<data name="Entity Details" xml:space="preserve">
|
||||
<value>Entity Details</value>
|
||||
</data>
|
||||
<data name="Attributes" xml:space="preserve">
|
||||
<value>Attributes</value>
|
||||
</data>
|
||||
<data name="Key" xml:space="preserve">
|
||||
<value>Key</value>
|
||||
</data>
|
||||
<data name="Value" xml:space="preserve">
|
||||
<value>Value</value>
|
||||
</data>
|
||||
<data name="Datapoints" xml:space="preserve">
|
||||
<value>Datapoints</value>
|
||||
</data>
|
||||
<data name="Name" xml:space="preserve">
|
||||
<value>Name</value>
|
||||
</data>
|
||||
<data name="ProbMethod" xml:space="preserve">
|
||||
<value>ProbMethod</value>
|
||||
</data>
|
||||
<data name="SimilarityMethod" xml:space="preserve">
|
||||
<value>SimilarityMethod</value>
|
||||
</data>
|
||||
<data name="Close" xml:space="preserve">
|
||||
<value>Close</value>
|
||||
</data>
|
||||
<data name="Query Details" xml:space="preserve">
|
||||
<value>Query Details</value>
|
||||
</data>
|
||||
<data name="Access times" xml:space="preserve">
|
||||
<value>Access times</value>
|
||||
</data>
|
||||
<data name="Results" xml:space="preserve">
|
||||
<value>Results</value>
|
||||
</data>
|
||||
<data name="Score" xml:space="preserve">
|
||||
<value>Score</value>
|
||||
</data>
|
||||
<data name="Query Update" xml:space="preserve">
|
||||
<value>Query Update</value>
|
||||
</data>
|
||||
<data name="Rename searchdomain" xml:space="preserve">
|
||||
<value>Rename searchdomain</value>
|
||||
</data>
|
||||
<data name="Delete searchdomain" xml:space="preserve">
|
||||
<value>Delete searchdomain</value>
|
||||
</data>
|
||||
<data name="Create searchdomain" xml:space="preserve">
|
||||
<value>Create searchdomain</value>
|
||||
</data>
|
||||
<data name="Searchdomain name" xml:space="preserve">
|
||||
<value>Searchdomain name</value>
|
||||
</data>
|
||||
<data name="Enable cache reconciliation" xml:space="preserve">
|
||||
<value>Enable cache reconciliation</value>
|
||||
</data>
|
||||
<data name="Create entity" xml:space="preserve">
|
||||
<value>Create entity</value>
|
||||
</data>
|
||||
<data name="Entity name" xml:space="preserve">
|
||||
<value>Entity name</value>
|
||||
</data>
|
||||
<data name="Probmethod" xml:space="preserve">
|
||||
<value>Probmethod</value>
|
||||
</data>
|
||||
<data name="Add attribute" xml:space="preserve">
|
||||
<value>Add attribute</value>
|
||||
</data>
|
||||
<data name="Probmethod_embedding" xml:space="preserve">
|
||||
<value>Probmethod_embedding</value>
|
||||
</data>
|
||||
<data name="Similarity method" xml:space="preserve">
|
||||
<value>Similarity method</value>
|
||||
</data>
|
||||
<data name="Model" xml:space="preserve">
|
||||
<value>Model</value>
|
||||
</data>
|
||||
<data name="Add datapoint" xml:space="preserve">
|
||||
<value>Add datapoint</value>
|
||||
</data>
|
||||
<data name="Delete entity" xml:space="preserve">
|
||||
<value>Delete entity</value>
|
||||
</data>
|
||||
<data name="Update entity" xml:space="preserve">
|
||||
<value>Update entity</value>
|
||||
</data>
|
||||
<data name="Action" xml:space="preserve">
|
||||
<value>Action</value>
|
||||
</data>
|
||||
<data name="Delete query" xml:space="preserve">
|
||||
<value>Delete query</value>
|
||||
</data>
|
||||
<data name="Creating entity" xml:space="preserve">
|
||||
<value>Creating entity</value>
|
||||
</data>
|
||||
<data name="Entity was created successfully" xml:space="preserve">
|
||||
<value>Entity was created successfully</value>
|
||||
</data>
|
||||
<data name="Failed to create entity" xml:space="preserve">
|
||||
<value>Failed to create entity</value>
|
||||
</data>
|
||||
<data name="Searchdomain was created successfully" xml:space="preserve">
|
||||
<value>Searchdomain was created successfully</value>
|
||||
</data>
|
||||
<data name="Failed to create searchdomain" xml:space="preserve">
|
||||
<value>Failed to create searchdomain</value>
|
||||
</data>
|
||||
<data name="Searchdomain cache was cleared successfully" xml:space="preserve">
|
||||
<value>Searchdomain cache was cleared successfully</value>
|
||||
</data>
|
||||
<data name="Failed to clear searchdomain cache" xml:space="preserve">
|
||||
<value>Failed to clear searchdomain cache</value>
|
||||
</data>
|
||||
<data name="Entity was deleted successfully" xml:space="preserve">
|
||||
<value>Entity was deleted successfully</value>
|
||||
</data>
|
||||
<data name="Failed to delete entity" xml:space="preserve">
|
||||
<value>Failed to delete entity</value>
|
||||
</data>
|
||||
<data name="Updating entity" xml:space="preserve">
|
||||
<value>Updating entity</value>
|
||||
</data>
|
||||
<data name="Entity was updated successfully" xml:space="preserve">
|
||||
<value>Entity was updated successfully</value>
|
||||
</data>
|
||||
<data name="Failed to update entity" xml:space="preserve">
|
||||
<value>Failed to update entity</value>
|
||||
</data>
|
||||
<data name="Search query was deleted successfully" xml:space="preserve">
|
||||
<value>Search query was deleted successfully</value>
|
||||
</data>
|
||||
<data name="Failed to delete search query" xml:space="preserve">
|
||||
<value>Failed to delete search query</value>
|
||||
</data>
|
||||
<data name="Searchdomain was created successfully" xml:space="preserve">
|
||||
<value>Searchdomain was created successfully</value>
|
||||
</data>
|
||||
<data name="Updating search query failed" xml:space="preserve">
|
||||
<value>Updating search query failed</value>
|
||||
</data>
|
||||
<data name="Searchdomain was deleted successfully" xml:space="preserve">
|
||||
<value>Searchdomain was deleted successfully</value>
|
||||
</data>
|
||||
<data name="Failed to delete searchdomain" xml:space="preserve">
|
||||
<value>Failed to delete searchdomain</value>
|
||||
</data>
|
||||
<data name="Searchdomain was renamed successfully" xml:space="preserve">
|
||||
<value>Searchdomain was renamed successfully</value>
|
||||
</data>
|
||||
<data name="Failed to rename searchdomain" xml:space="preserve">
|
||||
<value>Failed to rename searchdomain</value>
|
||||
</data>
|
||||
<data name="Searchdomain settings were updated successfully" xml:space="preserve">
|
||||
<value>Searchdomain settings were updated successfully</value>
|
||||
</data>
|
||||
<data name="Updating searchdomain settings failed" xml:space="preserve">
|
||||
<value>Updating searchdomain settings failed</value>
|
||||
</data>
|
||||
<data name="Unable to fetch searchdomain config" xml:space="preserve">
|
||||
<value>Unable to fetch searchdomain config</value>
|
||||
</data>
|
||||
<data name="Unable to fetch searchdomain cache utilization" xml:space="preserve">
|
||||
<value>"Unable to fetch searchdomain cache utilization</value>
|
||||
</data>
|
||||
<data name="Details" xml:space="preserve">
|
||||
<value>Details</value>
|
||||
</data>
|
||||
<data name="Remove attribute" xml:space="preserve">
|
||||
<value>Remove attribute</value>
|
||||
</data>
|
||||
<data name="Remove" xml:space="preserve">
|
||||
<value>Remove</value>
|
||||
</data>
|
||||
<data name="Close alert" xml:space="preserve">
|
||||
<value>Close alert</value>
|
||||
</data>
|
||||
</root>
|
||||
@@ -5,6 +5,7 @@ using ElmahCore.Mvc.Logger;
|
||||
using MySql.Data.MySqlClient;
|
||||
using Server.Helper;
|
||||
using Shared.Models;
|
||||
using AdaptiveExpressions;
|
||||
|
||||
namespace Server;
|
||||
|
||||
@@ -19,13 +20,12 @@ public class Searchdomain
|
||||
public Dictionary<string, DateTimedSearchResult> searchCache; // Key: query, Value: Search results for that query (with timestamp)
|
||||
public List<Entity> entityCache;
|
||||
public List<string> modelsInUse;
|
||||
public Dictionary<string, Dictionary<string, float[]>> embeddingCache;
|
||||
public int embeddingCacheMaxSize = 10000000;
|
||||
public LRUCache<string, Dictionary<string, float[]>> embeddingCache;
|
||||
private readonly MySqlConnection connection;
|
||||
public SQLHelper helper;
|
||||
private readonly ILogger _logger;
|
||||
|
||||
public Searchdomain(string searchdomain, string connectionString, AIProvider aIProvider, Dictionary<string, Dictionary<string, float[]>> embeddingCache, ILogger logger, string provider = "sqlserver", bool runEmpty = false)
|
||||
public Searchdomain(string searchdomain, string connectionString, AIProvider aIProvider, LRUCache<string, Dictionary<string, float[]>> embeddingCache, ILogger logger, string provider = "sqlserver", bool runEmpty = false)
|
||||
{
|
||||
_connectionString = connectionString;
|
||||
_provider = provider.ToLower();
|
||||
@@ -96,8 +96,16 @@ public class Searchdomain
|
||||
string probmethodString = datapointReader.GetString(3);
|
||||
string similarityMethodString = datapointReader.GetString(4);
|
||||
string hash = datapointReader.GetString(5);
|
||||
ProbMethod probmethod = new(probmethodString, _logger);
|
||||
SimilarityMethod similarityMethod = new(similarityMethodString, _logger);
|
||||
ProbMethodEnum probmethodEnum = (ProbMethodEnum)Enum.Parse(
|
||||
typeof(ProbMethodEnum),
|
||||
probmethodString
|
||||
);
|
||||
SimilarityMethodEnum similairtyMethodEnum = (SimilarityMethodEnum)Enum.Parse(
|
||||
typeof(SimilarityMethodEnum),
|
||||
similarityMethodString
|
||||
);
|
||||
ProbMethod probmethod = new(probmethodEnum, _logger);
|
||||
SimilarityMethod similarityMethod = new(similairtyMethodEnum, _logger);
|
||||
if (embedding_unassigned.TryGetValue(id, out Dictionary<string, float[]>? embeddings) && probmethod is not null)
|
||||
{
|
||||
embedding_unassigned.Remove(id);
|
||||
@@ -151,7 +159,6 @@ public class Searchdomain
|
||||
}
|
||||
entityReader.Close();
|
||||
modelsInUse = GetModels(entityCache);
|
||||
embeddingCache = []; // TODO remove this and implement proper remediation to improve performance
|
||||
}
|
||||
|
||||
public List<(float, string)> Search(string query, int? topN = null)
|
||||
@@ -162,33 +169,13 @@ public class Searchdomain
|
||||
return [.. cachedResult.Results.Select(r => (r.Score, r.Name))];
|
||||
}
|
||||
|
||||
if (!embeddingCache.TryGetValue(query, out Dictionary<string, float[]>? queryEmbeddings))
|
||||
{
|
||||
queryEmbeddings = Datapoint.GenerateEmbeddings(query, modelsInUse, aIProvider);
|
||||
if (embeddingCache.Count < embeddingCacheMaxSize) // TODO add better way of managing cache limit hits
|
||||
{ // Idea: Add access count to each entry. On limit hit, sort the entries by access count and remove the bottom 10% of entries
|
||||
embeddingCache.Add(query, queryEmbeddings);
|
||||
}
|
||||
} // TODO implement proper cache remediation for embeddingCache here
|
||||
Dictionary<string, float[]> queryEmbeddings = GetQueryEmbeddings(query);
|
||||
|
||||
List<(float, string)> result = [];
|
||||
|
||||
foreach (Entity entity in entityCache)
|
||||
{
|
||||
List<(string, float)> datapointProbs = [];
|
||||
foreach (Datapoint datapoint in entity.datapoints)
|
||||
{
|
||||
SimilarityMethod similarityMethod = datapoint.similarityMethod;
|
||||
List<(string, float)> list = [];
|
||||
foreach ((string, float[]) embedding in datapoint.embeddings)
|
||||
{
|
||||
string key = embedding.Item1;
|
||||
float value = similarityMethod.method(queryEmbeddings[embedding.Item1], embedding.Item2);
|
||||
list.Add((key, value));
|
||||
}
|
||||
datapointProbs.Add((datapoint.name, datapoint.probMethod.method(list)));
|
||||
}
|
||||
result.Add((entity.probMethod(datapointProbs), entity.name));
|
||||
result.Add((EvaluateEntityAgainstQueryEmbeddings(entity, queryEmbeddings), entity.name));
|
||||
}
|
||||
IEnumerable<(float, string)> sortedResults = result.OrderByDescending(s => s.Item1);
|
||||
if (topN is not null)
|
||||
@@ -204,6 +191,49 @@ public class Searchdomain
|
||||
return results;
|
||||
}
|
||||
|
||||
public Dictionary<string, float[]> GetQueryEmbeddings(string query)
|
||||
{
|
||||
bool hasQuery = embeddingCache.TryGet(query, out Dictionary<string, float[]> queryEmbeddings);
|
||||
bool allModelsInQuery = queryEmbeddings is not null && modelsInUse.All(model => queryEmbeddings.ContainsKey(model));
|
||||
if (!(hasQuery && allModelsInQuery) || queryEmbeddings is null)
|
||||
{
|
||||
queryEmbeddings = Datapoint.GenerateEmbeddings(query, modelsInUse, aIProvider, embeddingCache);
|
||||
if (!embeddingCache.TryGet(query, out var embeddingCacheForCurrentQuery))
|
||||
{
|
||||
embeddingCache.Set(query, queryEmbeddings);
|
||||
}
|
||||
else // embeddingCache already has an entry for this query, so the missing model-embedding pairs have to be filled in
|
||||
{
|
||||
foreach (KeyValuePair<string, float[]> kvp in queryEmbeddings) // kvp.Key = model, kvp.Value = embedding
|
||||
{
|
||||
if (!embeddingCache.TryGet(kvp.Key, out var _))
|
||||
{
|
||||
embeddingCacheForCurrentQuery[kvp.Key] = kvp.Value;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return queryEmbeddings;
|
||||
}
|
||||
|
||||
private static float EvaluateEntityAgainstQueryEmbeddings(Entity entity, Dictionary<string, float[]> queryEmbeddings)
|
||||
{
|
||||
List<(string, float)> datapointProbs = [];
|
||||
foreach (Datapoint datapoint in entity.datapoints)
|
||||
{
|
||||
SimilarityMethod similarityMethod = datapoint.similarityMethod;
|
||||
List<(string, float)> list = [];
|
||||
foreach ((string, float[]) embedding in datapoint.embeddings)
|
||||
{
|
||||
string key = embedding.Item1;
|
||||
float value = similarityMethod.method(queryEmbeddings[embedding.Item1], embedding.Item2);
|
||||
list.Add((key, value));
|
||||
}
|
||||
datapointProbs.Add((datapoint.name, datapoint.probMethod.method(list)));
|
||||
}
|
||||
return entity.probMethod(datapointProbs);
|
||||
}
|
||||
|
||||
public static List<string> GetModels(List<Entity> entities)
|
||||
{
|
||||
List<string> result = [];
|
||||
@@ -250,6 +280,53 @@ public class Searchdomain
|
||||
return JsonSerializer.Deserialize<SearchdomainSettings>(settingsString);
|
||||
}
|
||||
|
||||
public void ReconciliateOrInvalidateCacheForNewOrUpdatedEntity(Entity entity)
|
||||
{
|
||||
if (settings.CacheReconciliation)
|
||||
{
|
||||
foreach (KeyValuePair<string, DateTimedSearchResult> element in searchCache)
|
||||
{
|
||||
string query = element.Key;
|
||||
DateTimedSearchResult searchResult = element.Value;
|
||||
|
||||
Dictionary<string, float[]> queryEmbeddings = GetQueryEmbeddings(query);
|
||||
float evaluationResult = EvaluateEntityAgainstQueryEmbeddings(entity, queryEmbeddings);
|
||||
|
||||
searchResult.Results.RemoveAll(x => x.Name == entity.name); // If entity already exists in that results list: remove it.
|
||||
|
||||
ResultItem newItem = new(evaluationResult, entity.name);
|
||||
int index = searchResult.Results.BinarySearch(
|
||||
newItem,
|
||||
Comparer<ResultItem>.Create((a, b) => b.Score.CompareTo(a.Score)) // Invert searching order
|
||||
);
|
||||
if (index < 0) // If not found, BinarySearch gives the bitwise complement
|
||||
index = ~index;
|
||||
searchResult.Results.Insert(index, newItem);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
InvalidateSearchCache();
|
||||
}
|
||||
}
|
||||
|
||||
public void ReconciliateOrInvalidateCacheForDeletedEntity(Entity entity)
|
||||
{
|
||||
if (settings.CacheReconciliation)
|
||||
{
|
||||
foreach (KeyValuePair<string, DateTimedSearchResult> element in searchCache)
|
||||
{
|
||||
string query = element.Key;
|
||||
DateTimedSearchResult searchResult = element.Value;
|
||||
searchResult.Results.RemoveAll(x => x.Name == entity.name);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
InvalidateSearchCache();
|
||||
}
|
||||
}
|
||||
|
||||
public void InvalidateSearchCache()
|
||||
{
|
||||
searchCache = [];
|
||||
|
||||
@@ -3,6 +3,9 @@ using System.Data.Common;
|
||||
using Server.Migrations;
|
||||
using Server.Helper;
|
||||
using Server.Exceptions;
|
||||
using AdaptiveExpressions;
|
||||
using Shared.Models;
|
||||
using System.Text.Json;
|
||||
|
||||
namespace Server;
|
||||
|
||||
@@ -16,7 +19,8 @@ public class SearchdomainManager
|
||||
private readonly string connectionString;
|
||||
private MySqlConnection connection;
|
||||
public SQLHelper helper;
|
||||
public Dictionary<string, Dictionary<string, float[]>> embeddingCache;
|
||||
public LRUCache<string, Dictionary<string, float[]>> embeddingCache;
|
||||
public int EmbeddingCacheMaxCount;
|
||||
|
||||
public SearchdomainManager(ILogger<SearchdomainManager> logger, IConfiguration config, AIProvider aIProvider, DatabaseHelper databaseHelper)
|
||||
{
|
||||
@@ -24,7 +28,8 @@ public class SearchdomainManager
|
||||
_config = config;
|
||||
this.aIProvider = aIProvider;
|
||||
_databaseHelper = databaseHelper;
|
||||
embeddingCache = [];
|
||||
EmbeddingCacheMaxCount = config.GetValue<int?>("Embeddingsearch:EmbeddingCacheMaxCount") ?? 1000000;
|
||||
embeddingCache = new(EmbeddingCacheMaxCount);
|
||||
connectionString = _config.GetSection("Embeddingsearch").GetConnectionString("SQL") ?? "";
|
||||
connection = new MySqlConnection(connectionString);
|
||||
connection.Open();
|
||||
@@ -66,7 +71,7 @@ public class SearchdomainManager
|
||||
{
|
||||
var searchdomain = GetSearchdomain(searchdomainName);
|
||||
searchdomain.UpdateEntityCache();
|
||||
searchdomain.InvalidateSearchCache(); // TODO implement cache remediation (Suggestion: searchdomain-wide setting for cache remediation / invalidation - )
|
||||
searchdomain.InvalidateSearchCache();
|
||||
}
|
||||
|
||||
public List<string> ListSearchdomains()
|
||||
@@ -84,6 +89,10 @@ public class SearchdomainManager
|
||||
}
|
||||
}
|
||||
|
||||
public int CreateSearchdomain(string searchdomain, SearchdomainSettings settings)
|
||||
{
|
||||
return CreateSearchdomain(searchdomain, JsonSerializer.Serialize(settings));
|
||||
}
|
||||
public int CreateSearchdomain(string searchdomain, string settings = "{}")
|
||||
{
|
||||
if (searchdomains.TryGetValue(searchdomain, out Searchdomain? value))
|
||||
|
||||
@@ -7,6 +7,7 @@
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="AdaptiveExpressions" Version="4.23.0" />
|
||||
<PackageReference Include="ElmahCore" Version="2.1.2" />
|
||||
<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
|
||||
<PackageReference Include="Serilog.AspNetCore" Version="9.0.0" />
|
||||
|
||||
@@ -1,16 +1,18 @@
|
||||
using System.Numerics.Tensors;
|
||||
using System.Text.Json;
|
||||
using Shared.Models;
|
||||
|
||||
namespace Server;
|
||||
|
||||
public class SimilarityMethod
|
||||
{
|
||||
public SimilarityMethods.similarityMethodDelegate method;
|
||||
public SimilarityMethodEnum similarityMethodEnum;
|
||||
public string name;
|
||||
|
||||
public SimilarityMethod(string name, ILogger logger)
|
||||
public SimilarityMethod(SimilarityMethodEnum similarityMethodEnum, ILogger logger)
|
||||
{
|
||||
this.name = name;
|
||||
this.similarityMethodEnum = similarityMethodEnum;
|
||||
this.name = similarityMethodEnum.ToString();
|
||||
SimilarityMethods.similarityMethodDelegate? probMethod = SimilarityMethods.GetMethod(name);
|
||||
if (probMethod is null)
|
||||
{
|
||||
@@ -21,14 +23,6 @@ public class SimilarityMethod
|
||||
}
|
||||
}
|
||||
|
||||
public enum SimilarityMethodEnum
|
||||
{
|
||||
Cosine,
|
||||
Euclidian,
|
||||
Manhattan,
|
||||
Pearson
|
||||
}
|
||||
|
||||
public static class SimilarityMethods
|
||||
{
|
||||
public delegate float similarityMethodProtoDelegate(float[] vector1, float[] vector2);
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
@using Server.Models
|
||||
@using System.Web
|
||||
@using Shared.Models
|
||||
@using Server.Services
|
||||
@using Server
|
||||
|
||||
@@ -744,8 +745,8 @@
|
||||
"datapoints": datapoints
|
||||
}];
|
||||
showToast("@T["Creating entity"]", "primary");
|
||||
fetch(`/Entity/Index`, {
|
||||
method: 'POST',
|
||||
fetch(`/Entity`, {
|
||||
method: 'PUT',
|
||||
headers: {
|
||||
'Content-Type': 'application/json'
|
||||
},
|
||||
@@ -777,8 +778,12 @@
|
||||
const cacheReconciliation = document.getElementById('createSearchdomainWithCacheReconciliation').checked;
|
||||
const settings = { CacheReconciliation: cacheReconciliation };
|
||||
// Implement create logic here
|
||||
fetch(`/Searchdomain/Create?searchdomain=${encodeURIComponent(name)}&settings=${JSON.stringify(settings)}`, {
|
||||
method: 'GET'
|
||||
fetch(`/Searchdomain?searchdomain=${encodeURIComponent(name)}`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json'
|
||||
},
|
||||
body: JSON.stringify(settings)
|
||||
}).then(response => {
|
||||
if (response.ok) {
|
||||
showToast("@T["Searchdomain was created successfully"]", "success");
|
||||
@@ -799,8 +804,8 @@
|
||||
.getElementById('cacheClear')
|
||||
.addEventListener('click', () => {
|
||||
const domainKey = getSelectedDomainKey();
|
||||
fetch(`/Searchdomain/ClearSearchCache?searchdomain=${encodeURIComponent(domains[domainKey])}`, {
|
||||
method: 'GET'
|
||||
fetch(`/Searchdomain/SearchCache/Clear?searchdomain=${encodeURIComponent(domains[domainKey])}`, {
|
||||
method: 'POST'
|
||||
}).then(response => {
|
||||
if (response.ok) {
|
||||
showToast("@T["Searchdomain cache was cleared successfully"]", "success");
|
||||
@@ -822,8 +827,8 @@
|
||||
.addEventListener('click', () => {
|
||||
const domainKey = getSelectedDomainKey();
|
||||
const entityName = document.getElementById('EntityConfirmDelete').getAttribute('data-name');
|
||||
fetch(`/Entity/Delete?searchdomain=${encodeURIComponent(domains[domainKey])}&entityName=${entityName}`, {
|
||||
method: 'GET'
|
||||
fetch(`/Entity?searchdomain=${encodeURIComponent(domains[domainKey])}&entityName=${entityName}`, {
|
||||
method: 'DELETE'
|
||||
}).then(async response => {
|
||||
let result = await response.json();
|
||||
if (response.ok && result.Success) {
|
||||
@@ -831,10 +836,11 @@
|
||||
console.log('Entity deleted successfully');
|
||||
selectDomain(getSelectedDomainKey());
|
||||
} else {
|
||||
showToast("@T["Failed to delete entity"]", "danger");
|
||||
showToast("@Html.Raw(T["Failed to delete entity"])", "danger");
|
||||
console.error('Failed to delete entity:', result.Message);
|
||||
}
|
||||
}).catch(error => {
|
||||
showToast("@Html.Raw(T["Failed to delete entity"])", "danger");
|
||||
console.error('Error deleting entity:', error);
|
||||
});
|
||||
});
|
||||
@@ -868,8 +874,8 @@
|
||||
"datapoints": datapoints
|
||||
}];
|
||||
showToast("@T["Updating entity"]", "primary");
|
||||
fetch(`/Entity/Index`, {
|
||||
method: 'POST',
|
||||
fetch(`/Entity`, {
|
||||
method: 'PUT',
|
||||
headers: {
|
||||
'Content-Type': 'application/json'
|
||||
},
|
||||
@@ -895,7 +901,7 @@
|
||||
.addEventListener('click', () => {
|
||||
let searchdomain = domains[getSelectedDomainKey()];
|
||||
let query = document.getElementById('deleteQueryConfirmationModalName').textContent;
|
||||
fetch(`/Searchdomain/Searches?searchdomain=${searchdomain}&query=${query}`, {
|
||||
fetch(`/Searchdomain/Query?searchdomain=${searchdomain}&query=${query}`, {
|
||||
method: 'DELETE',
|
||||
headers: {
|
||||
'Content-Type': 'application/json'
|
||||
@@ -907,11 +913,11 @@
|
||||
console.log('Search query was deleted successfully');
|
||||
selectDomain(getSelectedDomainKey());
|
||||
} else {
|
||||
showToast("@T["Failed to delete search query"]", "danger");
|
||||
showToast("@Html.Raw(T["Failed to delete search query"])", "danger");
|
||||
console.error('Failed to delete search query:', result.Message);
|
||||
}
|
||||
}).catch(error => {
|
||||
showToast("@T["Failed to delete search query"]", "danger");
|
||||
showToast("@Html.Raw(T["Failed to delete search query"])", "danger");
|
||||
console.error('Failed to delete search query:', error);
|
||||
});
|
||||
});
|
||||
@@ -922,7 +928,7 @@
|
||||
let query = document.getElementById('queryUpdateQueryName').textContent;
|
||||
let data = getQueryUpdateTableData();
|
||||
console.log()
|
||||
fetch(`/Searchdomain/Searches?searchdomain=${searchdomain}&query=${query}`, {
|
||||
fetch(`/Searchdomain/Query?searchdomain=${searchdomain}&query=${query}`, {
|
||||
method: 'PATCH',
|
||||
headers: {
|
||||
'Content-Type': 'application/json'
|
||||
@@ -947,8 +953,8 @@
|
||||
|
||||
function deleteSearchdomain(domainKey) {
|
||||
// Implement delete logic here
|
||||
fetch(`/Searchdomain/Delete?searchdomain=${encodeURI(domains[domainKey])}`, {
|
||||
method: 'GET'
|
||||
fetch(`/Searchdomain?searchdomain=${encodeURI(domains[domainKey])}`, {
|
||||
method: 'DELETE'
|
||||
}).then(async response => {
|
||||
var result = await response.json();;
|
||||
if (response.ok && result.Success === true) {
|
||||
@@ -958,19 +964,19 @@
|
||||
domainItem.remove();
|
||||
console.log('Searchdomain deleted successfully');
|
||||
} else {
|
||||
showToast("@T["Failed to delete searchdomain"]", "danger");
|
||||
showToast("@Html.Raw(T["Failed to delete searchdomain"])", "danger");
|
||||
console.error('Failed to delete searchdomain:', result.Message);
|
||||
}
|
||||
}).catch(error => {
|
||||
showToast("@T["Failed to delete searchdomain"]", "danger");
|
||||
showToast("@Html.Raw(T["Failed to delete searchdomain"])", "danger");
|
||||
console.error('Error deleting searchdomain:', error);
|
||||
});
|
||||
}
|
||||
|
||||
function renameSearchdomain(domainKey, newName) {
|
||||
// Implement rename logic here
|
||||
fetch(`/Searchdomain/Update?searchdomain=${encodeURI(domains[domainKey])}&newName=${newName}`, {
|
||||
method: 'GET'
|
||||
fetch(`/Searchdomain?searchdomain=${encodeURI(domains[domainKey])}&newName=${newName}`, {
|
||||
method: 'PUT'
|
||||
}).then(async response => {
|
||||
var result = await response.json();
|
||||
if (response.ok && result.Success === true) {
|
||||
@@ -994,8 +1000,8 @@
|
||||
|
||||
function updateSearchdomainConfig(domainKey, newSettings) {
|
||||
// Implement update logic here
|
||||
fetch(`/Searchdomain/UpdateSettings?searchdomain=${encodeURIComponent(domains[domainKey])}`, {
|
||||
method: 'POST',
|
||||
fetch(`/Searchdomain/Settings?searchdomain=${encodeURIComponent(domains[domainKey])}`, {
|
||||
method: 'PUT',
|
||||
headers: {
|
||||
'Content-Type': 'application/json'
|
||||
},
|
||||
@@ -1020,17 +1026,17 @@
|
||||
}
|
||||
|
||||
function getSearchdomainConfig(domainKey) {
|
||||
return fetch(`/Searchdomain/GetSettings?searchdomain=${encodeURIComponent(domains[domainKey])}`)
|
||||
return fetch(`/Searchdomain/Settings?searchdomain=${encodeURIComponent(domains[domainKey])}`)
|
||||
.then(r => r.json());
|
||||
}
|
||||
|
||||
function getSearchdomainCacheUtilization(domainKey) {
|
||||
return fetch(`/Searchdomain/GetSearchCacheSize?searchdomain=${encodeURIComponent(domains[domainKey])}`)
|
||||
return fetch(`/Searchdomain/SearchCache/Size?searchdomain=${encodeURIComponent(domains[domainKey])}`)
|
||||
.then(r => r.json());
|
||||
}
|
||||
|
||||
function getSearchdomainDatabaseUtilization(domainKey) {
|
||||
return fetch(`/Searchdomain/GetDatabaseSize?searchdomain=${encodeURIComponent(domains[domainKey])}`)
|
||||
return fetch(`/Searchdomain/Database/Size?searchdomain=${encodeURIComponent(domains[domainKey])}`)
|
||||
.then(r => r.json());
|
||||
}
|
||||
|
||||
@@ -1052,7 +1058,7 @@
|
||||
let databaseUtilizationPromise = getSearchdomainDatabaseUtilization(getSelectedDomainKey());
|
||||
|
||||
/* ---------- ENTITIES ---------- */
|
||||
let entitiesUrl = `/Entity/List?searchdomain=${encodeURIComponent(domainName)}&returnEmbeddings=false&returnModels=true`;
|
||||
let entitiesUrl = `/Entities?searchdomain=${encodeURIComponent(domainName)}&returnEmbeddings=false&returnModels=true`;
|
||||
let entitiesCard = document.querySelector("#entitiesTable").parentElement;
|
||||
clearEntitiesTable();
|
||||
showThrobber(entitiesCard);
|
||||
@@ -1071,7 +1077,7 @@
|
||||
});
|
||||
|
||||
/* ---------- QUERIES ---------- */
|
||||
let queriesUrl = `/Searchdomain/GetSearches?searchdomain=${encodeURIComponent(domainName)}`;
|
||||
let queriesUrl = `/Searchdomain/Queries?searchdomain=${encodeURIComponent(domainName)}`;
|
||||
let queriesCard = document.querySelector("#queriesTable").parentElement;
|
||||
clearQueriesTable();
|
||||
showThrobber(queriesCard);
|
||||
@@ -1091,7 +1097,6 @@
|
||||
searchdomainConfigPromise.then(searchdomainConfig => {
|
||||
if (searchdomainConfig != null && searchdomainConfig.Settings != null)
|
||||
{
|
||||
console.log(searchdomainConfig);
|
||||
configElementCachereconciliation.checked = searchdomainConfig.Settings.CacheReconciliation;
|
||||
configElementCachereconciliation.disabled = false;
|
||||
} else {
|
||||
@@ -1197,7 +1202,7 @@
|
||||
|
||||
var deleteButton = document.createElement('button');
|
||||
deleteButton.className = 'btn btn-danger btn-sm';
|
||||
deleteButton.textContent = '@T["Delete"]';
|
||||
deleteButton.textContent = '@Html.Raw(T["Delete"])';
|
||||
deleteButton.setAttribute("data-index", entities.findIndex(en => en == entity));
|
||||
deleteButton.addEventListener('click', () => {
|
||||
const modal = new bootstrap.Modal(
|
||||
@@ -1255,7 +1260,7 @@
|
||||
|
||||
const btnDelete = document.createElement('button');
|
||||
btnDelete.className = 'btn btn-sm btn-danger';
|
||||
btnDelete.textContent = '@T["Delete"]';
|
||||
btnDelete.textContent = '@Html.Raw(T["Delete"])';
|
||||
btnDelete.addEventListener('click', () => {
|
||||
const modal = new bootstrap.Modal(
|
||||
document.getElementById('deleteQueryModal')
|
||||
@@ -1296,7 +1301,7 @@
|
||||
|
||||
function showEntityDetails(entity) {
|
||||
// Title
|
||||
document.getElementById('entityDetailsTitle').innerText = entity.Name;
|
||||
document.getElementById('entityDetailsTitle').innerText = '@T["Details"] - ' + entity.Name;
|
||||
|
||||
// Attributes
|
||||
const attrBody = document.getElementById('entityAttributesBody');
|
||||
@@ -1442,7 +1447,7 @@
|
||||
const tdAction = document.createElement('td');
|
||||
const deleteButton = document.createElement('button');
|
||||
deleteButton.classList.add('btn', 'btn-danger', 'btn-sm');
|
||||
deleteButton.innerText = '@T["Delete"]';
|
||||
deleteButton.innerText = '@Html.Raw(T["Delete"])';
|
||||
deleteButton.onclick = function() {
|
||||
row.remove();
|
||||
};
|
||||
@@ -1541,7 +1546,7 @@
|
||||
var tdKey = document.createElement('td');
|
||||
var keyInput = document.createElement('input');
|
||||
keyInput.classList.add('form-control');
|
||||
keyInput.ariaLabel = '@T["Key"]';
|
||||
keyInput.ariaLabel = '@Html.Raw(T["Key"])';
|
||||
keyInput.value = key;
|
||||
tdKey.append(keyInput);
|
||||
var tdValue = document.createElement('td');
|
||||
|
||||
@@ -1,6 +0,0 @@
|
||||
@{
|
||||
ViewData["Title"] = "Privacy Policy";
|
||||
}
|
||||
<h1>@ViewData["Title"]</h1>
|
||||
|
||||
<p>Use this page to detail your site's privacy policy.</p>
|
||||
@@ -54,7 +54,7 @@
|
||||
|
||||
<footer class="border-top footer text-muted">
|
||||
<div class="container">
|
||||
© 2025 - embeddingsearch - <a asp-area="" asp-controller="Home" asp-action="Privacy">Privacy</a>
|
||||
© 2025 - embeddingsearch
|
||||
</div>
|
||||
</footer>
|
||||
<script src="~/lib/jquery/dist/jquery.min.js"></script>
|
||||
|
||||
@@ -24,6 +24,7 @@
|
||||
"172.17.0.1"
|
||||
]
|
||||
},
|
||||
"EmbeddingCacheMaxCount": 5,
|
||||
"AiProviders": {
|
||||
"ollama": {
|
||||
"handler": "ollama",
|
||||
@@ -35,6 +36,15 @@
|
||||
"ApiKey": "Some API key here"
|
||||
}
|
||||
},
|
||||
"SimpleAuth": {
|
||||
"Users": [
|
||||
{
|
||||
"Username": "admin",
|
||||
"Password": "UnsafePractice.67",
|
||||
"Roles": ["Admin"]
|
||||
}
|
||||
]
|
||||
},
|
||||
"ApiKeys": ["Some UUID here", "Another UUID here"],
|
||||
"UseHttpsRedirection": true
|
||||
}
|
||||
|
||||
12
src/Shared/Models/BaseModels.cs
Normal file
12
src/Shared/Models/BaseModels.cs
Normal file
@@ -0,0 +1,12 @@
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace Shared.Models;
|
||||
|
||||
public class SuccesMessageBaseModel
|
||||
{
|
||||
[JsonPropertyName("Success")]
|
||||
public required bool Success { get; set; }
|
||||
[JsonPropertyName("Message")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public string? Message { get; set; }
|
||||
}
|
||||
@@ -3,14 +3,10 @@ using System.Text.Json.Serialization;
|
||||
namespace Shared.Models;
|
||||
|
||||
|
||||
public class EntityQueryResults
|
||||
public class EntityQueryResults : SuccesMessageBaseModel
|
||||
{
|
||||
[JsonPropertyName("Results")]
|
||||
public required List<EntityQueryResult> Results { get; set; }
|
||||
[JsonPropertyName("Success")]
|
||||
public required bool Success { get; set; }
|
||||
[JsonPropertyName("Message")]
|
||||
public string? Message { get; set; }
|
||||
}
|
||||
|
||||
public class EntityQueryResult
|
||||
@@ -19,20 +15,19 @@ public class EntityQueryResult
|
||||
public required string Name { get; set; }
|
||||
[JsonPropertyName("Value")]
|
||||
public float Value { get; set; }
|
||||
[JsonPropertyName("Attributes")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public Dictionary<string, string>? Attributes { get; set; }
|
||||
}
|
||||
|
||||
public class EntityIndexResult
|
||||
{
|
||||
[JsonPropertyName("Success")]
|
||||
public required bool Success { get; set; }
|
||||
[JsonPropertyName("Message")]
|
||||
public string? Message { get; set; }
|
||||
}
|
||||
public class EntityIndexResult : SuccesMessageBaseModel {}
|
||||
|
||||
public class EntityListResults
|
||||
{
|
||||
[JsonPropertyName("Results")]
|
||||
public required List<EntityListResult> Results { get; set; }
|
||||
[JsonPropertyName("Message")]
|
||||
public string? Message { get; set; }
|
||||
[JsonPropertyName("Success")]
|
||||
public required bool Success { get; set; }
|
||||
}
|
||||
@@ -77,11 +72,5 @@ public class EmbeddingResult
|
||||
public required float[] Embeddings { get; set; }
|
||||
}
|
||||
|
||||
public class EntityDeleteResults
|
||||
{
|
||||
[JsonPropertyName("Success")]
|
||||
public required bool Success { get; set; }
|
||||
[JsonPropertyName("Message")]
|
||||
public string? Message { get; set; }
|
||||
}
|
||||
public class EntityDeleteResults : SuccesMessageBaseModel {}
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@ namespace Shared.Models;
|
||||
public class JSONEntity
|
||||
{
|
||||
public required string Name { get; set; }
|
||||
public required string Probmethod { get; set; }
|
||||
public required ProbMethodEnum Probmethod { get; set; }
|
||||
public required string Searchdomain { get; set; }
|
||||
public required Dictionary<string, string> Attributes { get; set; }
|
||||
public required JSONDatapoint[] Datapoints { get; set; }
|
||||
@@ -13,7 +13,27 @@ public class JSONDatapoint
|
||||
{
|
||||
public required string Name { get; set; }
|
||||
public required string? Text { get; set; }
|
||||
public required string Probmethod_embedding { get; set; }
|
||||
public required string SimilarityMethod { get; set; }
|
||||
public required ProbMethodEnum Probmethod_embedding { get; set; }
|
||||
public required SimilarityMethodEnum SimilarityMethod { get; set; }
|
||||
public required string[] Model { get; set; }
|
||||
}
|
||||
|
||||
public enum ProbMethodEnum
|
||||
{
|
||||
Mean,
|
||||
HarmonicMean,
|
||||
QuadraticMean,
|
||||
GeometricMean,
|
||||
EVEWAvg,
|
||||
HVEWAvg,
|
||||
LVEWAvg,
|
||||
DictionaryWeightedAverage
|
||||
}
|
||||
|
||||
public enum SimilarityMethodEnum
|
||||
{
|
||||
Cosine,
|
||||
Euclidian,
|
||||
Manhattan,
|
||||
Pearson
|
||||
}
|
||||
@@ -11,109 +11,46 @@ public class SearchdomainListResults
|
||||
public string? Message { get; set; }
|
||||
}
|
||||
|
||||
public class SearchdomainCreateResults
|
||||
public class SearchdomainCreateResults : SuccesMessageBaseModel
|
||||
{
|
||||
[JsonPropertyName("Success")]
|
||||
public required bool Success { get; set; }
|
||||
|
||||
[JsonPropertyName("Message")]
|
||||
public string? Message { get; set; }
|
||||
|
||||
[JsonPropertyName("Id")]
|
||||
public int? Id { get; set; }
|
||||
}
|
||||
|
||||
public class SearchdomainUpdateResults
|
||||
public class SearchdomainUpdateResults : SuccesMessageBaseModel {}
|
||||
|
||||
public class SearchdomainDeleteResults : SuccesMessageBaseModel
|
||||
{
|
||||
[JsonPropertyName("Success")]
|
||||
public required bool Success { get; set; }
|
||||
|
||||
[JsonPropertyName("Message")]
|
||||
public string? Message { get; set; }
|
||||
}
|
||||
|
||||
public class SearchdomainDeleteResults
|
||||
{
|
||||
[JsonPropertyName("Success")]
|
||||
public required bool Success { get; set; }
|
||||
|
||||
[JsonPropertyName("Message")]
|
||||
public string? Message { get; set; }
|
||||
|
||||
[JsonPropertyName("DeletedEntities")]
|
||||
public required int DeletedEntities { get; set; }
|
||||
}
|
||||
|
||||
public class SearchdomainSearchesResults
|
||||
public class SearchdomainSearchesResults : SuccesMessageBaseModel
|
||||
{
|
||||
[JsonPropertyName("Success")]
|
||||
public required bool Success { get; set; }
|
||||
|
||||
[JsonPropertyName("Message")]
|
||||
public string? Message { get; set; }
|
||||
[JsonPropertyName("Searches")]
|
||||
public required Dictionary<string, DateTimedSearchResult> Searches { get; set; }
|
||||
}
|
||||
|
||||
public class SearchdomainDeleteSearchResult
|
||||
public class SearchdomainDeleteSearchResult : SuccesMessageBaseModel {}
|
||||
|
||||
public class SearchdomainUpdateSearchResult : SuccesMessageBaseModel {}
|
||||
|
||||
public class SearchdomainSettingsResults : SuccesMessageBaseModel
|
||||
{
|
||||
[JsonPropertyName("Success")]
|
||||
public required bool Success { get; set; }
|
||||
|
||||
[JsonPropertyName("Message")]
|
||||
public string? Message { get; set; }
|
||||
}
|
||||
|
||||
public class SearchdomainUpdateSearchResult
|
||||
{
|
||||
[JsonPropertyName("Success")]
|
||||
public required bool Success { get; set; }
|
||||
|
||||
[JsonPropertyName("Message")]
|
||||
public string? Message { get; set; }
|
||||
}
|
||||
|
||||
public class SearchdomainSettingsResults
|
||||
{
|
||||
[JsonPropertyName("Success")]
|
||||
public required bool Success { get; set; }
|
||||
|
||||
[JsonPropertyName("Message")]
|
||||
public string? Message { get; set; }
|
||||
|
||||
[JsonPropertyName("Settings")]
|
||||
public required SearchdomainSettings? Settings { get; set; }
|
||||
}
|
||||
|
||||
public class SearchdomainSearchCacheSizeResults
|
||||
public class SearchdomainSearchCacheSizeResults : SuccesMessageBaseModel
|
||||
{
|
||||
[JsonPropertyName("Success")]
|
||||
public required bool Success { get; set; }
|
||||
|
||||
[JsonPropertyName("Message")]
|
||||
public string? Message { get; set; }
|
||||
|
||||
[JsonPropertyName("SearchCacheSizeBytes")]
|
||||
public required long? SearchCacheSizeBytes { get; set; }
|
||||
}
|
||||
|
||||
public class SearchdomainInvalidateCacheResults
|
||||
public class SearchdomainInvalidateCacheResults : SuccesMessageBaseModel {}
|
||||
|
||||
public class SearchdomainGetDatabaseSizeResult : SuccesMessageBaseModel
|
||||
{
|
||||
[JsonPropertyName("Success")]
|
||||
public required bool Success { get; set; }
|
||||
|
||||
[JsonPropertyName("Message")]
|
||||
public string? Message { get; set; }
|
||||
}
|
||||
|
||||
public class SearchdomainGetDatabaseSizeResult
|
||||
{
|
||||
[JsonPropertyName("Success")]
|
||||
public required bool Success { get; set; }
|
||||
|
||||
[JsonPropertyName("Message")]
|
||||
public string? Message { get; set; }
|
||||
|
||||
[JsonPropertyName("SearchdomainDatabaseSizeBytes")]
|
||||
public required long? SearchdomainDatabaseSizeBytes { get; set; }
|
||||
}
|
||||
|
||||
@@ -2,14 +2,8 @@ using System.Text.Json.Serialization;
|
||||
|
||||
namespace Shared.Models;
|
||||
|
||||
public class ServerGetModelsResult
|
||||
public class ServerGetModelsResult : SuccesMessageBaseModel
|
||||
{
|
||||
[JsonPropertyName("Success")]
|
||||
public required bool Success { get; set; }
|
||||
|
||||
[JsonPropertyName("Message")]
|
||||
public string? Message { get; set; }
|
||||
|
||||
[JsonPropertyName("Models")]
|
||||
public string[]? Models { get; set; }
|
||||
}
|
||||
Reference in New Issue
Block a user