Compare commits
12 Commits
b5545bb5c8
...
41-create-
| Author | SHA1 | Date | |
|---|---|---|---|
| e64f4ca555 | |||
| 55a475704e | |||
|
|
89f8aa6591 | ||
| a3cc5115fc | |||
| 8b36c65437 | |||
| ee12986fef | |||
| fd76da265b | |||
| 4aa3015954 | |||
| fb1766b2b5 | |||
| 6fe96d41a2 | |||
| fc5e8ceeee | |||
| 33502640a2 |
@@ -55,10 +55,7 @@ public class EntityController : ControllerBase
|
|||||||
try
|
try
|
||||||
{
|
{
|
||||||
List<Entity>? entities = _searchdomainHelper.EntitiesFromJSON(
|
List<Entity>? entities = _searchdomainHelper.EntitiesFromJSON(
|
||||||
[],
|
_domainManager,
|
||||||
_domainManager.embeddingCache,
|
|
||||||
_domainManager.aIProvider,
|
|
||||||
_domainManager.helper,
|
|
||||||
_logger,
|
_logger,
|
||||||
JsonSerializer.Serialize(jsonEntities));
|
JsonSerializer.Serialize(jsonEntities));
|
||||||
if (entities is not null && jsonEntities is not null)
|
if (entities is not null && jsonEntities is not null)
|
||||||
@@ -92,8 +89,13 @@ public class EntityController : ControllerBase
|
|||||||
}
|
}
|
||||||
|
|
||||||
[HttpGet("List")]
|
[HttpGet("List")]
|
||||||
public ActionResult<EntityListResults> List(string searchdomain, bool returnEmbeddings = false)
|
public ActionResult<EntityListResults> List(string searchdomain, bool returnModels = false, bool returnEmbeddings = false)
|
||||||
{
|
{
|
||||||
|
if (returnEmbeddings && !returnModels)
|
||||||
|
{
|
||||||
|
_logger.LogError("Invalid request for {searchdomain} - embeddings return requested but without models - not possible!", [searchdomain]);
|
||||||
|
return Ok(new EntityQueryResults() {Results = [], Success = false, Message = "Invalid request" });
|
||||||
|
}
|
||||||
Searchdomain searchdomain_;
|
Searchdomain searchdomain_;
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
@@ -118,12 +120,12 @@ public class EntityController : ControllerBase
|
|||||||
List<DatapointResult> datapointResults = [];
|
List<DatapointResult> datapointResults = [];
|
||||||
foreach (Datapoint datapoint in entity.datapoints)
|
foreach (Datapoint datapoint in entity.datapoints)
|
||||||
{
|
{
|
||||||
if (returnEmbeddings)
|
if (returnModels)
|
||||||
{
|
{
|
||||||
List<EmbeddingResult> embeddingResults = [];
|
List<EmbeddingResult> embeddingResults = [];
|
||||||
foreach ((string, float[]) embedding in datapoint.embeddings)
|
foreach ((string, float[]) embedding in datapoint.embeddings)
|
||||||
{
|
{
|
||||||
embeddingResults.Add(new EmbeddingResult() {Model = embedding.Item1, Embeddings = embedding.Item2});
|
embeddingResults.Add(new EmbeddingResult() {Model = embedding.Item1, Embeddings = returnEmbeddings ? embedding.Item2 : []});
|
||||||
}
|
}
|
||||||
datapointResults.Add(new DatapointResult() {Name = datapoint.name, ProbMethod = datapoint.probMethod.name, SimilarityMethod = datapoint.similarityMethod.name, Embeddings = embeddingResults});
|
datapointResults.Add(new DatapointResult() {Name = datapoint.name, ProbMethod = datapoint.probMethod.name, SimilarityMethod = datapoint.similarityMethod.name, Embeddings = embeddingResults});
|
||||||
}
|
}
|
||||||
@@ -135,6 +137,7 @@ public class EntityController : ControllerBase
|
|||||||
EntityListResult entityListResult = new()
|
EntityListResult entityListResult = new()
|
||||||
{
|
{
|
||||||
Name = entity.name,
|
Name = entity.name,
|
||||||
|
ProbMethod = entity.probMethodName,
|
||||||
Attributes = attributeResults,
|
Attributes = attributeResults,
|
||||||
Datapoints = datapointResults
|
Datapoints = datapointResults
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
using System.Text.Json;
|
using System.Text.Json;
|
||||||
using ElmahCore;
|
using ElmahCore;
|
||||||
|
using Microsoft.AspNetCore.Http.HttpResults;
|
||||||
using Microsoft.AspNetCore.Mvc;
|
using Microsoft.AspNetCore.Mvc;
|
||||||
using Server.Exceptions;
|
using Server.Exceptions;
|
||||||
using Server.Helper;
|
using Server.Helper;
|
||||||
@@ -156,6 +157,64 @@ public class SearchdomainController : ControllerBase
|
|||||||
return Ok(new SearchdomainSearchesResults() { Searches = searchCache, Success = true });
|
return Ok(new SearchdomainSearchesResults() { Searches = searchCache, Success = true });
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[HttpDelete("Searches")]
|
||||||
|
public ActionResult<SearchdomainDeleteSearchResult> DeleteSearch(string searchdomain, string query)
|
||||||
|
{
|
||||||
|
Searchdomain searchdomain_;
|
||||||
|
try
|
||||||
|
{
|
||||||
|
searchdomain_ = _domainManager.GetSearchdomain(searchdomain);
|
||||||
|
}
|
||||||
|
catch (SearchdomainNotFoundException)
|
||||||
|
{
|
||||||
|
_logger.LogError("Unable to retrieve the searchdomain {searchdomain} - it likely does not exist yet", [searchdomain]);
|
||||||
|
return Ok(new SearchdomainDeleteSearchResult() { Success = false, Message = "Searchdomain not found" });
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
_logger.LogError("Unable to retrieve the searchdomain {searchdomain} - {ex.Message} - {ex.StackTrace}", [searchdomain, ex.Message, ex.StackTrace]);
|
||||||
|
return Ok(new SearchdomainDeleteSearchResult() { Success = false, Message = ex.Message });
|
||||||
|
}
|
||||||
|
Dictionary<string, DateTimedSearchResult> searchCache = searchdomain_.searchCache;
|
||||||
|
bool containsKey = searchCache.ContainsKey(query);
|
||||||
|
if (containsKey)
|
||||||
|
{
|
||||||
|
searchCache.Remove(query);
|
||||||
|
return Ok(new SearchdomainDeleteSearchResult() {Success = true});
|
||||||
|
}
|
||||||
|
return Ok(new SearchdomainDeleteSearchResult() {Success = false, Message = "Query not found in search cache"});
|
||||||
|
}
|
||||||
|
|
||||||
|
[HttpPatch("Searches")]
|
||||||
|
public ActionResult<SearchdomainUpdateSearchResult> UpdateSearch(string searchdomain, string query, [FromBody]List<ResultItem> results)
|
||||||
|
{
|
||||||
|
Searchdomain searchdomain_;
|
||||||
|
try
|
||||||
|
{
|
||||||
|
searchdomain_ = _domainManager.GetSearchdomain(searchdomain);
|
||||||
|
}
|
||||||
|
catch (SearchdomainNotFoundException)
|
||||||
|
{
|
||||||
|
_logger.LogError("Unable to retrieve the searchdomain {searchdomain} - it likely does not exist yet", [searchdomain]);
|
||||||
|
return Ok(new SearchdomainUpdateSearchResult() { Success = false, Message = "Searchdomain not found" });
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
_logger.LogError("Unable to retrieve the searchdomain {searchdomain} - {ex.Message} - {ex.StackTrace}", [searchdomain, ex.Message, ex.StackTrace]);
|
||||||
|
return Ok(new SearchdomainUpdateSearchResult() { Success = false, Message = ex.Message });
|
||||||
|
}
|
||||||
|
Dictionary<string, DateTimedSearchResult> searchCache = searchdomain_.searchCache;
|
||||||
|
bool containsKey = searchCache.ContainsKey(query);
|
||||||
|
if (containsKey)
|
||||||
|
{
|
||||||
|
DateTimedSearchResult element = searchCache[query];
|
||||||
|
element.Results = results;
|
||||||
|
searchCache[query] = element;
|
||||||
|
return Ok(new SearchdomainUpdateSearchResult() {Success = true});
|
||||||
|
}
|
||||||
|
return Ok(new SearchdomainUpdateSearchResult() {Success = false, Message = "Query not found in search cache"});
|
||||||
|
}
|
||||||
|
|
||||||
[HttpGet("GetSettings")]
|
[HttpGet("GetSettings")]
|
||||||
public ActionResult<SearchdomainSettingsResults> GetSettings(string searchdomain)
|
public ActionResult<SearchdomainSettingsResults> GetSettings(string searchdomain)
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -1,9 +1,10 @@
|
|||||||
namespace Server;
|
namespace Server;
|
||||||
|
|
||||||
public class Entity(Dictionary<string, string> attributes, Probmethods.probMethodDelegate probMethod, List<Datapoint> datapoints, string name)
|
public class Entity(Dictionary<string, string> attributes, Probmethods.probMethodDelegate probMethod, string probMethodName, List<Datapoint> datapoints, string name)
|
||||||
{
|
{
|
||||||
public Dictionary<string, string> attributes = attributes;
|
public Dictionary<string, string> attributes = attributes;
|
||||||
public Probmethods.probMethodDelegate probMethod = probMethod;
|
public Probmethods.probMethodDelegate probMethod = probMethod;
|
||||||
|
public string probMethodName = probMethodName;
|
||||||
public List<Datapoint> datapoints = datapoints;
|
public List<Datapoint> datapoints = datapoints;
|
||||||
public int id;
|
public int id;
|
||||||
public string name = name;
|
public string name = name;
|
||||||
|
|||||||
@@ -44,8 +44,12 @@ public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelp
|
|||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
public List<Entity>? EntitiesFromJSON(List<Entity> entityCache, Dictionary<string, Dictionary<string, float[]>> embeddingCache, AIProvider aIProvider, SQLHelper helper, ILogger logger, string json)
|
public List<Entity>? EntitiesFromJSON(SearchdomainManager searchdomainManager, ILogger logger, string json)
|
||||||
{
|
{
|
||||||
|
Dictionary<string, Dictionary<string, float[]>> embeddingCache = searchdomainManager.embeddingCache;
|
||||||
|
AIProvider aIProvider = searchdomainManager.aIProvider;
|
||||||
|
SQLHelper helper = searchdomainManager.helper;
|
||||||
|
|
||||||
List<JSONEntity>? jsonEntities = JsonSerializer.Deserialize<List<JSONEntity>>(json);
|
List<JSONEntity>? jsonEntities = JsonSerializer.Deserialize<List<JSONEntity>>(json);
|
||||||
if (jsonEntities is null)
|
if (jsonEntities is null)
|
||||||
{
|
{
|
||||||
@@ -72,8 +76,7 @@ public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelp
|
|||||||
ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = 16 }; // <-- This is needed! Otherwise if we try to index 100+ entities at once, it spawns 100 threads, exploding the SQL pool
|
ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = 16 }; // <-- This is needed! Otherwise if we try to index 100+ entities at once, it spawns 100 threads, exploding the SQL pool
|
||||||
Parallel.ForEach(jsonEntities, parallelOptions, jSONEntity =>
|
Parallel.ForEach(jsonEntities, parallelOptions, jSONEntity =>
|
||||||
{
|
{
|
||||||
using var tempHelper = helper.DuplicateConnection();
|
var entity = EntityFromJSON(searchdomainManager, logger, jSONEntity);
|
||||||
var entity = EntityFromJSON(entityCache, embeddingCache, aIProvider, tempHelper, logger, jSONEntity);
|
|
||||||
if (entity is not null)
|
if (entity is not null)
|
||||||
{
|
{
|
||||||
retVal.Enqueue(entity);
|
retVal.Enqueue(entity);
|
||||||
@@ -82,37 +85,133 @@ public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelp
|
|||||||
return [.. retVal];
|
return [.. retVal];
|
||||||
}
|
}
|
||||||
|
|
||||||
public Entity? EntityFromJSON(List<Entity> entityCache, Dictionary<string, Dictionary<string, float[]>> embeddingCache, AIProvider aIProvider, SQLHelper helper, ILogger logger, JSONEntity jsonEntity) //string json)
|
public Entity? EntityFromJSON(SearchdomainManager searchdomainManager, ILogger logger, JSONEntity jsonEntity) //string json)
|
||||||
|
{
|
||||||
|
SQLHelper helper = searchdomainManager.helper.DuplicateConnection();
|
||||||
|
Searchdomain searchdomain = searchdomainManager.GetSearchdomain(jsonEntity.Searchdomain);
|
||||||
|
List<Entity> entityCache = searchdomain.entityCache;
|
||||||
|
AIProvider aIProvider = searchdomain.aIProvider;
|
||||||
|
Dictionary<string, Dictionary<string, float[]>> embeddingCache = searchdomain.embeddingCache;
|
||||||
|
Entity? preexistingEntity = entityCache.FirstOrDefault(entity => entity.name == jsonEntity.Name);
|
||||||
|
|
||||||
|
if (preexistingEntity is not null)
|
||||||
{
|
{
|
||||||
Dictionary<string, Dictionary<string, float[]>> embeddingsLUT = []; // embeddingsLUT: hash -> model -> [embeddingValues * n]
|
|
||||||
int? preexistingEntityID = _databaseHelper.GetEntityID(helper, jsonEntity.Name, jsonEntity.Searchdomain);
|
int? preexistingEntityID = _databaseHelper.GetEntityID(helper, jsonEntity.Name, jsonEntity.Searchdomain);
|
||||||
if (preexistingEntityID is not null)
|
if (preexistingEntityID is null)
|
||||||
{
|
{
|
||||||
lock (helper.connection) // TODO change this to helper and do A/B tests (i.e. before/after)
|
_logger.LogCritical("Unable to index entity {jsonEntity.Name} because it already exists in the searchdomain but not in the database.", [jsonEntity.Name]);
|
||||||
|
throw new Exception($"Unable to index entity {jsonEntity.Name} because it already exists in the searchdomain but not in the database.");
|
||||||
|
}
|
||||||
|
Dictionary<string, string> attributes = jsonEntity.Attributes;
|
||||||
|
|
||||||
|
// Attribute
|
||||||
|
foreach (KeyValuePair<string, string> attributesKV in preexistingEntity.attributes.ToList())
|
||||||
{
|
{
|
||||||
|
string oldAttributeKey = attributesKV.Key;
|
||||||
|
string oldAttribute = attributesKV.Value;
|
||||||
|
bool newHasAttribute = jsonEntity.Attributes.TryGetValue(oldAttributeKey, out string? newAttribute);
|
||||||
|
if (newHasAttribute && newAttribute is not null && newAttribute != oldAttribute)
|
||||||
|
{
|
||||||
|
// Attribute - Updated
|
||||||
Dictionary<string, dynamic> parameters = new()
|
Dictionary<string, dynamic> parameters = new()
|
||||||
{
|
{
|
||||||
{ "id", preexistingEntityID }
|
{ "newValue", newAttribute },
|
||||||
|
{ "entityId", preexistingEntityID },
|
||||||
|
{ "attribute", oldAttributeKey}
|
||||||
};
|
};
|
||||||
System.Data.Common.DbDataReader reader = helper.ExecuteSQLCommand("SELECT e.model, e.embedding, d.hash FROM datapoint d JOIN embedding e ON d.id = e.id_datapoint WHERE d.id_entity = @id", parameters);
|
helper.ExecuteSQLNonQuery("UPDATE attribute SET value=@newValue WHERE id_entity=@entityId AND attribute=@attribute", parameters);
|
||||||
while (reader.Read())
|
preexistingEntity.attributes[oldAttributeKey] = newAttribute;
|
||||||
|
} else if (!newHasAttribute)
|
||||||
{
|
{
|
||||||
string model = reader.GetString(0);
|
// Attribute - Deleted
|
||||||
long length = reader.GetBytes(1, 0, null, 0, 0);
|
Dictionary<string, dynamic> parameters = new()
|
||||||
byte[] embeddingBytes = new byte[length];
|
|
||||||
reader.GetBytes(1, 0, embeddingBytes, 0, (int)length);
|
|
||||||
float[] embeddingValues = FloatArrayFromBytes(embeddingBytes);
|
|
||||||
string hash = reader.GetString(2);
|
|
||||||
if (!embeddingsLUT.ContainsKey(hash))
|
|
||||||
{
|
{
|
||||||
embeddingsLUT[hash] = [];
|
{ "entityId", preexistingEntityID },
|
||||||
|
{ "attribute", oldAttributeKey}
|
||||||
|
};
|
||||||
|
helper.ExecuteSQLNonQuery("DELETE FROM attribute WHERE id_entity=@entityId AND attribute=@attribute", parameters);
|
||||||
|
preexistingEntity.attributes.Remove(oldAttributeKey);
|
||||||
}
|
}
|
||||||
embeddingsLUT[hash].TryAdd(model, embeddingValues);
|
|
||||||
}
|
}
|
||||||
reader.Close();
|
foreach (var attributesKV in jsonEntity.Attributes)
|
||||||
|
{
|
||||||
|
string newAttributeKey = attributesKV.Key;
|
||||||
|
string newAttribute = attributesKV.Value;
|
||||||
|
bool preexistingHasAttribute = preexistingEntity.attributes.TryGetValue(newAttributeKey, out string? preexistingAttribute);
|
||||||
|
if (!preexistingHasAttribute)
|
||||||
|
{
|
||||||
|
// Attribute - New
|
||||||
|
DatabaseHelper.DatabaseInsertAttribute(helper, newAttributeKey, newAttribute, (int)preexistingEntityID);
|
||||||
|
preexistingEntity.attributes.Add(newAttributeKey, newAttribute);
|
||||||
}
|
}
|
||||||
_databaseHelper.RemoveEntity(entityCache, helper, jsonEntity.Name, jsonEntity.Searchdomain); // TODO only remove entity if there is actually a change somewhere. Perhaps create 3 datapoint lists to operate with: 1. delete, 2. update, 3. create
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Datapoint
|
||||||
|
foreach (Datapoint datapoint in preexistingEntity.datapoints.ToList())
|
||||||
|
{
|
||||||
|
bool newEntityHasDatapoint = jsonEntity.Datapoints.Any(x => x.Name == datapoint.name);
|
||||||
|
if (!newEntityHasDatapoint)
|
||||||
|
{
|
||||||
|
// Datapoint - Deleted
|
||||||
|
Dictionary<string, dynamic> parameters = new()
|
||||||
|
{
|
||||||
|
{ "datapointName", datapoint.name },
|
||||||
|
{ "entityId", preexistingEntityID}
|
||||||
|
};
|
||||||
|
helper.ExecuteSQLNonQuery("DELETE e FROM embedding e JOIN datapoint d ON e.id_datapoint=d.id WHERE d.name=@datapointName AND d.id_entity=@entityId", parameters);
|
||||||
|
helper.ExecuteSQLNonQuery("DELETE FROM datapoint WHERE id_entity=@entityId AND name=@datapointName", parameters);
|
||||||
|
preexistingEntity.datapoints.Remove(datapoint);
|
||||||
|
} else
|
||||||
|
{
|
||||||
|
JSONDatapoint? newEntityDatapoint = jsonEntity.Datapoints.FirstOrDefault(x => x.Name == datapoint.name);
|
||||||
|
if (newEntityDatapoint is not null && newEntityDatapoint.Text is not null)
|
||||||
|
{
|
||||||
|
// Datapoint - Updated (text)
|
||||||
|
Dictionary<string, dynamic> parameters = new()
|
||||||
|
{
|
||||||
|
{ "datapointName", datapoint.name },
|
||||||
|
{ "entityId", preexistingEntityID}
|
||||||
|
};
|
||||||
|
helper.ExecuteSQLNonQuery("DELETE e FROM embedding e JOIN datapoint d ON e.id_datapoint=d.id WHERE d.name=@datapointName AND d.id_entity=@entityId", parameters);
|
||||||
|
helper.ExecuteSQLNonQuery("DELETE FROM datapoint WHERE id_entity=@entityId AND name=@datapointName", parameters);
|
||||||
|
preexistingEntity.datapoints.Remove(datapoint);
|
||||||
|
Datapoint newDatapoint = DatabaseInsertDatapointWithEmbeddings(helper, searchdomain, newEntityDatapoint, (int)preexistingEntityID);
|
||||||
|
preexistingEntity.datapoints.Add(newDatapoint);
|
||||||
|
|
||||||
|
}
|
||||||
|
if (newEntityDatapoint is not null && (newEntityDatapoint.Probmethod_embedding != datapoint.probMethod.name || newEntityDatapoint.SimilarityMethod != datapoint.similarityMethod.name))
|
||||||
|
{
|
||||||
|
// Datapoint - Updated (probmethod or similaritymethod)
|
||||||
|
Dictionary<string, dynamic> parameters = new()
|
||||||
|
{
|
||||||
|
{ "probmethod", newEntityDatapoint.Probmethod_embedding },
|
||||||
|
{ "similaritymethod", newEntityDatapoint.SimilarityMethod },
|
||||||
|
{ "datapointName", datapoint.name },
|
||||||
|
{ "entityId", preexistingEntityID}
|
||||||
|
};
|
||||||
|
helper.ExecuteSQLNonQuery("UPDATE datapoint SET probmethod_embedding=@probmethod, similaritymethod=@similaritymethod WHERE id_entity=@entityId AND name=@datapointName", parameters);
|
||||||
|
Datapoint preexistingDatapoint = preexistingEntity.datapoints.First(x => x == datapoint); // The for loop is a copy. This retrieves the original such that it can be updated.
|
||||||
|
preexistingDatapoint.probMethod = datapoint.probMethod;
|
||||||
|
preexistingDatapoint.similarityMethod = datapoint.similarityMethod;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
foreach (JSONDatapoint jsonDatapoint in jsonEntity.Datapoints)
|
||||||
|
{
|
||||||
|
bool oldEntityHasDatapoint = preexistingEntity.datapoints.Any(x => x.name == jsonDatapoint.Name);
|
||||||
|
if (!oldEntityHasDatapoint)
|
||||||
|
{
|
||||||
|
// Datapoint - New
|
||||||
|
Datapoint datapoint = DatabaseInsertDatapointWithEmbeddings(helper, searchdomain, jsonDatapoint, (int)preexistingEntityID);
|
||||||
|
preexistingEntity.datapoints.Add(datapoint);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
return preexistingEntity;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
int id_entity = DatabaseHelper.DatabaseInsertEntity(helper, jsonEntity.Name, jsonEntity.Probmethod, _databaseHelper.GetSearchdomainID(helper, jsonEntity.Searchdomain));
|
int id_entity = DatabaseHelper.DatabaseInsertEntity(helper, jsonEntity.Name, jsonEntity.Probmethod, _databaseHelper.GetSearchdomainID(helper, jsonEntity.Searchdomain));
|
||||||
foreach (KeyValuePair<string, string> attribute in jsonEntity.Attributes)
|
foreach (KeyValuePair<string, string> attribute in jsonEntity.Attributes)
|
||||||
{
|
{
|
||||||
@@ -123,30 +222,28 @@ public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelp
|
|||||||
foreach (JSONDatapoint jsonDatapoint in jsonEntity.Datapoints)
|
foreach (JSONDatapoint jsonDatapoint in jsonEntity.Datapoints)
|
||||||
{
|
{
|
||||||
string hash = Convert.ToBase64String(SHA256.HashData(Encoding.UTF8.GetBytes(jsonDatapoint.Text)));
|
string hash = Convert.ToBase64String(SHA256.HashData(Encoding.UTF8.GetBytes(jsonDatapoint.Text)));
|
||||||
Dictionary<string, float[]> embeddings = [];
|
Datapoint datapoint = DatabaseInsertDatapointWithEmbeddings(helper, searchdomain, jsonDatapoint, id_entity, hash);
|
||||||
if (embeddingsLUT.ContainsKey(hash))
|
datapoints.Add(datapoint);
|
||||||
{
|
|
||||||
Dictionary<string, float[]> hashLUT = embeddingsLUT[hash];
|
|
||||||
foreach (string model in jsonDatapoint.Model)
|
|
||||||
{
|
|
||||||
if (hashLUT.ContainsKey(model))
|
|
||||||
{
|
|
||||||
embeddings.Add(model, hashLUT[model]);
|
|
||||||
}
|
}
|
||||||
else
|
|
||||||
|
var probMethod = Probmethods.GetMethod(jsonEntity.Probmethod) ?? throw new ProbMethodNotFoundException(jsonEntity.Probmethod);
|
||||||
|
Entity entity = new(jsonEntity.Attributes, probMethod, jsonEntity.Probmethod, datapoints, jsonEntity.Name)
|
||||||
{
|
{
|
||||||
var additionalEmbeddings = Datapoint.GenerateEmbeddings(jsonDatapoint.Text, [model], aIProvider, embeddingCache);
|
id = id_entity
|
||||||
embeddings.Add(model, additionalEmbeddings.First().Value);
|
};
|
||||||
|
entityCache.Add(entity);
|
||||||
|
return entity;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
else
|
public Datapoint DatabaseInsertDatapointWithEmbeddings(SQLHelper helper, Searchdomain searchdomain, JSONDatapoint jsonDatapoint, int id_entity, string? hash = null)
|
||||||
{
|
{
|
||||||
embeddings = Datapoint.GenerateEmbeddings(jsonDatapoint.Text, [.. jsonDatapoint.Model], aIProvider, embeddingCache);
|
if (jsonDatapoint.Text is null)
|
||||||
|
{
|
||||||
|
throw new Exception("jsonDatapoint.Text must not be null at this point");
|
||||||
}
|
}
|
||||||
var probMethod_embedding = new ProbMethod(jsonDatapoint.Probmethod_embedding, logger) ?? throw new ProbMethodNotFoundException(jsonDatapoint.Probmethod_embedding);
|
hash ??= Convert.ToBase64String(SHA256.HashData(Encoding.UTF8.GetBytes(jsonDatapoint.Text)));
|
||||||
var similarityMethod = new SimilarityMethod(jsonDatapoint.SimilarityMethod, logger) ?? throw new SimilarityMethodNotFoundException(jsonDatapoint.SimilarityMethod);
|
Datapoint datapoint = BuildDatapointFromJsonDatapoint(jsonDatapoint, id_entity, searchdomain, hash);
|
||||||
Datapoint datapoint = new(jsonDatapoint.Name, probMethod_embedding, similarityMethod, hash, [.. embeddings.Select(kv => (kv.Key, kv.Value))]);
|
|
||||||
int id_datapoint = DatabaseHelper.DatabaseInsertDatapoint(helper, jsonDatapoint.Name, jsonDatapoint.Probmethod_embedding, jsonDatapoint.SimilarityMethod, hash, id_entity); // TODO make this a bulk add action to reduce number of queries
|
int id_datapoint = DatabaseHelper.DatabaseInsertDatapoint(helper, jsonDatapoint.Name, jsonDatapoint.Probmethod_embedding, jsonDatapoint.SimilarityMethod, hash, id_entity); // TODO make this a bulk add action to reduce number of queries
|
||||||
List<(string model, byte[] embedding)> data = [];
|
List<(string model, byte[] embedding)> data = [];
|
||||||
foreach ((string, float[]) embedding in datapoint.embeddings)
|
foreach ((string, float[]) embedding in datapoint.embeddings)
|
||||||
@@ -154,15 +251,22 @@ public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelp
|
|||||||
data.Add((embedding.Item1, BytesFromFloatArray(embedding.Item2)));
|
data.Add((embedding.Item1, BytesFromFloatArray(embedding.Item2)));
|
||||||
}
|
}
|
||||||
DatabaseHelper.DatabaseInsertEmbeddingBulk(helper, id_datapoint, data);
|
DatabaseHelper.DatabaseInsertEmbeddingBulk(helper, id_datapoint, data);
|
||||||
datapoints.Add(datapoint);
|
return datapoint;
|
||||||
}
|
}
|
||||||
|
|
||||||
var probMethod = Probmethods.GetMethod(jsonEntity.Probmethod) ?? throw new ProbMethodNotFoundException(jsonEntity.Probmethod);
|
public Datapoint BuildDatapointFromJsonDatapoint(JSONDatapoint jsonDatapoint, int entityId, Searchdomain searchdomain, string? hash = null)
|
||||||
Entity entity = new(jsonEntity.Attributes, probMethod, datapoints, jsonEntity.Name)
|
|
||||||
{
|
{
|
||||||
id = id_entity
|
if (jsonDatapoint.Text is null)
|
||||||
};
|
{
|
||||||
entityCache.Add(entity);
|
throw new Exception("jsonDatapoint.Text must not be null at this point");
|
||||||
return entity;
|
}
|
||||||
|
using SQLHelper helper = searchdomain.helper.DuplicateConnection();
|
||||||
|
Dictionary<string, Dictionary<string, float[]>> embeddingCache = searchdomain.embeddingCache;
|
||||||
|
hash ??= Convert.ToBase64String(SHA256.HashData(Encoding.UTF8.GetBytes(jsonDatapoint.Text)));
|
||||||
|
DatabaseHelper.DatabaseInsertDatapoint(helper, jsonDatapoint.Name, jsonDatapoint.Probmethod_embedding, jsonDatapoint.SimilarityMethod, hash, entityId);
|
||||||
|
Dictionary<string, float[]> embeddings = Datapoint.GenerateEmbeddings(jsonDatapoint.Text, [.. jsonDatapoint.Model], searchdomain.aIProvider, embeddingCache);
|
||||||
|
var probMethod_embedding = new ProbMethod(jsonDatapoint.Probmethod_embedding, logger) ?? throw new ProbMethodNotFoundException(jsonDatapoint.Probmethod_embedding);
|
||||||
|
var similarityMethod = new SimilarityMethod(jsonDatapoint.SimilarityMethod, logger) ?? throw new SimilarityMethodNotFoundException(jsonDatapoint.SimilarityMethod);
|
||||||
|
return new Datapoint(jsonDatapoint.Name, probMethod_embedding, similarityMethod, hash, [.. embeddings.Select(kv => (kv.Key, kv.Value))]);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -142,7 +142,7 @@ public class Searchdomain
|
|||||||
Probmethods.probMethodDelegate? probmethod = Probmethods.GetMethod(probmethodString);
|
Probmethods.probMethodDelegate? probmethod = Probmethods.GetMethod(probmethodString);
|
||||||
if (datapoint_unassigned.TryGetValue(id, out List<Datapoint>? datapoints) && probmethod is not null)
|
if (datapoint_unassigned.TryGetValue(id, out List<Datapoint>? datapoints) && probmethod is not null)
|
||||||
{
|
{
|
||||||
Entity entity = new(attributes, probmethod, datapoints, name)
|
Entity entity = new(attributes, probmethod, probmethodString, datapoints, name)
|
||||||
{
|
{
|
||||||
id = id
|
id = id
|
||||||
};
|
};
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -1,4 +1,7 @@
|
|||||||
<!DOCTYPE html>
|
@using Server.Services
|
||||||
|
@inject LocalizationService T
|
||||||
|
|
||||||
|
<!DOCTYPE html>
|
||||||
<html lang="en">
|
<html lang="en">
|
||||||
<head>
|
<head>
|
||||||
<meta charset="utf-8" />
|
<meta charset="utf-8" />
|
||||||
@@ -6,6 +9,11 @@
|
|||||||
<title>@ViewData["Title"] - embeddingsearch</title>
|
<title>@ViewData["Title"] - embeddingsearch</title>
|
||||||
<link rel="stylesheet" href="~/lib/bootstrap/dist/css/bootstrap.min.css" />
|
<link rel="stylesheet" href="~/lib/bootstrap/dist/css/bootstrap.min.css" />
|
||||||
<link rel="stylesheet" href="~/css/site.css" asp-append-version="true" />
|
<link rel="stylesheet" href="~/css/site.css" asp-append-version="true" />
|
||||||
|
<script>
|
||||||
|
window.appTranslations = {
|
||||||
|
closeAlert: '@T["Close alert"]'
|
||||||
|
};
|
||||||
|
</script>
|
||||||
</head>
|
</head>
|
||||||
<body>
|
<body>
|
||||||
<header>
|
<header>
|
||||||
|
|||||||
@@ -2,3 +2,50 @@
|
|||||||
// for details on configuring this project to bundle and minify static web assets.
|
// for details on configuring this project to bundle and minify static web assets.
|
||||||
|
|
||||||
// Write your JavaScript code.
|
// Write your JavaScript code.
|
||||||
|
function createToastContainer() {
|
||||||
|
const container = document.createElement('div');
|
||||||
|
container.id = 'toastContainer';
|
||||||
|
container.className = 'toast-container position-fixed bottom-0 end-0 p-3';
|
||||||
|
container.setAttribute("aria-live", "polite");
|
||||||
|
container.setAttribute("aria-atomic", "true");
|
||||||
|
|
||||||
|
const liveRegion = document.createElement('div');
|
||||||
|
liveRegion.id = 'toastLiveRegion';
|
||||||
|
liveRegion.className = 'visually-hidden';
|
||||||
|
liveRegion.setAttribute('aria-live', 'assertive');
|
||||||
|
liveRegion.setAttribute('aria-atomic', 'true');
|
||||||
|
container.appendChild(liveRegion);
|
||||||
|
|
||||||
|
document.body.appendChild(container);
|
||||||
|
return container;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Simple toast helper
|
||||||
|
function showToast(message, type) {
|
||||||
|
const toastContainer = document.getElementById('toastContainer') || createToastContainer();
|
||||||
|
const toast = document.createElement('div');
|
||||||
|
toast.className = `toast align-items-center text-white bg-${type} border-0`;
|
||||||
|
toast.role = 'alert';
|
||||||
|
var useDarkElements = type === "warning"
|
||||||
|
toast.innerHTML = `
|
||||||
|
<div class="d-flex">
|
||||||
|
<div class="toast-body">${message}</div>
|
||||||
|
<button type="button" class="btn-close${useDarkElements ? "" : " btn-close-white"} me-2 m-auto"${useDarkElements ? ' style="filter: unset;"' : ""} data-bs-dismiss="toast" aria-label="${window.appTranslations.closeAlert}"></button>
|
||||||
|
</div>
|
||||||
|
`;
|
||||||
|
if (useDarkElements) {
|
||||||
|
toast.classList.remove("text-white");
|
||||||
|
toast.classList.add("text-dark");
|
||||||
|
}
|
||||||
|
toastContainer.appendChild(toast);
|
||||||
|
|
||||||
|
const liveRegion = document.getElementById('toastLiveRegion');
|
||||||
|
if (liveRegion) {
|
||||||
|
liveRegion.textContent = '';
|
||||||
|
setTimeout(() => liveRegion.textContent = message, 500);
|
||||||
|
}
|
||||||
|
|
||||||
|
const bsToast = new bootstrap.Toast(toast, { delay: 10000 });
|
||||||
|
bsToast.show();
|
||||||
|
toast.addEventListener('hidden.bs.toast', () => toast.remove());
|
||||||
|
}
|
||||||
@@ -41,6 +41,8 @@ public class EntityListResult
|
|||||||
{
|
{
|
||||||
[JsonPropertyName("Name")]
|
[JsonPropertyName("Name")]
|
||||||
public required string Name { get; set; }
|
public required string Name { get; set; }
|
||||||
|
[JsonPropertyName("ProbMethod")]
|
||||||
|
public required string ProbMethod { get; set; }
|
||||||
[JsonPropertyName("Attributes")]
|
[JsonPropertyName("Attributes")]
|
||||||
public required List<AttributeResult> Attributes { get; set; }
|
public required List<AttributeResult> Attributes { get; set; }
|
||||||
[JsonPropertyName("Datapoints")]
|
[JsonPropertyName("Datapoints")]
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ public class JSONEntity
|
|||||||
public class JSONDatapoint
|
public class JSONDatapoint
|
||||||
{
|
{
|
||||||
public required string Name { get; set; }
|
public required string Name { get; set; }
|
||||||
public required string Text { get; set; }
|
public required string? Text { get; set; }
|
||||||
public required string Probmethod_embedding { get; set; }
|
public required string Probmethod_embedding { get; set; }
|
||||||
public required string SimilarityMethod { get; set; }
|
public required string SimilarityMethod { get; set; }
|
||||||
public required string[] Model { get; set; }
|
public required string[] Model { get; set; }
|
||||||
|
|||||||
@@ -2,12 +2,19 @@
|
|||||||
using System.Text.Json.Serialization;
|
using System.Text.Json.Serialization;
|
||||||
|
|
||||||
namespace Shared.Models;
|
namespace Shared.Models;
|
||||||
public readonly struct ResultItem(float score, string name)
|
public readonly struct ResultItem
|
||||||
{
|
{
|
||||||
[JsonPropertyName("Score")]
|
[JsonPropertyName("Score")]
|
||||||
public readonly float Score { get; } = score;
|
public readonly float Score { get; }
|
||||||
[JsonPropertyName("Name")]
|
[JsonPropertyName("Name")]
|
||||||
public readonly string Name { get; } = name;
|
public readonly string Name { get; }
|
||||||
|
|
||||||
|
[JsonConstructor]
|
||||||
|
public ResultItem(float score, string name)
|
||||||
|
{
|
||||||
|
Score = score;
|
||||||
|
Name = name;
|
||||||
|
}
|
||||||
|
|
||||||
public static long EstimateSize(ResultItem item)
|
public static long EstimateSize(ResultItem item)
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -55,6 +55,24 @@ public class SearchdomainSearchesResults
|
|||||||
public required Dictionary<string, DateTimedSearchResult> Searches { get; set; }
|
public required Dictionary<string, DateTimedSearchResult> Searches { get; set; }
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public class SearchdomainDeleteSearchResult
|
||||||
|
{
|
||||||
|
[JsonPropertyName("Success")]
|
||||||
|
public required bool Success { get; set; }
|
||||||
|
|
||||||
|
[JsonPropertyName("Message")]
|
||||||
|
public string? Message { get; set; }
|
||||||
|
}
|
||||||
|
|
||||||
|
public class SearchdomainUpdateSearchResult
|
||||||
|
{
|
||||||
|
[JsonPropertyName("Success")]
|
||||||
|
public required bool Success { get; set; }
|
||||||
|
|
||||||
|
[JsonPropertyName("Message")]
|
||||||
|
public string? Message { get; set; }
|
||||||
|
}
|
||||||
|
|
||||||
public class SearchdomainSettingsResults
|
public class SearchdomainSettingsResults
|
||||||
{
|
{
|
||||||
[JsonPropertyName("Success")]
|
[JsonPropertyName("Success")]
|
||||||
|
|||||||
Reference in New Issue
Block a user