Compare commits
12 Commits
b5545bb5c8
...
41-create-
| Author | SHA1 | Date | |
|---|---|---|---|
| e64f4ca555 | |||
| 55a475704e | |||
|
|
89f8aa6591 | ||
| a3cc5115fc | |||
| 8b36c65437 | |||
| ee12986fef | |||
| fd76da265b | |||
| 4aa3015954 | |||
| fb1766b2b5 | |||
| 6fe96d41a2 | |||
| fc5e8ceeee | |||
| 33502640a2 |
@@ -55,10 +55,7 @@ public class EntityController : ControllerBase
|
||||
try
|
||||
{
|
||||
List<Entity>? entities = _searchdomainHelper.EntitiesFromJSON(
|
||||
[],
|
||||
_domainManager.embeddingCache,
|
||||
_domainManager.aIProvider,
|
||||
_domainManager.helper,
|
||||
_domainManager,
|
||||
_logger,
|
||||
JsonSerializer.Serialize(jsonEntities));
|
||||
if (entities is not null && jsonEntities is not null)
|
||||
@@ -92,8 +89,13 @@ public class EntityController : ControllerBase
|
||||
}
|
||||
|
||||
[HttpGet("List")]
|
||||
public ActionResult<EntityListResults> List(string searchdomain, bool returnEmbeddings = false)
|
||||
public ActionResult<EntityListResults> List(string searchdomain, bool returnModels = false, bool returnEmbeddings = false)
|
||||
{
|
||||
if (returnEmbeddings && !returnModels)
|
||||
{
|
||||
_logger.LogError("Invalid request for {searchdomain} - embeddings return requested but without models - not possible!", [searchdomain]);
|
||||
return Ok(new EntityQueryResults() {Results = [], Success = false, Message = "Invalid request" });
|
||||
}
|
||||
Searchdomain searchdomain_;
|
||||
try
|
||||
{
|
||||
@@ -118,12 +120,12 @@ public class EntityController : ControllerBase
|
||||
List<DatapointResult> datapointResults = [];
|
||||
foreach (Datapoint datapoint in entity.datapoints)
|
||||
{
|
||||
if (returnEmbeddings)
|
||||
if (returnModels)
|
||||
{
|
||||
List<EmbeddingResult> embeddingResults = [];
|
||||
foreach ((string, float[]) embedding in datapoint.embeddings)
|
||||
{
|
||||
embeddingResults.Add(new EmbeddingResult() {Model = embedding.Item1, Embeddings = embedding.Item2});
|
||||
embeddingResults.Add(new EmbeddingResult() {Model = embedding.Item1, Embeddings = returnEmbeddings ? embedding.Item2 : []});
|
||||
}
|
||||
datapointResults.Add(new DatapointResult() {Name = datapoint.name, ProbMethod = datapoint.probMethod.name, SimilarityMethod = datapoint.similarityMethod.name, Embeddings = embeddingResults});
|
||||
}
|
||||
@@ -135,6 +137,7 @@ public class EntityController : ControllerBase
|
||||
EntityListResult entityListResult = new()
|
||||
{
|
||||
Name = entity.name,
|
||||
ProbMethod = entity.probMethodName,
|
||||
Attributes = attributeResults,
|
||||
Datapoints = datapointResults
|
||||
};
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
using System.Text.Json;
|
||||
using ElmahCore;
|
||||
using Microsoft.AspNetCore.Http.HttpResults;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Server.Exceptions;
|
||||
using Server.Helper;
|
||||
@@ -156,6 +157,64 @@ public class SearchdomainController : ControllerBase
|
||||
return Ok(new SearchdomainSearchesResults() { Searches = searchCache, Success = true });
|
||||
}
|
||||
|
||||
[HttpDelete("Searches")]
|
||||
public ActionResult<SearchdomainDeleteSearchResult> DeleteSearch(string searchdomain, string query)
|
||||
{
|
||||
Searchdomain searchdomain_;
|
||||
try
|
||||
{
|
||||
searchdomain_ = _domainManager.GetSearchdomain(searchdomain);
|
||||
}
|
||||
catch (SearchdomainNotFoundException)
|
||||
{
|
||||
_logger.LogError("Unable to retrieve the searchdomain {searchdomain} - it likely does not exist yet", [searchdomain]);
|
||||
return Ok(new SearchdomainDeleteSearchResult() { Success = false, Message = "Searchdomain not found" });
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError("Unable to retrieve the searchdomain {searchdomain} - {ex.Message} - {ex.StackTrace}", [searchdomain, ex.Message, ex.StackTrace]);
|
||||
return Ok(new SearchdomainDeleteSearchResult() { Success = false, Message = ex.Message });
|
||||
}
|
||||
Dictionary<string, DateTimedSearchResult> searchCache = searchdomain_.searchCache;
|
||||
bool containsKey = searchCache.ContainsKey(query);
|
||||
if (containsKey)
|
||||
{
|
||||
searchCache.Remove(query);
|
||||
return Ok(new SearchdomainDeleteSearchResult() {Success = true});
|
||||
}
|
||||
return Ok(new SearchdomainDeleteSearchResult() {Success = false, Message = "Query not found in search cache"});
|
||||
}
|
||||
|
||||
[HttpPatch("Searches")]
|
||||
public ActionResult<SearchdomainUpdateSearchResult> UpdateSearch(string searchdomain, string query, [FromBody]List<ResultItem> results)
|
||||
{
|
||||
Searchdomain searchdomain_;
|
||||
try
|
||||
{
|
||||
searchdomain_ = _domainManager.GetSearchdomain(searchdomain);
|
||||
}
|
||||
catch (SearchdomainNotFoundException)
|
||||
{
|
||||
_logger.LogError("Unable to retrieve the searchdomain {searchdomain} - it likely does not exist yet", [searchdomain]);
|
||||
return Ok(new SearchdomainUpdateSearchResult() { Success = false, Message = "Searchdomain not found" });
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError("Unable to retrieve the searchdomain {searchdomain} - {ex.Message} - {ex.StackTrace}", [searchdomain, ex.Message, ex.StackTrace]);
|
||||
return Ok(new SearchdomainUpdateSearchResult() { Success = false, Message = ex.Message });
|
||||
}
|
||||
Dictionary<string, DateTimedSearchResult> searchCache = searchdomain_.searchCache;
|
||||
bool containsKey = searchCache.ContainsKey(query);
|
||||
if (containsKey)
|
||||
{
|
||||
DateTimedSearchResult element = searchCache[query];
|
||||
element.Results = results;
|
||||
searchCache[query] = element;
|
||||
return Ok(new SearchdomainUpdateSearchResult() {Success = true});
|
||||
}
|
||||
return Ok(new SearchdomainUpdateSearchResult() {Success = false, Message = "Query not found in search cache"});
|
||||
}
|
||||
|
||||
[HttpGet("GetSettings")]
|
||||
public ActionResult<SearchdomainSettingsResults> GetSettings(string searchdomain)
|
||||
{
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
namespace Server;
|
||||
|
||||
public class Entity(Dictionary<string, string> attributes, Probmethods.probMethodDelegate probMethod, List<Datapoint> datapoints, string name)
|
||||
public class Entity(Dictionary<string, string> attributes, Probmethods.probMethodDelegate probMethod, string probMethodName, List<Datapoint> datapoints, string name)
|
||||
{
|
||||
public Dictionary<string, string> attributes = attributes;
|
||||
public Probmethods.probMethodDelegate probMethod = probMethod;
|
||||
public string probMethodName = probMethodName;
|
||||
public List<Datapoint> datapoints = datapoints;
|
||||
public int id;
|
||||
public string name = name;
|
||||
|
||||
@@ -44,8 +44,12 @@ public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelp
|
||||
return null;
|
||||
}
|
||||
|
||||
public List<Entity>? EntitiesFromJSON(List<Entity> entityCache, Dictionary<string, Dictionary<string, float[]>> embeddingCache, AIProvider aIProvider, SQLHelper helper, ILogger logger, string json)
|
||||
public List<Entity>? EntitiesFromJSON(SearchdomainManager searchdomainManager, ILogger logger, string json)
|
||||
{
|
||||
Dictionary<string, Dictionary<string, float[]>> embeddingCache = searchdomainManager.embeddingCache;
|
||||
AIProvider aIProvider = searchdomainManager.aIProvider;
|
||||
SQLHelper helper = searchdomainManager.helper;
|
||||
|
||||
List<JSONEntity>? jsonEntities = JsonSerializer.Deserialize<List<JSONEntity>>(json);
|
||||
if (jsonEntities is null)
|
||||
{
|
||||
@@ -72,8 +76,7 @@ public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelp
|
||||
ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = 16 }; // <-- This is needed! Otherwise if we try to index 100+ entities at once, it spawns 100 threads, exploding the SQL pool
|
||||
Parallel.ForEach(jsonEntities, parallelOptions, jSONEntity =>
|
||||
{
|
||||
using var tempHelper = helper.DuplicateConnection();
|
||||
var entity = EntityFromJSON(entityCache, embeddingCache, aIProvider, tempHelper, logger, jSONEntity);
|
||||
var entity = EntityFromJSON(searchdomainManager, logger, jSONEntity);
|
||||
if (entity is not null)
|
||||
{
|
||||
retVal.Enqueue(entity);
|
||||
@@ -82,37 +85,133 @@ public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelp
|
||||
return [.. retVal];
|
||||
}
|
||||
|
||||
public Entity? EntityFromJSON(List<Entity> entityCache, Dictionary<string, Dictionary<string, float[]>> embeddingCache, AIProvider aIProvider, SQLHelper helper, ILogger logger, JSONEntity jsonEntity) //string json)
|
||||
public Entity? EntityFromJSON(SearchdomainManager searchdomainManager, ILogger logger, JSONEntity jsonEntity) //string json)
|
||||
{
|
||||
SQLHelper helper = searchdomainManager.helper.DuplicateConnection();
|
||||
Searchdomain searchdomain = searchdomainManager.GetSearchdomain(jsonEntity.Searchdomain);
|
||||
List<Entity> entityCache = searchdomain.entityCache;
|
||||
AIProvider aIProvider = searchdomain.aIProvider;
|
||||
Dictionary<string, Dictionary<string, float[]>> embeddingCache = searchdomain.embeddingCache;
|
||||
Entity? preexistingEntity = entityCache.FirstOrDefault(entity => entity.name == jsonEntity.Name);
|
||||
|
||||
if (preexistingEntity is not null)
|
||||
{
|
||||
Dictionary<string, Dictionary<string, float[]>> embeddingsLUT = []; // embeddingsLUT: hash -> model -> [embeddingValues * n]
|
||||
int? preexistingEntityID = _databaseHelper.GetEntityID(helper, jsonEntity.Name, jsonEntity.Searchdomain);
|
||||
if (preexistingEntityID is not null)
|
||||
if (preexistingEntityID is null)
|
||||
{
|
||||
lock (helper.connection) // TODO change this to helper and do A/B tests (i.e. before/after)
|
||||
_logger.LogCritical("Unable to index entity {jsonEntity.Name} because it already exists in the searchdomain but not in the database.", [jsonEntity.Name]);
|
||||
throw new Exception($"Unable to index entity {jsonEntity.Name} because it already exists in the searchdomain but not in the database.");
|
||||
}
|
||||
Dictionary<string, string> attributes = jsonEntity.Attributes;
|
||||
|
||||
// Attribute
|
||||
foreach (KeyValuePair<string, string> attributesKV in preexistingEntity.attributes.ToList())
|
||||
{
|
||||
string oldAttributeKey = attributesKV.Key;
|
||||
string oldAttribute = attributesKV.Value;
|
||||
bool newHasAttribute = jsonEntity.Attributes.TryGetValue(oldAttributeKey, out string? newAttribute);
|
||||
if (newHasAttribute && newAttribute is not null && newAttribute != oldAttribute)
|
||||
{
|
||||
// Attribute - Updated
|
||||
Dictionary<string, dynamic> parameters = new()
|
||||
{
|
||||
{ "id", preexistingEntityID }
|
||||
{ "newValue", newAttribute },
|
||||
{ "entityId", preexistingEntityID },
|
||||
{ "attribute", oldAttributeKey}
|
||||
};
|
||||
System.Data.Common.DbDataReader reader = helper.ExecuteSQLCommand("SELECT e.model, e.embedding, d.hash FROM datapoint d JOIN embedding e ON d.id = e.id_datapoint WHERE d.id_entity = @id", parameters);
|
||||
while (reader.Read())
|
||||
helper.ExecuteSQLNonQuery("UPDATE attribute SET value=@newValue WHERE id_entity=@entityId AND attribute=@attribute", parameters);
|
||||
preexistingEntity.attributes[oldAttributeKey] = newAttribute;
|
||||
} else if (!newHasAttribute)
|
||||
{
|
||||
string model = reader.GetString(0);
|
||||
long length = reader.GetBytes(1, 0, null, 0, 0);
|
||||
byte[] embeddingBytes = new byte[length];
|
||||
reader.GetBytes(1, 0, embeddingBytes, 0, (int)length);
|
||||
float[] embeddingValues = FloatArrayFromBytes(embeddingBytes);
|
||||
string hash = reader.GetString(2);
|
||||
if (!embeddingsLUT.ContainsKey(hash))
|
||||
// Attribute - Deleted
|
||||
Dictionary<string, dynamic> parameters = new()
|
||||
{
|
||||
embeddingsLUT[hash] = [];
|
||||
{ "entityId", preexistingEntityID },
|
||||
{ "attribute", oldAttributeKey}
|
||||
};
|
||||
helper.ExecuteSQLNonQuery("DELETE FROM attribute WHERE id_entity=@entityId AND attribute=@attribute", parameters);
|
||||
preexistingEntity.attributes.Remove(oldAttributeKey);
|
||||
}
|
||||
embeddingsLUT[hash].TryAdd(model, embeddingValues);
|
||||
}
|
||||
reader.Close();
|
||||
foreach (var attributesKV in jsonEntity.Attributes)
|
||||
{
|
||||
string newAttributeKey = attributesKV.Key;
|
||||
string newAttribute = attributesKV.Value;
|
||||
bool preexistingHasAttribute = preexistingEntity.attributes.TryGetValue(newAttributeKey, out string? preexistingAttribute);
|
||||
if (!preexistingHasAttribute)
|
||||
{
|
||||
// Attribute - New
|
||||
DatabaseHelper.DatabaseInsertAttribute(helper, newAttributeKey, newAttribute, (int)preexistingEntityID);
|
||||
preexistingEntity.attributes.Add(newAttributeKey, newAttribute);
|
||||
}
|
||||
_databaseHelper.RemoveEntity(entityCache, helper, jsonEntity.Name, jsonEntity.Searchdomain); // TODO only remove entity if there is actually a change somewhere. Perhaps create 3 datapoint lists to operate with: 1. delete, 2. update, 3. create
|
||||
}
|
||||
|
||||
// Datapoint
|
||||
foreach (Datapoint datapoint in preexistingEntity.datapoints.ToList())
|
||||
{
|
||||
bool newEntityHasDatapoint = jsonEntity.Datapoints.Any(x => x.Name == datapoint.name);
|
||||
if (!newEntityHasDatapoint)
|
||||
{
|
||||
// Datapoint - Deleted
|
||||
Dictionary<string, dynamic> parameters = new()
|
||||
{
|
||||
{ "datapointName", datapoint.name },
|
||||
{ "entityId", preexistingEntityID}
|
||||
};
|
||||
helper.ExecuteSQLNonQuery("DELETE e FROM embedding e JOIN datapoint d ON e.id_datapoint=d.id WHERE d.name=@datapointName AND d.id_entity=@entityId", parameters);
|
||||
helper.ExecuteSQLNonQuery("DELETE FROM datapoint WHERE id_entity=@entityId AND name=@datapointName", parameters);
|
||||
preexistingEntity.datapoints.Remove(datapoint);
|
||||
} else
|
||||
{
|
||||
JSONDatapoint? newEntityDatapoint = jsonEntity.Datapoints.FirstOrDefault(x => x.Name == datapoint.name);
|
||||
if (newEntityDatapoint is not null && newEntityDatapoint.Text is not null)
|
||||
{
|
||||
// Datapoint - Updated (text)
|
||||
Dictionary<string, dynamic> parameters = new()
|
||||
{
|
||||
{ "datapointName", datapoint.name },
|
||||
{ "entityId", preexistingEntityID}
|
||||
};
|
||||
helper.ExecuteSQLNonQuery("DELETE e FROM embedding e JOIN datapoint d ON e.id_datapoint=d.id WHERE d.name=@datapointName AND d.id_entity=@entityId", parameters);
|
||||
helper.ExecuteSQLNonQuery("DELETE FROM datapoint WHERE id_entity=@entityId AND name=@datapointName", parameters);
|
||||
preexistingEntity.datapoints.Remove(datapoint);
|
||||
Datapoint newDatapoint = DatabaseInsertDatapointWithEmbeddings(helper, searchdomain, newEntityDatapoint, (int)preexistingEntityID);
|
||||
preexistingEntity.datapoints.Add(newDatapoint);
|
||||
|
||||
}
|
||||
if (newEntityDatapoint is not null && (newEntityDatapoint.Probmethod_embedding != datapoint.probMethod.name || newEntityDatapoint.SimilarityMethod != datapoint.similarityMethod.name))
|
||||
{
|
||||
// Datapoint - Updated (probmethod or similaritymethod)
|
||||
Dictionary<string, dynamic> parameters = new()
|
||||
{
|
||||
{ "probmethod", newEntityDatapoint.Probmethod_embedding },
|
||||
{ "similaritymethod", newEntityDatapoint.SimilarityMethod },
|
||||
{ "datapointName", datapoint.name },
|
||||
{ "entityId", preexistingEntityID}
|
||||
};
|
||||
helper.ExecuteSQLNonQuery("UPDATE datapoint SET probmethod_embedding=@probmethod, similaritymethod=@similaritymethod WHERE id_entity=@entityId AND name=@datapointName", parameters);
|
||||
Datapoint preexistingDatapoint = preexistingEntity.datapoints.First(x => x == datapoint); // The for loop is a copy. This retrieves the original such that it can be updated.
|
||||
preexistingDatapoint.probMethod = datapoint.probMethod;
|
||||
preexistingDatapoint.similarityMethod = datapoint.similarityMethod;
|
||||
}
|
||||
}
|
||||
}
|
||||
foreach (JSONDatapoint jsonDatapoint in jsonEntity.Datapoints)
|
||||
{
|
||||
bool oldEntityHasDatapoint = preexistingEntity.datapoints.Any(x => x.name == jsonDatapoint.Name);
|
||||
if (!oldEntityHasDatapoint)
|
||||
{
|
||||
// Datapoint - New
|
||||
Datapoint datapoint = DatabaseInsertDatapointWithEmbeddings(helper, searchdomain, jsonDatapoint, (int)preexistingEntityID);
|
||||
preexistingEntity.datapoints.Add(datapoint);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
return preexistingEntity;
|
||||
}
|
||||
else
|
||||
{
|
||||
int id_entity = DatabaseHelper.DatabaseInsertEntity(helper, jsonEntity.Name, jsonEntity.Probmethod, _databaseHelper.GetSearchdomainID(helper, jsonEntity.Searchdomain));
|
||||
foreach (KeyValuePair<string, string> attribute in jsonEntity.Attributes)
|
||||
{
|
||||
@@ -123,30 +222,28 @@ public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelp
|
||||
foreach (JSONDatapoint jsonDatapoint in jsonEntity.Datapoints)
|
||||
{
|
||||
string hash = Convert.ToBase64String(SHA256.HashData(Encoding.UTF8.GetBytes(jsonDatapoint.Text)));
|
||||
Dictionary<string, float[]> embeddings = [];
|
||||
if (embeddingsLUT.ContainsKey(hash))
|
||||
{
|
||||
Dictionary<string, float[]> hashLUT = embeddingsLUT[hash];
|
||||
foreach (string model in jsonDatapoint.Model)
|
||||
{
|
||||
if (hashLUT.ContainsKey(model))
|
||||
{
|
||||
embeddings.Add(model, hashLUT[model]);
|
||||
Datapoint datapoint = DatabaseInsertDatapointWithEmbeddings(helper, searchdomain, jsonDatapoint, id_entity, hash);
|
||||
datapoints.Add(datapoint);
|
||||
}
|
||||
else
|
||||
|
||||
var probMethod = Probmethods.GetMethod(jsonEntity.Probmethod) ?? throw new ProbMethodNotFoundException(jsonEntity.Probmethod);
|
||||
Entity entity = new(jsonEntity.Attributes, probMethod, jsonEntity.Probmethod, datapoints, jsonEntity.Name)
|
||||
{
|
||||
var additionalEmbeddings = Datapoint.GenerateEmbeddings(jsonDatapoint.Text, [model], aIProvider, embeddingCache);
|
||||
embeddings.Add(model, additionalEmbeddings.First().Value);
|
||||
id = id_entity
|
||||
};
|
||||
entityCache.Add(entity);
|
||||
return entity;
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
|
||||
public Datapoint DatabaseInsertDatapointWithEmbeddings(SQLHelper helper, Searchdomain searchdomain, JSONDatapoint jsonDatapoint, int id_entity, string? hash = null)
|
||||
{
|
||||
embeddings = Datapoint.GenerateEmbeddings(jsonDatapoint.Text, [.. jsonDatapoint.Model], aIProvider, embeddingCache);
|
||||
if (jsonDatapoint.Text is null)
|
||||
{
|
||||
throw new Exception("jsonDatapoint.Text must not be null at this point");
|
||||
}
|
||||
var probMethod_embedding = new ProbMethod(jsonDatapoint.Probmethod_embedding, logger) ?? throw new ProbMethodNotFoundException(jsonDatapoint.Probmethod_embedding);
|
||||
var similarityMethod = new SimilarityMethod(jsonDatapoint.SimilarityMethod, logger) ?? throw new SimilarityMethodNotFoundException(jsonDatapoint.SimilarityMethod);
|
||||
Datapoint datapoint = new(jsonDatapoint.Name, probMethod_embedding, similarityMethod, hash, [.. embeddings.Select(kv => (kv.Key, kv.Value))]);
|
||||
hash ??= Convert.ToBase64String(SHA256.HashData(Encoding.UTF8.GetBytes(jsonDatapoint.Text)));
|
||||
Datapoint datapoint = BuildDatapointFromJsonDatapoint(jsonDatapoint, id_entity, searchdomain, hash);
|
||||
int id_datapoint = DatabaseHelper.DatabaseInsertDatapoint(helper, jsonDatapoint.Name, jsonDatapoint.Probmethod_embedding, jsonDatapoint.SimilarityMethod, hash, id_entity); // TODO make this a bulk add action to reduce number of queries
|
||||
List<(string model, byte[] embedding)> data = [];
|
||||
foreach ((string, float[]) embedding in datapoint.embeddings)
|
||||
@@ -154,15 +251,22 @@ public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelp
|
||||
data.Add((embedding.Item1, BytesFromFloatArray(embedding.Item2)));
|
||||
}
|
||||
DatabaseHelper.DatabaseInsertEmbeddingBulk(helper, id_datapoint, data);
|
||||
datapoints.Add(datapoint);
|
||||
return datapoint;
|
||||
}
|
||||
|
||||
var probMethod = Probmethods.GetMethod(jsonEntity.Probmethod) ?? throw new ProbMethodNotFoundException(jsonEntity.Probmethod);
|
||||
Entity entity = new(jsonEntity.Attributes, probMethod, datapoints, jsonEntity.Name)
|
||||
public Datapoint BuildDatapointFromJsonDatapoint(JSONDatapoint jsonDatapoint, int entityId, Searchdomain searchdomain, string? hash = null)
|
||||
{
|
||||
id = id_entity
|
||||
};
|
||||
entityCache.Add(entity);
|
||||
return entity;
|
||||
if (jsonDatapoint.Text is null)
|
||||
{
|
||||
throw new Exception("jsonDatapoint.Text must not be null at this point");
|
||||
}
|
||||
using SQLHelper helper = searchdomain.helper.DuplicateConnection();
|
||||
Dictionary<string, Dictionary<string, float[]>> embeddingCache = searchdomain.embeddingCache;
|
||||
hash ??= Convert.ToBase64String(SHA256.HashData(Encoding.UTF8.GetBytes(jsonDatapoint.Text)));
|
||||
DatabaseHelper.DatabaseInsertDatapoint(helper, jsonDatapoint.Name, jsonDatapoint.Probmethod_embedding, jsonDatapoint.SimilarityMethod, hash, entityId);
|
||||
Dictionary<string, float[]> embeddings = Datapoint.GenerateEmbeddings(jsonDatapoint.Text, [.. jsonDatapoint.Model], searchdomain.aIProvider, embeddingCache);
|
||||
var probMethod_embedding = new ProbMethod(jsonDatapoint.Probmethod_embedding, logger) ?? throw new ProbMethodNotFoundException(jsonDatapoint.Probmethod_embedding);
|
||||
var similarityMethod = new SimilarityMethod(jsonDatapoint.SimilarityMethod, logger) ?? throw new SimilarityMethodNotFoundException(jsonDatapoint.SimilarityMethod);
|
||||
return new Datapoint(jsonDatapoint.Name, probMethod_embedding, similarityMethod, hash, [.. embeddings.Select(kv => (kv.Key, kv.Value))]);
|
||||
}
|
||||
}
|
||||
@@ -142,7 +142,7 @@ public class Searchdomain
|
||||
Probmethods.probMethodDelegate? probmethod = Probmethods.GetMethod(probmethodString);
|
||||
if (datapoint_unassigned.TryGetValue(id, out List<Datapoint>? datapoints) && probmethod is not null)
|
||||
{
|
||||
Entity entity = new(attributes, probmethod, datapoints, name)
|
||||
Entity entity = new(attributes, probmethod, probmethodString, datapoints, name)
|
||||
{
|
||||
id = id
|
||||
};
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,4 +1,7 @@
|
||||
<!DOCTYPE html>
|
||||
@using Server.Services
|
||||
@inject LocalizationService T
|
||||
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
@@ -6,6 +9,11 @@
|
||||
<title>@ViewData["Title"] - embeddingsearch</title>
|
||||
<link rel="stylesheet" href="~/lib/bootstrap/dist/css/bootstrap.min.css" />
|
||||
<link rel="stylesheet" href="~/css/site.css" asp-append-version="true" />
|
||||
<script>
|
||||
window.appTranslations = {
|
||||
closeAlert: '@T["Close alert"]'
|
||||
};
|
||||
</script>
|
||||
</head>
|
||||
<body>
|
||||
<header>
|
||||
|
||||
@@ -2,3 +2,50 @@
|
||||
// for details on configuring this project to bundle and minify static web assets.
|
||||
|
||||
// Write your JavaScript code.
|
||||
function createToastContainer() {
|
||||
const container = document.createElement('div');
|
||||
container.id = 'toastContainer';
|
||||
container.className = 'toast-container position-fixed bottom-0 end-0 p-3';
|
||||
container.setAttribute("aria-live", "polite");
|
||||
container.setAttribute("aria-atomic", "true");
|
||||
|
||||
const liveRegion = document.createElement('div');
|
||||
liveRegion.id = 'toastLiveRegion';
|
||||
liveRegion.className = 'visually-hidden';
|
||||
liveRegion.setAttribute('aria-live', 'assertive');
|
||||
liveRegion.setAttribute('aria-atomic', 'true');
|
||||
container.appendChild(liveRegion);
|
||||
|
||||
document.body.appendChild(container);
|
||||
return container;
|
||||
}
|
||||
|
||||
// Simple toast helper
|
||||
function showToast(message, type) {
|
||||
const toastContainer = document.getElementById('toastContainer') || createToastContainer();
|
||||
const toast = document.createElement('div');
|
||||
toast.className = `toast align-items-center text-white bg-${type} border-0`;
|
||||
toast.role = 'alert';
|
||||
var useDarkElements = type === "warning"
|
||||
toast.innerHTML = `
|
||||
<div class="d-flex">
|
||||
<div class="toast-body">${message}</div>
|
||||
<button type="button" class="btn-close${useDarkElements ? "" : " btn-close-white"} me-2 m-auto"${useDarkElements ? ' style="filter: unset;"' : ""} data-bs-dismiss="toast" aria-label="${window.appTranslations.closeAlert}"></button>
|
||||
</div>
|
||||
`;
|
||||
if (useDarkElements) {
|
||||
toast.classList.remove("text-white");
|
||||
toast.classList.add("text-dark");
|
||||
}
|
||||
toastContainer.appendChild(toast);
|
||||
|
||||
const liveRegion = document.getElementById('toastLiveRegion');
|
||||
if (liveRegion) {
|
||||
liveRegion.textContent = '';
|
||||
setTimeout(() => liveRegion.textContent = message, 500);
|
||||
}
|
||||
|
||||
const bsToast = new bootstrap.Toast(toast, { delay: 10000 });
|
||||
bsToast.show();
|
||||
toast.addEventListener('hidden.bs.toast', () => toast.remove());
|
||||
}
|
||||
@@ -41,6 +41,8 @@ public class EntityListResult
|
||||
{
|
||||
[JsonPropertyName("Name")]
|
||||
public required string Name { get; set; }
|
||||
[JsonPropertyName("ProbMethod")]
|
||||
public required string ProbMethod { get; set; }
|
||||
[JsonPropertyName("Attributes")]
|
||||
public required List<AttributeResult> Attributes { get; set; }
|
||||
[JsonPropertyName("Datapoints")]
|
||||
|
||||
@@ -12,7 +12,7 @@ public class JSONEntity
|
||||
public class JSONDatapoint
|
||||
{
|
||||
public required string Name { get; set; }
|
||||
public required string Text { get; set; }
|
||||
public required string? Text { get; set; }
|
||||
public required string Probmethod_embedding { get; set; }
|
||||
public required string SimilarityMethod { get; set; }
|
||||
public required string[] Model { get; set; }
|
||||
|
||||
@@ -2,12 +2,19 @@
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace Shared.Models;
|
||||
public readonly struct ResultItem(float score, string name)
|
||||
public readonly struct ResultItem
|
||||
{
|
||||
[JsonPropertyName("Score")]
|
||||
public readonly float Score { get; } = score;
|
||||
public readonly float Score { get; }
|
||||
[JsonPropertyName("Name")]
|
||||
public readonly string Name { get; } = name;
|
||||
public readonly string Name { get; }
|
||||
|
||||
[JsonConstructor]
|
||||
public ResultItem(float score, string name)
|
||||
{
|
||||
Score = score;
|
||||
Name = name;
|
||||
}
|
||||
|
||||
public static long EstimateSize(ResultItem item)
|
||||
{
|
||||
|
||||
@@ -55,6 +55,24 @@ public class SearchdomainSearchesResults
|
||||
public required Dictionary<string, DateTimedSearchResult> Searches { get; set; }
|
||||
}
|
||||
|
||||
public class SearchdomainDeleteSearchResult
|
||||
{
|
||||
[JsonPropertyName("Success")]
|
||||
public required bool Success { get; set; }
|
||||
|
||||
[JsonPropertyName("Message")]
|
||||
public string? Message { get; set; }
|
||||
}
|
||||
|
||||
public class SearchdomainUpdateSearchResult
|
||||
{
|
||||
[JsonPropertyName("Success")]
|
||||
public required bool Success { get; set; }
|
||||
|
||||
[JsonPropertyName("Message")]
|
||||
public string? Message { get; set; }
|
||||
}
|
||||
|
||||
public class SearchdomainSettingsResults
|
||||
{
|
||||
[JsonPropertyName("Success")]
|
||||
|
||||
Reference in New Issue
Block a user