Fixed Indexer SwaggerUI, Added worker & call management endpoints to Indexer, restructuring to improve project separation

src/Server/AIProvider.cs

@@ -8,7 +8,7 @@ using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
 using Server.Exceptions;
 
-namespace server;
+namespace Server;
 
 public class AIProvider
 {

src/Server/ApiKeyMiddleware.cs

@@ -1,37 +0,0 @@
-using Microsoft.Extensions.Primitives;
-
-namespace Server;
-
-public class ApiKeyMiddleware
-{
-    private readonly RequestDelegate _next;
-    private readonly IConfiguration _configuration;
-
-    public ApiKeyMiddleware(RequestDelegate next, IConfiguration configuration)
-    {
-        _next = next;
-        _configuration = configuration;
-    }
-
-    public async Task InvokeAsync(HttpContext context)
-    {
-        if (!context.Request.Headers.TryGetValue("X-API-KEY", out StringValues extractedApiKey))
-        {
-            context.Response.StatusCode = 401;
-            await context.Response.WriteAsync("API Key is missing.");
-            return;
-        }
-
-        var validApiKeys = _configuration.GetSection("Embeddingsearch").GetSection("ApiKeys").Get<List<string>>();
-#pragma warning disable CS8604
-        if (validApiKeys == null || !validApiKeys.Contains(extractedApiKey)) // CS8604 extractedApiKey is not null here, but the compiler still thinks that it might be.
-        {
-            context.Response.StatusCode = 403;
-            await context.Response.WriteAsync("Invalid API Key.");
-            return;
-        }
-#pragma warning restore CS8604
-
-        await _next(context);
-    }
-}

src/Server/Controllers/EntityController.cs

@@ -1,7 +1,8 @@
 using Microsoft.AspNetCore.Mvc;
 using System.Text.Json;
 using System.Text.Json.Nodes;
-using Server.Models;
+using Shared.Models;
+using Server.Helper;
 namespace Server.Controllers;
 
 [ApiController]

src/Server/Controllers/SearchdomainController.cs

@@ -1,6 +1,6 @@
 using ElmahCore;
 using Microsoft.AspNetCore.Mvc;
-using Server.Models;
+using Shared.Models;
 
 namespace Server.Controllers;
 

src/Server/Datapoint.cs

@@ -6,7 +6,6 @@ using System.Threading.Tasks;
 using Microsoft.Extensions.AI;
 using OllamaSharp;
 using OllamaSharp.Models;
-using server;
 
 namespace Server;
 

src/Server/Helper/SearchdomainHelper.cs

@@ -2,11 +2,9 @@ using System.Collections.Concurrent;
 using System.Security.Cryptography;
 using System.Text;
 using System.Text.Json;
-using MySql.Data.MySqlClient;
-using OllamaSharp;
-using server;
+using Shared.Models;
 
-namespace Server;
+namespace Server.Helper;
 
 public static class SearchdomainHelper
 {
@@ -41,7 +39,7 @@ public static class SearchdomainHelper
         }
         return null;
     }
-    
+
     public static List<Entity>? EntitiesFromJSON(List<Entity> entityCache, Dictionary<string, Dictionary<string, float[]>> embeddingCache, AIProvider aIProvider, SQLHelper helper, ILogger logger, string json)
     {
         List<JSONEntity>? jsonEntities = JsonSerializer.Deserialize<List<JSONEntity>>(json);
@@ -78,14 +76,14 @@ public static class SearchdomainHelper
         });
         return [.. retVal];
     }
-    
+
     public static Entity? EntityFromJSON(List<Entity> entityCache, Dictionary<string, Dictionary<string, float[]>> embeddingCache, AIProvider aIProvider, SQLHelper helper, ILogger logger, JSONEntity jsonEntity) //string json)
     {
         Dictionary<string, Dictionary<string, float[]>> embeddingsLUT = [];
         int? preexistingEntityID = DatabaseHelper.GetEntityID(helper, jsonEntity.Name, jsonEntity.Searchdomain);
         if (preexistingEntityID is not null)
         {
-            lock (helper.connection)
+            lock (helper.connection) // TODO change this to helper and do A/B tests (i.e. before/after)
             {
                 Dictionary<string, dynamic> parameters = new()
                 {

src/Server/JSONModels.cs

@@ -1,18 +0,0 @@
-namespace Server;
-
-public class JSONEntity
-{
-    public required string Name { get; set; }
-    public required string Probmethod { get; set; }
-    public required string Searchdomain { get; set; }
-    public required Dictionary<string, string> Attributes { get; set; }
-    public required JSONDatapoint[] Datapoints { get; set; }
-}
-
-public class JSONDatapoint
-{
-    public required string Name { get; set; }
-    public required string Text { get; set; }
-    public required string Probmethod_embedding { get; set; }
-    public required string[] Model { get; set; }
-}

src/Server/Models/EntityResults.cs

@@ -1,75 +0,0 @@
-using System.Text.Json.Serialization;
-
-namespace Server.Models;
-
-
-public class EntityQueryResults
-{
-    [JsonPropertyName("Results")]
-    public required List<EntityQueryResult> Results { get; set; }
-}
-
-public class EntityQueryResult
-{
-    [JsonPropertyName("Name")]
-    public required string Name { get; set; }
-    [JsonPropertyName("Value")]
-    public float Value { get; set; }
-}
-
-public class EntityIndexResult
-{
-    [JsonPropertyName("Success")]
-    public required bool Success { get; set; }
-}
-
-public class EntityListResults
-{
-    [JsonPropertyName("Results")]
-    public required List<EntityListResult> Results { get; set; }
-    [JsonPropertyName("Success")]
-    public required bool Success { get; set; }
-}
-
-public class EntityListResult
-{
-    [JsonPropertyName("Name")]
-    public required string Name { get; set; }
-    [JsonPropertyName("Attributes")]
-    public required List<AttributeResult> Attributes { get; set; }
-    [JsonPropertyName("Datapoints")]
-    public required List<DatapointResult> Datapoints { get; set; }
-}
-
-public class AttributeResult
-{
-    [JsonPropertyName("Name")]
-    public required string Name { get; set; }
-    [JsonPropertyName("Value")]
-    public required string Value { get; set; }
-}
-
-public class DatapointResult
-{
-    [JsonPropertyName("Name")]
-    public required string Name { get; set; }
-    [JsonPropertyName("ProbMethod")]
-    public required string ProbMethod { get; set; }
-    [JsonPropertyName("Embeddings")]
-    public required List<EmbeddingResult>? Embeddings { get; set; } 
-}
-
-public class EmbeddingResult
-{
-    [JsonPropertyName("Model")]
-    public required string Model { get; set; }
-    [JsonPropertyName("Embeddings")]
-    public required float[] Embeddings { get; set; }
-}
-
-public class EntityDeleteResults
-{
-    [JsonPropertyName("Success")]
-    public required bool Success { get; set; }
-}
-

src/Server/Models/SearchdomainResults.cs

@@ -1,32 +0,0 @@
-using System.Text.Json.Serialization;
-
-namespace Server.Models;
-
-public class SearchdomainListResults
-{
-    [JsonPropertyName("Searchdomains")] // Otherwise the api returns {"searchdomains": [...]} and the client requires {"Searchdomains": [...]}
-    public required List<string> Searchdomains { get; set; }
-}
-
-public class SearchdomainCreateResults
-{
-    [JsonPropertyName("Success")]
-    public required bool Success { get; set; }
-
-    [JsonPropertyName("Id")]
-    public int? Id { get; set; }
-}
-
-public class SearchdomainUpdateResults
-{
-    [JsonPropertyName("Success")]
-    public required bool Success { get; set; }
-}
-
-public class SearchdomainDeleteResults
-{
-    [JsonPropertyName("Success")]
-    public required bool Success { get; set; }
-    [JsonPropertyName("DeletedEntities")]
-    public required int DeletedEntities { get; set; }
-}

src/Server/Program.cs

@@ -1,7 +1,6 @@
 using ElmahCore;
 using ElmahCore.Mvc;
 using Serilog;
-using server;
 using Server;
 using Server.HealthChecks;
 
@@ -72,7 +71,7 @@ if (IsDevelopment || useSwagger)
 }
 if (UseMiddleware == true && !IsDevelopment)
 {
-    app.UseMiddleware<ApiKeyMiddleware>();
+    app.UseMiddleware<Shared.ApiKeyMiddleware>();
 }
 
 app.UseAuthorization();

src/Server/Searchdomain.cs

@@ -17,11 +17,10 @@ using Mysqlx.Resultset;
 using System.Collections.Immutable;
 using System.Text.Json;
 using System.Numerics.Tensors;
-using Server;
 using System.Security.Cryptography;
 using System.Text;
 using System.Collections.Concurrent;
-using server;
+using Server.Helper;
 
 namespace Server;
 

src/Server/SearchdomainManager.cs

@@ -4,7 +4,6 @@ using OllamaSharp;
 using Microsoft.IdentityModel.Tokens;
 using Server.Exceptions;
 using Server.Migrations;
-using server;
 
 namespace Server;
 

src/Server/Server.csproj

@@ -22,4 +22,8 @@
       <PackageReference Include="System.Data.Sqlite" Version="1.0.119" />
       <PackageReference Include="System.Numerics.Tensors" Version="9.0.3" />
   </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Shared\Shared.csproj" />
+  </ItemGroup>
 </Project>

src/Server/appsettings.json

@@ -13,7 +13,7 @@
       { "Name": "File", "Args": { "path": "logs/log.txt", "rollingInterval": "Day", "retainedFileCountLimit": 7 } }
     ],
     "Properties": {
-        "Application": "Indexer"
+        "Application": "Embeddingsearch.Server"
     }
   },
   "EmbeddingsearchIndexer": {