lucretia/embeddingsearch
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixed Indexer SwaggerUI, Added worker & call management endpoints to Indexer, restructuring to improve project separation

Browse Source
This commit is contained in:
LD-Reborn
2025-07-12 22:55:56 +02:00
parent a884a2734d
commit e8e5b742d1
30 changed files with 558 additions and 189 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
.gitignore vendored
View File

@@ -14,4 +14,6 @@ src/Indexer/bin
src/Indexer/obj
src/Indexer/Scripts/__pycache__
src/Indexer/logs
src/Server/logs
src/Server/logs
src/Shared/bin
src/Shared/obj

4
src/Client/Client.cs
View File

@@ -7,7 +7,7 @@ using System.Text.Json;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Configuration;
using System.Reflection.Metadata.Ecma335;
using Server.Models;
using Shared.Models;
namespace Client;
@@ -86,7 +86,7 @@ public class Client
return await GetUrlAndProcessJson<EntityQueryResults>(url);
}
public async Task<EntityIndexResult> EntityIndexAsync(List<Server.JSONEntity> jsonEntity)
public async Task<EntityIndexResult> EntityIndexAsync(List<JSONEntity> jsonEntity)
{
return await EntityIndexAsync(JsonSerializer.Serialize(jsonEntity));
}

2
src/Client/Client.csproj
View File

@@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<ItemGroup>
<ProjectReference Include="..\Server\Server.csproj" />
<ProjectReference Include="..\Shared\Shared.csproj" />
</ItemGroup>
<PropertyGroup>

122
src/Indexer/Controllers/CallsController.cs Normal file
View File

@@ -0,0 +1,122 @@
using ElmahCore;
using Microsoft.AspNetCore.Mvc;
using Indexer.Models;
namespace Indexer.Controllers;
[ApiController]
[Route("[controller]")]
public class CallsController : ControllerBase
{
private readonly ILogger<WorkerController> _logger;
private readonly IConfiguration _config;
private readonly IConfigurationRoot _configurationRoot;
private readonly WorkerCollection _workerCollection;
public CallsController(ILogger<WorkerController> logger, IConfiguration config, IConfigurationRoot configurationRoot, WorkerCollection workerCollection)
{
_logger = logger;
_config = config;
_configurationRoot = configurationRoot;
_workerCollection = workerCollection;
}
[HttpGet("List")]
public ActionResult<CallListResults> List(string name)
{
bool success = true;
List<CallListResult> calls = [];
var configWorkerSection = _config.GetSection("EmbeddingsearchIndexer:Worker");
_workerCollection.Workers.TryGetValue(name, out Worker? worker);
if (worker is null)
{
success = false;
_logger.LogError("No worker found under the name {name}.", [name]);
HttpContext.RaiseError(new Exception($"No worker found under the name {name}"));
}
else
{
foreach (ICall call in worker.Calls)
{
CallListResult callListResult = new()
{
CallConfig = call.CallConfig,
IsRunning = call.IsRunning,
LastExecution = call.LastExecution,
LastSuccessfulExecution = call.LastSuccessfulExecution,
HealthStatus = call.HealthCheck().Status.ToString()
};
calls.Add(callListResult);
}
}
return new CallListResults() { Calls = calls, Success = success };
}
[HttpGet("Enable")]
public ActionResult<WorkerStartResult> Enable(string name)
{
_workerCollection.Workers.TryGetValue(name, out Worker? worker);
if (worker is null)
{
_logger.LogError("Unable to start calls in worker {name} - no running worker with this name.", [name]);
return new WorkerStartResult { Success = false };
}
_logger.LogInformation("Starting calls in worker {name}.", [name]);
foreach (ICall call in worker.Calls)
{
call.Start();
}
_logger.LogInformation("Starting calls in worker {name}.", [name]);
return new WorkerStartResult { Success = true };
}
[HttpGet("Disable")]
public ActionResult<WorkerStopResult> Disable(string name)
{
_workerCollection.Workers.TryGetValue(name, out Worker? worker);
if (worker is null)
{
_logger.LogError("Unable to stop calls in worker {name} - no running worker with this name.", [name]);
return new WorkerStopResult { Success = false };
}
_logger.LogInformation("Stopping calls in worker {name}.", [name]);
foreach (ICall call in worker.Calls)
{
call.Stop();
}
_logger.LogInformation("Stopped calls in worker {name}.", [name]);
return new WorkerStopResult { Success = true };
}
[HttpGet("Reload")]
public ActionResult<WorkerReloadConfigResult> Reload()
{
try
{
_logger.LogInformation("Reloading configuration");
_configurationRoot.Reload();
_logger.LogInformation("Reloaded configuration");
_logger.LogInformation("Destroying workers");
foreach (KeyValuePair<string, Worker> workerKVPair in _workerCollection.Workers)
{
Worker worker = workerKVPair.Value;
foreach (ICall call in worker.Calls)
{
call.Stop();
}
_workerCollection.Workers.Remove(workerKVPair.Key);
_logger.LogInformation("Destroyed worker {workerKVPair.Key}", [workerKVPair.Key]);
}
_logger.LogInformation("Destroyed workers");
_workerCollection.InitializeWorkers();
return new WorkerReloadConfigResult { Success = true };
}
catch (Exception ex)
{
_logger.LogError("Exception {ex.Message} happened while trying to reload the worker configuration.", [ex.Message]);
HttpContext.RaiseError(ex);
return new WorkerReloadConfigResult { Success = false };
}
}
}

74
src/Indexer/Controllers/WorkerController.cs Normal file
View File

@@ -0,0 +1,74 @@
using ElmahCore;
using Microsoft.AspNetCore.Mvc;
using Indexer.Models;
namespace Indexer.Controllers;
[ApiController]
[Route("[controller]")]
public class WorkerController : ControllerBase
{
private readonly ILogger<WorkerController> _logger;
private readonly IConfiguration _config;
private readonly IConfigurationRoot _configurationRoot;
private readonly WorkerCollection _workerCollection;
public WorkerController(ILogger<WorkerController> logger, IConfiguration config, IConfigurationRoot configurationRoot, WorkerCollection workerCollection)
{
_logger = logger;
_config = config;
_configurationRoot = configurationRoot;
_workerCollection = workerCollection;
}
[HttpGet("List")]
public ActionResult<WorkerListResults> List() // List the workers (and perhaps current execution status, maybe also health status and retry count?)
{
bool success = true;
List<WorkerListResult> workerListResultList = [];
try
{
foreach (KeyValuePair<string, Worker> workerKVPair in _workerCollection.Workers)
{
Worker worker = workerKVPair.Value;
WorkerListResult workerListResult = new()
{
Name = worker.Name,
Script = worker.Config.Script,
HealthStatus = worker.HealthCheck().Status.ToString()
};
workerListResultList.Add(workerListResult);
}
}
catch (Exception ex)
{
success = false;
_logger.LogError("Unable to list workers due to exception: {ex.Message}", [ex.Message]);
HttpContext.RaiseError(ex);
}
WorkerListResults workerListResults = new()
{
Workers = workerListResultList,
Success = success
};
return workerListResults;
}
[HttpGet("TriggerUpdate")]
public ActionResult<WorkerTriggerUpdateResult> TriggerUpdate(string name)
{
_workerCollection.Workers.TryGetValue(name, out Worker? worker);
if (worker is null)
{
_logger.LogError("Unable to trigger worker {name} - no running worker with this name.", [name]);
return new WorkerTriggerUpdateResult { Success = false };
}
_logger.LogInformation("triggering worker {name}.", [name]);
ManualTriggerCallbackInfos callbackInfos = new();
worker.Scriptable.Update(callbackInfos);
_logger.LogInformation("triggered worker {name}.", [name]);
return new WorkerTriggerUpdateResult { Success = true };
}
}

2
src/Indexer/Indexer.csproj
View File

@@ -16,7 +16,7 @@
<PackageReference Include="Pythonnet" Version="3.0.5" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Server\Server.csproj" />
<ProjectReference Include="..\Client\Client.csproj" />
<ProjectReference Include="..\Shared\Shared.csproj" />
</ItemGroup>
</Project>

3
src/Indexer/IndexerHealthChecks.cs
View File

@@ -17,8 +17,9 @@ public class WorkerHealthCheck : IHealthCheck
bool hasDegraded = false;
bool hasUnhealthy = false;
Dictionary<string, HealthStatus> degradedWorkerList = [];
foreach (Worker worker in _workerCollection.Workers)
foreach (KeyValuePair<string, Worker> workerKVPair in _workerCollection.Workers)
{
Worker worker = workerKVPair.Value;
HealthCheckResult workerHealth = worker.HealthCheck();
hasDegraded |= workerHealth.Status == HealthStatus.Degraded;
hasUnhealthy |= workerHealth.Status == HealthStatus.Unhealthy;

2
src/Indexer/Models/Interfaces.cs
View File

@@ -7,6 +7,8 @@ public interface IScriptable
ILogger _logger { get; set; }
void Init();
void Update(ICallbackInfos callbackInfos);
void Stop();
bool IsScript(string filePath);
}

56
src/Indexer/Models/Script.cs
View File

@@ -10,12 +10,14 @@ public class PythonScriptable : IScriptable
public PyModule scope;
public dynamic sys;
public string source;
public bool SourceLoaded { get; set; }
public ScriptUpdateInfo UpdateInfo { get; set; }
public ILogger _logger { get; set; }
public PythonScriptable(ScriptToolSet toolSet, ILogger logger)
{
_logger = logger;
Runtime.PythonDLL = @"libpython3.12.so";
SourceLoaded = false;
Runtime.PythonDLL ??= @"libpython3.12.so";
if (!PythonEngine.IsInitialized)
{
PythonEngine.Initialize();
@@ -39,36 +41,22 @@ public class PythonScriptable : IScriptable
public void Init()
{
int retryCounter = 0;
retry:
try
{
using (Py.GIL())
{
pyToolSet = ToolSet.ToPython();
scope.Set("toolset", pyToolSet);
scope.Exec(source);
scope.Exec("init(toolset)");
}
}
catch (Exception ex)
{
UpdateInfo = new() { DateTime = DateTime.Now, Successful = false, Exception = ex };
if (retryCounter < 3)
{
_logger.LogWarning("Unable to init the scriptable - retrying", [ToolSet.filePath, ex]);
retryCounter++;
goto retry;
}
_logger.LogError("Unable to init the scriptable", [ToolSet.filePath, ex]);
throw;
}
UpdateInfo = new() { DateTime = DateTime.Now, Successful = true };
ExecFunction("init");
}
public void Update(ICallbackInfos callbackInfos)
{
int retryCounter = 0;
ExecFunction("update");
}
public void Stop()
{
ExecFunction("stop");
}
public void ExecFunction(string name, ICallbackInfos? callbackInfos = null)
{
int retryCounter = 0;
retry:
try
{
@@ -77,7 +65,12 @@ public class PythonScriptable : IScriptable
pyToolSet = ToolSet.ToPython();
pyToolSet.SetAttr("callbackInfos", callbackInfos.ToPython());
scope.Set("toolset", pyToolSet);
scope.Exec("update(toolset)");
if (!SourceLoaded)
{
scope.Exec(source);
SourceLoaded = true;
}
scope.Exec($"{name}(toolset)");
}
}
catch (Exception ex)
@@ -85,12 +78,11 @@ public class PythonScriptable : IScriptable
UpdateInfo = new() { DateTime = DateTime.Now, Successful = false, Exception = ex };
if (retryCounter < 3)
{
_logger.LogWarning("Execution of script failed to an exception - retrying", [ToolSet.filePath, ex]);
_logger.LogWarning("Execution of {name} function in script {Toolset.filePath} failed to an exception {ex.Message}", [name, ToolSet.filePath, ex.Message]);
retryCounter++;
goto retry;
}
_logger.LogError("Execution of script failed to an exception", [ToolSet.filePath, ex]);
throw;
_logger.LogError("Execution of {name} function in script {Toolset.filePath} failed to an exception {ex.Message}", [name, ToolSet.filePath, ex.Message]);
}
UpdateInfo = new() { DateTime = DateTime.Now, Successful = true };
}
@@ -128,6 +120,8 @@ public class IntervalCallbackInfos : ICallbackInfos
}
public class ManualTriggerCallbackInfos : ICallbackInfos {}
public struct ScriptUpdateInfo
{
public DateTime DateTime { get; set; }

178
src/Indexer/Models/Worker.cs
View File

@@ -1,15 +1,125 @@
using System.Diagnostics;
using Microsoft.Extensions.Diagnostics.HealthChecks;
using Indexer.Exceptions;
namespace Indexer.Models;
public class WorkerCollection
{
public List<Worker> Workers;
public Dictionary<string, Worker> Workers;
public List<Type> types;
public WorkerCollection()
private readonly ILogger _logger;
private readonly IConfiguration _configuration;
private readonly Client.Client client;
public WorkerCollection(ILogger<WorkerCollection> logger, IConfiguration configuration, Client.Client client)
{
Workers = [];
types = [typeof(PythonScriptable)];
_logger = logger;
_configuration = configuration;
this.client = client;
}
public void InitializeWorkers()
{
_logger.LogInformation("Initializing workers");
// Load and configure all workers
var sectionMain = _configuration.GetSection("EmbeddingsearchIndexer");
if (!sectionMain.Exists())
{
_logger.LogCritical("Unable to load section \"EmbeddingsearchIndexer\"");
throw new IndexerConfigurationException("Unable to load section \"EmbeddingsearchIndexer\"");
}
WorkerCollectionConfig? sectionWorker = (WorkerCollectionConfig?)sectionMain.Get(typeof(WorkerCollectionConfig)); //GetValue<WorkerCollectionConfig>("Worker");
if (sectionWorker is not null)
{
foreach (WorkerConfig workerConfig in sectionWorker.Worker)
{
ScriptToolSet toolSet = new(workerConfig.Script, client);
InitializeWorker(toolSet, workerConfig);
}
}
else
{
_logger.LogCritical("Unable to load section \"Worker\"");
throw new IndexerConfigurationException("Unable to load section \"Worker\"");
}
_logger.LogInformation("Initialized workers");
}
public void InitializeWorker(ScriptToolSet toolSet, WorkerConfig workerConfig)
{
_logger.LogInformation("Initializing worker: {Name}", workerConfig.Name);
Worker worker = new(workerConfig.Name, workerConfig, GetScriptable(toolSet));
Workers[workerConfig.Name] = worker;
foreach (CallConfig callConfig in workerConfig.Calls)
{
_logger.LogInformation("Initializing call of type: {Type}", callConfig.Type);
switch (callConfig.Type)
{
case "interval":
if (callConfig.Interval is null)
{
_logger.LogError("Interval not set for a Call in Worker \"{Name}\"", workerConfig.Name);
throw new IndexerConfigurationException($"Interval not set for a Call in Worker \"{workerConfig.Name}\"");
}
var timer = new System.Timers.Timer((double)callConfig.Interval);
timer.AutoReset = true;
timer.Enabled = true;
DateTime now = DateTime.Now;
IntervalCall call = new(timer, worker.Scriptable, _logger, callConfig)
{
LastExecution = now,
LastSuccessfulExecution = now
};
timer.Elapsed += (sender, e) =>
{
try
{
call.LastExecution = DateTime.Now;
worker.Scriptable.Update(new IntervalCallbackInfos() { sender = sender, e = e });
call.LastSuccessfulExecution = DateTime.Now;
}
catch (Exception ex)
{
_logger.LogError("Exception occurred in a Call of Worker \"{name}\": \"{ex}\"", worker.Name, ex.Message);
}
};
worker.Calls.Add(call);
break;
case "schedule": // TODO implement scheduled tasks using Quartz
throw new NotImplementedException("schedule not implemented yet");
case "fileupdate":
if (callConfig.Path is null)
{
_logger.LogError("Path not set for a Call in Worker \"{Name}\"", workerConfig.Name);
throw new IndexerConfigurationException($"Path not set for a Call in Worker \"{workerConfig.Name}\"");
}
throw new NotImplementedException("fileupdate not implemented yet");
//break;
default:
throw new IndexerConfigurationException($"Unknown Type specified for a Call in Worker \"{workerConfig.Name}\"");
}
}
}
public IScriptable GetScriptable(ScriptToolSet toolSet)
{
string fileName = toolSet.filePath;
foreach (Type type in types)
{
IScriptable? instance = (IScriptable?)Activator.CreateInstance(type, [toolSet, _logger]);
if (instance is not null && instance.IsScript(fileName))
{
return instance;
}
}
_logger.LogError("Unable to determine the script's language: \"{fileName}\"", fileName);
throw new UnknownScriptLanguageException(fileName);
}
}
@@ -76,6 +186,12 @@ public class CallConfig
public interface ICall
{
public HealthCheckResult HealthCheck();
public void Start();
public void Stop();
public bool IsRunning { get; set; }
public CallConfig CallConfig { get; set; }
public DateTime? LastExecution { get; set; }
public DateTime? LastSuccessfulExecution { get; set; }
}
public class IntervalCall : ICall
@@ -83,11 +199,31 @@ public class IntervalCall : ICall
public System.Timers.Timer Timer;
public IScriptable Scriptable;
public ILogger _logger;
public IntervalCall(System.Timers.Timer timer, IScriptable scriptable, ILogger logger)
public bool IsRunning { get; set; }
public CallConfig CallConfig { get; set; }
public DateTime? LastExecution { get; set; }
public DateTime? LastSuccessfulExecution { get; set; }
public IntervalCall(System.Timers.Timer timer, IScriptable scriptable, ILogger logger, CallConfig callConfig)
{
Timer = timer;
Scriptable = scriptable;
_logger = logger;
CallConfig = callConfig;
IsRunning = true;
}
public void Start()
{
Timer.Start();
IsRunning = true;
}
public void Stop()
{
Scriptable.Stop();
Timer.Stop();
IsRunning = false;
}
public HealthCheckResult HealthCheck()
@@ -113,6 +249,24 @@ public class IntervalCall : ICall
public class ScheduleCall : ICall
{
public bool IsRunning { get; set; }
public CallConfig CallConfig { get; set; }
public DateTime? LastExecution { get; set; }
public DateTime? LastSuccessfulExecution { get; set; }
public ScheduleCall(CallConfig callConfig)
{
CallConfig = callConfig;
}
public void Start()
{
}
public void Stop()
{
}
public HealthCheckResult HealthCheck()
{
return HealthCheckResult.Unhealthy(); // Not implemented yet
@@ -121,6 +275,24 @@ public class ScheduleCall : ICall
public class FileUpdateCall : ICall
{
public bool IsRunning { get; set; }
public CallConfig CallConfig { get; set; }
public DateTime? LastExecution { get; set; }
public DateTime? LastSuccessfulExecution { get; set; }
public FileUpdateCall(CallConfig callConfig)
{
CallConfig = callConfig;
}
public void Start()
{
}
public void Stop()
{
}
public HealthCheckResult HealthCheck()
{
return HealthCheckResult.Unhealthy(); // Not implemented yet

68
src/Indexer/Models/WorkerResults.cs Normal file
View File

@@ -0,0 +1,68 @@
using System.Text.Json.Serialization;
using Microsoft.Extensions.Diagnostics.HealthChecks;
namespace Indexer.Models;
public class WorkerListResults
{
[JsonPropertyName("WorkerList")]
public required List<WorkerListResult> Workers { get; set; }
[JsonPropertyName("Success")]
public required bool Success { get; set; }
}
public class WorkerListResult
{
[JsonPropertyName("Name")]
public required string Name { get; set; }
[JsonPropertyName("Script")]
public required string Script { get; set; }
[JsonPropertyName("HealthStatus")]
public required string HealthStatus { get; set; }
}
public class CallListResults
{
[JsonPropertyName("Calls")]
public required List<CallListResult> Calls { get; set; }
[JsonPropertyName("Success")]
public required bool Success { get; set; }
}
public class CallListResult
{
[JsonPropertyName("CallConfig")]
public required CallConfig CallConfig { get; set; }
[JsonPropertyName("IsRunning")]
public required bool IsRunning { get; set; }
[JsonPropertyName("LastExecution")]
public required DateTime? LastExecution { get; set; }
[JsonPropertyName("LastSuccessfulExecution")]
public required DateTime? LastSuccessfulExecution { get; set; }
[JsonPropertyName("HealthStatus")]
public required string HealthStatus { get; set; }
}
public class WorkerStopResult
{
[JsonPropertyName("Success")]
public required bool Success { get; set; }
}
public class WorkerStartResult
{
[JsonPropertyName("Success")]
public required bool Success { get; set; }
}
public class WorkerTriggerUpdateResult
{
[JsonPropertyName("Success")]
public required bool Success { get; set; }
}
public class WorkerReloadConfigResult
{
[JsonPropertyName("Success")]
public required bool Success { get; set; }
}

7
src/Indexer/Program.cs
View File

@@ -3,7 +3,6 @@ using Indexer.Models;
using Indexer.Services;
using ElmahCore;
using ElmahCore.Mvc;
using Server;
using ElmahCore.Mvc.Logger;
using Serilog;
@@ -20,12 +19,12 @@ Log.Logger = new LoggerConfiguration()
.CreateLogger();
builder.Logging.AddSerilog();
builder.Services.AddHttpContextAccessor();
builder.Services.AddSingleton<IConfigurationRoot>(builder.Configuration);
builder.Services.AddSingleton<Client.Client>();
builder.Services.AddSingleton<WorkerCollection>();
builder.Services.AddHostedService<IndexerService>();
builder.Services.AddHealthChecks()
.AddCheck<WorkerHealthCheck>("WorkerHealthCheck");
builder.Services.AddElmah<XmlFileErrorLog>(Options =>
{
Options.LogPath = builder.Configuration.GetValue<string>("EmbeddingsearchIndexer:Elmah:LogFolder") ?? "~/logs";
@@ -67,9 +66,11 @@ if (app.Environment.IsDevelopment())
}
else
{
app.UseMiddleware<ApiKeyMiddleware>();
app.UseMiddleware<Shared.ApiKeyMiddleware>();
}
// app.UseHttpsRedirection();
app.MapControllers();
app.Run();

93
src/Indexer/Services/IndexerService.cs
View File

@@ -6,103 +6,18 @@ namespace Indexer.Services;
public class IndexerService : IHostedService
{
private readonly WorkerCollection workerCollection;
private readonly IConfiguration _config;
private readonly Client.Client client;
public WorkerCollection workerCollection;
public ILogger<IndexerService> _logger;
public IndexerService(WorkerCollection workerCollection, IConfiguration configuration, Client.Client client, ILogger<IndexerService> logger, IHttpContextAccessor httpContextAccessor)
public IndexerService(WorkerCollection workerCollection, Client.Client client, ILogger<IndexerService> logger)
{
this._config = configuration;
this.client = client;
this.workerCollection = workerCollection;
_logger = logger;
_logger.LogInformation("Initializing IndexerService");
// Load and configure all workers
var sectionMain = _config.GetSection("EmbeddingsearchIndexer");
if (!sectionMain.Exists())
{
_logger.LogCritical("Unable to load section \"EmbeddingsearchIndexer\"");
throw new IndexerConfigurationException("Unable to load section \"EmbeddingsearchIndexer\"");
}
WorkerCollectionConfig? sectionWorker = (WorkerCollectionConfig?)sectionMain.Get(typeof(WorkerCollectionConfig)); //GetValue<WorkerCollectionConfig>("Worker");
if (sectionWorker is not null)
{
foreach (WorkerConfig workerConfig in sectionWorker.Worker)
{
_logger.LogInformation("Initializing worker: {Name}", workerConfig.Name);
ScriptToolSet toolSet = new(workerConfig.Script, client);
Worker worker = new(workerConfig.Name, workerConfig, GetScriptable(toolSet));
workerCollection.Workers.Add(worker);
foreach (CallConfig callConfig in workerConfig.Calls)
{
_logger.LogInformation("Initializing call of type: {Type}", callConfig.Type);
switch (callConfig.Type)
{
case "interval":
if (callConfig.Interval is null)
{
_logger.LogError("Interval not set for a Call in Worker \"{Name}\"", workerConfig.Name);
throw new IndexerConfigurationException($"Interval not set for a Call in Worker \"{workerConfig.Name}\"");
}
var timer = new System.Timers.Timer((double)callConfig.Interval);
timer.Elapsed += (sender, e) =>
{
try
{
worker.Scriptable.Update(new IntervalCallbackInfos() { sender = sender, e = e });
}
catch (Exception ex)
{
_logger.LogError("Exception occurred in a Call of Worker \"{name}\": \"{ex}\"", worker.Name, ex.Message);
httpContextAccessor.HttpContext.RaiseError(ex);
}
};
timer.AutoReset = true;
timer.Enabled = true;
IntervalCall call = new(timer, worker.Scriptable, _logger);
worker.Calls.Add(call);
break;
case "schedule": // TODO implement scheduled tasks using Quartz
throw new NotImplementedException("schedule not implemented yet");
case "fileupdate":
if (callConfig.Path is null)
{
_logger.LogError("Path not set for a Call in Worker \"{Name}\"", workerConfig.Name);
throw new IndexerConfigurationException($"Path not set for a Call in Worker \"{workerConfig.Name}\"");
}
throw new NotImplementedException("fileupdate not implemented yet");
//break;
default:
throw new IndexerConfigurationException($"Unknown Type specified for a Call in Worker \"{workerConfig.Name}\"");
}
}
}
}
else
{
_logger.LogCritical("Unable to load section \"Worker\"");
throw new IndexerConfigurationException("Unable to load section \"Worker\"");
}
workerCollection.InitializeWorkers();
_logger.LogInformation("Initialized IndexerService");
}
public IScriptable GetScriptable(ScriptToolSet toolSet)
{
string fileName = toolSet.filePath;
foreach (Type type in workerCollection.types)
{
IScriptable? instance = (IScriptable?)Activator.CreateInstance(type, [toolSet, _logger]);
if (instance is not null && instance.IsScript(fileName))
{
return instance;
}
}
_logger.LogError("Unable to determine the script's language: \"{fileName}\"", fileName);
throw new UnknownScriptLanguageException(fileName);
}
public Task StartAsync(CancellationToken cancellationToken)
{
/*foreach (Worker worker in workerCollection.Workers)

2
src/Indexer/appsettings.json
View File

@@ -13,7 +13,7 @@
{ "Name": "File", "Args": { "path": "logs/log.txt", "rollingInterval": "Day", "retainedFileCountLimit": 7 } }
],
"Properties": {
"Application": "Indexer"
"Application": "Embeddingsearch.Indexer"
}
},
"EmbeddingsearchIndexer": {

2
src/Server/AIProvider.cs
View File

@@ -8,7 +8,7 @@ using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using Server.Exceptions;
namespace server;
namespace Server;
public class AIProvider
{

3
src/Server/Controllers/EntityController.cs
View File

@@ -1,7 +1,8 @@
using Microsoft.AspNetCore.Mvc;
using System.Text.Json;
using System.Text.Json.Nodes;
using Server.Models;
using Shared.Models;
using Server.Helper;
namespace Server.Controllers;
[ApiController]

2
src/Server/Controllers/SearchdomainController.cs
View File

@@ -1,6 +1,6 @@
using ElmahCore;
using Microsoft.AspNetCore.Mvc;
using Server.Models;
using Shared.Models;
namespace Server.Controllers;

1
src/Server/Datapoint.cs
View File

@@ -6,7 +6,6 @@ using System.Threading.Tasks;
using Microsoft.Extensions.AI;
using OllamaSharp;
using OllamaSharp.Models;
using server;
namespace Server;

12
src/Server/Helper/SearchdomainHelper.cs
View File

@@ -2,11 +2,9 @@ using System.Collections.Concurrent;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
using MySql.Data.MySqlClient;
using OllamaSharp;
using server;
using Shared.Models;
namespace Server;
namespace Server.Helper;
public static class SearchdomainHelper
{
@@ -41,7 +39,7 @@ public static class SearchdomainHelper
}
return null;
}
public static List<Entity>? EntitiesFromJSON(List<Entity> entityCache, Dictionary<string, Dictionary<string, float[]>> embeddingCache, AIProvider aIProvider, SQLHelper helper, ILogger logger, string json)
{
List<JSONEntity>? jsonEntities = JsonSerializer.Deserialize<List<JSONEntity>>(json);
@@ -78,14 +76,14 @@ public static class SearchdomainHelper
});
return [.. retVal];
}
public static Entity? EntityFromJSON(List<Entity> entityCache, Dictionary<string, Dictionary<string, float[]>> embeddingCache, AIProvider aIProvider, SQLHelper helper, ILogger logger, JSONEntity jsonEntity) //string json)
{
Dictionary<string, Dictionary<string, float[]>> embeddingsLUT = [];
int? preexistingEntityID = DatabaseHelper.GetEntityID(helper, jsonEntity.Name, jsonEntity.Searchdomain);
if (preexistingEntityID is not null)
{
lock (helper.connection)
lock (helper.connection) // TODO change this to helper and do A/B tests (i.e. before/after)
{
Dictionary<string, dynamic> parameters = new()
{

3
src/Server/Program.cs
View File

@@ -1,7 +1,6 @@
using ElmahCore;
using ElmahCore.Mvc;
using Serilog;
using server;
using Server;
using Server.HealthChecks;
@@ -72,7 +71,7 @@ if (IsDevelopment || useSwagger)
}
if (UseMiddleware == true && !IsDevelopment)
{
app.UseMiddleware<ApiKeyMiddleware>();
app.UseMiddleware<Shared.ApiKeyMiddleware>();
}
app.UseAuthorization();

3
src/Server/Searchdomain.cs
View File

@@ -17,11 +17,10 @@ using Mysqlx.Resultset;
using System.Collections.Immutable;
using System.Text.Json;
using System.Numerics.Tensors;
using Server;
using System.Security.Cryptography;
using System.Text;
using System.Collections.Concurrent;
using server;
using Server.Helper;
namespace Server;

1
src/Server/SearchdomainManager.cs
View File

@@ -4,7 +4,6 @@ using OllamaSharp;
using Microsoft.IdentityModel.Tokens;
using Server.Exceptions;
using Server.Migrations;
using server;
namespace Server;

4
src/Server/Server.csproj
View File

@@ -22,4 +22,8 @@
<PackageReference Include="System.Data.Sqlite" Version="1.0.119" />
<PackageReference Include="System.Numerics.Tensors" Version="9.0.3" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Shared\Shared.csproj" />
</ItemGroup>
</Project>

2
src/Server/appsettings.json
View File

@@ -13,7 +13,7 @@
{ "Name": "File", "Args": { "path": "logs/log.txt", "rollingInterval": "Day", "retainedFileCountLimit": 7 } }
],
"Properties": {
"Application": "Indexer"
"Application": "Embeddingsearch.Server"
}
},
"EmbeddingsearchIndexer": {

74
src/Server/ApiKeyMiddleware.cs → src/Shared/ApiKeyMiddleware.cs
View File

@@ -1,37 +1,39 @@
using Microsoft.Extensions.Primitives;
namespace Server;
public class ApiKeyMiddleware
{
private readonly RequestDelegate _next;
private readonly IConfiguration _configuration;
public ApiKeyMiddleware(RequestDelegate next, IConfiguration configuration)
{
_next = next;
_configuration = configuration;
}
public async Task InvokeAsync(HttpContext context)
{
if (!context.Request.Headers.TryGetValue("X-API-KEY", out StringValues extractedApiKey))
{
context.Response.StatusCode = 401;
await context.Response.WriteAsync("API Key is missing.");
return;
}
var validApiKeys = _configuration.GetSection("Embeddingsearch").GetSection("ApiKeys").Get<List<string>>();
#pragma warning disable CS8604
if (validApiKeys == null || !validApiKeys.Contains(extractedApiKey)) // CS8604 extractedApiKey is not null here, but the compiler still thinks that it might be.
{
context.Response.StatusCode = 403;
await context.Response.WriteAsync("Invalid API Key.");
return;
}
#pragma warning restore CS8604
await _next(context);
}
using Microsoft.AspNetCore.Http;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Primitives;
namespace Shared;
public class ApiKeyMiddleware
{
private readonly RequestDelegate _next;
private readonly IConfiguration _configuration;
public ApiKeyMiddleware(RequestDelegate next, IConfiguration configuration)
{
_next = next;
_configuration = configuration;
}
public async Task InvokeAsync(HttpContext context)
{
if (!context.Request.Headers.TryGetValue("X-API-KEY", out StringValues extractedApiKey))
{
context.Response.StatusCode = 401;
await context.Response.WriteAsync("API Key is missing.");
return;
}
var validApiKeys = _configuration.GetSection("Embeddingsearch").GetSection("ApiKeys").Get<List<string>>();
#pragma warning disable CS8604
if (validApiKeys == null || !validApiKeys.Contains(extractedApiKey)) // CS8604 extractedApiKey is not null here, but the compiler still thinks that it might be.
{
context.Response.StatusCode = 403;
await context.Response.WriteAsync("Invalid API Key.");
return;
}
#pragma warning restore CS8604
await _next(context);
}
}

2
src/Server/Models/EntityResults.cs → src/Shared/Models/EntityResults.cs
View File

@@ -1,6 +1,6 @@
using System.Text.Json.Serialization;
namespace Server.Models;
namespace Shared.Models;
public class EntityQueryResults

2
src/Server/JSONModels.cs → src/Shared/Models/JSONModels.cs
View File

@@ -1,4 +1,4 @@
namespace Server;
namespace Shared.Models;
public class JSONEntity
{

2
src/Server/Models/SearchdomainResults.cs → src/Shared/Models/SearchdomainResults.cs
View File

@@ -1,6 +1,6 @@
using System.Text.Json.Serialization;
namespace Server.Models;
namespace Shared.Models;
public class SearchdomainListResults
{

11
src/Shared/Shared.csproj Normal file
View File

@@ -0,0 +1,11 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="ElmahCore" Version="2.1.2" />
</ItemGroup>
</Project>

6
src/embeddingsearch.sln
View File

@@ -9,6 +9,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Indexer", "Indexer\Indexer.
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Client", "Client\Client.csproj", "{4232CA30-ABC0-4EB9-A796-5E217719DD88}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Shared", "Shared\Shared.csproj", "{DD8D64EC-86E9-407E-ADC7-AC3F8F9E6A7F}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@@ -30,5 +32,9 @@ Global
{4232CA30-ABC0-4EB9-A796-5E217719DD88}.Debug|Any CPU.Build.0 = Debug|Any CPU
{4232CA30-ABC0-4EB9-A796-5E217719DD88}.Release|Any CPU.ActiveCfg = Release|Any CPU
{4232CA30-ABC0-4EB9-A796-5E217719DD88}.Release|Any CPU.Build.0 = Release|Any CPU
{DD8D64EC-86E9-407E-ADC7-AC3F8F9E6A7F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{DD8D64EC-86E9-407E-ADC7-AC3F8F9E6A7F}.Debug|Any CPU.Build.0 = Debug|Any CPU
{DD8D64EC-86E9-407E-ADC7-AC3F8F9E6A7F}.Release|Any CPU.ActiveCfg = Release|Any CPU
{DD8D64EC-86E9-407E-ADC7-AC3F8F9E6A7F}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
EndGlobal