Fixed Indexer SwaggerUI, Added worker & call management endpoints to Indexer, restructuring to improve project separation

.gitignore

@@ -15,3 +15,5 @@ src/Indexer/obj
 src/Indexer/Scripts/__pycache__
 src/Indexer/logs
 src/Server/logs
+src/Shared/bin
+src/Shared/obj

src/Client/Client.cs

@@ -7,7 +7,7 @@ using System.Text.Json;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Configuration;
 using System.Reflection.Metadata.Ecma335;
-using Server.Models;
+using Shared.Models;
 
 namespace Client;
 
@@ -86,7 +86,7 @@ public class Client
             return await GetUrlAndProcessJson<EntityQueryResults>(url);
         }
 
-        public async Task<EntityIndexResult> EntityIndexAsync(List<Server.JSONEntity> jsonEntity)
+        public async Task<EntityIndexResult> EntityIndexAsync(List<JSONEntity> jsonEntity)
         {
             return await EntityIndexAsync(JsonSerializer.Serialize(jsonEntity));
         }

src/Client/Client.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <ItemGroup>
-    <ProjectReference Include="..\Server\Server.csproj" />
+    <ProjectReference Include="..\Shared\Shared.csproj" />
   </ItemGroup>
 
   <PropertyGroup>

src/Indexer/Controllers/CallsController.cs

@@ -0,0 +1,122 @@
+using ElmahCore;
+using Microsoft.AspNetCore.Mvc;
+using Indexer.Models;
+
+namespace Indexer.Controllers;
+
+[ApiController]
+[Route("[controller]")]
+public class CallsController : ControllerBase
+{
+    private readonly ILogger<WorkerController> _logger;
+    private readonly IConfiguration _config;
+    private readonly IConfigurationRoot _configurationRoot;
+    private readonly WorkerCollection _workerCollection;
+
+    public CallsController(ILogger<WorkerController> logger, IConfiguration config, IConfigurationRoot configurationRoot, WorkerCollection workerCollection)
+    {
+        _logger = logger;
+        _config = config;
+        _configurationRoot = configurationRoot;
+        _workerCollection = workerCollection;
+    }
+
+    [HttpGet("List")]
+    public ActionResult<CallListResults> List(string name)
+    {
+        bool success = true;
+        List<CallListResult> calls = [];
+        var configWorkerSection = _config.GetSection("EmbeddingsearchIndexer:Worker");
+        _workerCollection.Workers.TryGetValue(name, out Worker? worker);
+        if (worker is null)
+        {
+            success = false;
+            _logger.LogError("No worker found under the name {name}.", [name]);
+            HttpContext.RaiseError(new Exception($"No worker found under the name {name}"));
+        }
+        else
+        {
+            foreach (ICall call in worker.Calls)
+            {
+                CallListResult callListResult = new()
+                {
+                    CallConfig = call.CallConfig,
+                    IsRunning = call.IsRunning,
+                    LastExecution = call.LastExecution,
+                    LastSuccessfulExecution = call.LastSuccessfulExecution,
+                    HealthStatus = call.HealthCheck().Status.ToString()
+                };
+                calls.Add(callListResult);
+            }
+        }
+        return new CallListResults() { Calls = calls, Success = success };
+    }
+
+    [HttpGet("Enable")]
+    public ActionResult<WorkerStartResult> Enable(string name)
+    {
+        _workerCollection.Workers.TryGetValue(name, out Worker? worker);
+        if (worker is null)
+        {
+            _logger.LogError("Unable to start calls in worker {name} - no running worker with this name.", [name]);
+            return new WorkerStartResult { Success = false };
+        }
+        _logger.LogInformation("Starting calls in worker {name}.", [name]);
+        foreach (ICall call in worker.Calls)
+        {
+            call.Start();
+        }
+        _logger.LogInformation("Starting calls in worker {name}.", [name]);
+        return new WorkerStartResult { Success = true };
+    }
+
+    [HttpGet("Disable")]
+    public ActionResult<WorkerStopResult> Disable(string name)
+    {
+        _workerCollection.Workers.TryGetValue(name, out Worker? worker);
+        if (worker is null)
+        {
+            _logger.LogError("Unable to stop calls in worker {name} - no running worker with this name.", [name]);
+            return new WorkerStopResult { Success = false };
+        }
+        _logger.LogInformation("Stopping calls in worker {name}.", [name]);
+        foreach (ICall call in worker.Calls)
+        {
+            call.Stop();
+        }
+        _logger.LogInformation("Stopped calls in worker {name}.", [name]);
+        return new WorkerStopResult { Success = true };
+    }
+
+    [HttpGet("Reload")]
+    public ActionResult<WorkerReloadConfigResult> Reload()
+    {
+        try
+        {
+            _logger.LogInformation("Reloading configuration");
+            _configurationRoot.Reload();
+            _logger.LogInformation("Reloaded configuration");
+            _logger.LogInformation("Destroying workers");
+            foreach (KeyValuePair<string, Worker> workerKVPair in _workerCollection.Workers)
+            {
+                Worker worker = workerKVPair.Value;
+                foreach (ICall call in worker.Calls)
+                {
+                    call.Stop();
+                }
+                _workerCollection.Workers.Remove(workerKVPair.Key);
+                _logger.LogInformation("Destroyed worker {workerKVPair.Key}", [workerKVPair.Key]);
+            }
+            _logger.LogInformation("Destroyed workers");
+            _workerCollection.InitializeWorkers();
+            return new WorkerReloadConfigResult { Success = true };
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError("Exception {ex.Message} happened while trying to reload the worker configuration.", [ex.Message]);
+            HttpContext.RaiseError(ex);
+            return new WorkerReloadConfigResult { Success = false };
+        }
+    }
+
+}

src/Indexer/Controllers/WorkerController.cs

@@ -0,0 +1,74 @@
+using ElmahCore;
+using Microsoft.AspNetCore.Mvc;
+using Indexer.Models;
+
+namespace Indexer.Controllers;
+
+[ApiController]
+[Route("[controller]")]
+public class WorkerController : ControllerBase
+{
+    private readonly ILogger<WorkerController> _logger;
+    private readonly IConfiguration _config;
+    private readonly IConfigurationRoot _configurationRoot;
+    private readonly WorkerCollection _workerCollection;
+
+    public WorkerController(ILogger<WorkerController> logger, IConfiguration config, IConfigurationRoot configurationRoot, WorkerCollection workerCollection)
+    {
+        _logger = logger;
+        _config = config;
+        _configurationRoot = configurationRoot;
+        _workerCollection = workerCollection;
+    }
+
+    [HttpGet("List")]
+    public ActionResult<WorkerListResults> List() // List the workers (and perhaps current execution status, maybe also health status and retry count?)
+    {
+        bool success = true;
+        List<WorkerListResult> workerListResultList = [];
+        try
+        {
+            foreach (KeyValuePair<string, Worker> workerKVPair in _workerCollection.Workers)
+            {
+                Worker worker = workerKVPair.Value;
+                WorkerListResult workerListResult = new()
+                {
+                    Name = worker.Name,
+                    Script = worker.Config.Script,
+                    HealthStatus = worker.HealthCheck().Status.ToString()
+                };
+                workerListResultList.Add(workerListResult);
+            }
+        }
+        catch (Exception ex)
+        {
+            success = false;
+            _logger.LogError("Unable to list workers due to exception: {ex.Message}", [ex.Message]);
+            HttpContext.RaiseError(ex);
+        }
+        WorkerListResults workerListResults = new()
+        {
+            Workers = workerListResultList,
+            Success = success
+        };
+        return workerListResults;
+    }
+
+    [HttpGet("TriggerUpdate")]
+    public ActionResult<WorkerTriggerUpdateResult> TriggerUpdate(string name)
+    {
+        _workerCollection.Workers.TryGetValue(name, out Worker? worker);
+        if (worker is null)
+        {
+            _logger.LogError("Unable to trigger worker {name} - no running worker with this name.", [name]);
+            return new WorkerTriggerUpdateResult { Success = false };
+        }
+        _logger.LogInformation("triggering worker {name}.", [name]);
+        ManualTriggerCallbackInfos callbackInfos = new();
+        worker.Scriptable.Update(callbackInfos);
+        _logger.LogInformation("triggered worker {name}.", [name]);
+        return new WorkerTriggerUpdateResult { Success = true };
+
+    }
+
+}

src/Indexer/Indexer.csproj

@@ -16,7 +16,7 @@
     <PackageReference Include="Pythonnet" Version="3.0.5" />
   </ItemGroup>
   <ItemGroup>
-    <ProjectReference Include="..\Server\Server.csproj" />
     <ProjectReference Include="..\Client\Client.csproj" />
+    <ProjectReference Include="..\Shared\Shared.csproj" />
   </ItemGroup>
 </Project>

src/Indexer/IndexerHealthChecks.cs

@@ -17,8 +17,9 @@ public class WorkerHealthCheck : IHealthCheck
         bool hasDegraded = false;
         bool hasUnhealthy = false;
         Dictionary<string, HealthStatus> degradedWorkerList = [];
-        foreach (Worker worker in _workerCollection.Workers)
+        foreach (KeyValuePair<string, Worker> workerKVPair in _workerCollection.Workers)
         {
+            Worker worker = workerKVPair.Value;
             HealthCheckResult workerHealth = worker.HealthCheck();
             hasDegraded |= workerHealth.Status == HealthStatus.Degraded;
             hasUnhealthy |= workerHealth.Status == HealthStatus.Unhealthy;

src/Indexer/Models/Interfaces.cs

@@ -7,6 +7,8 @@ public interface IScriptable
     ILogger _logger { get; set; }
     void Init();
     void Update(ICallbackInfos callbackInfos);
+    void Stop();
+
     bool IsScript(string filePath);
 }
 

src/Indexer/Models/Script.cs

@@ -10,12 +10,14 @@ public class PythonScriptable : IScriptable
     public PyModule scope;
     public dynamic sys;
     public string source;
+    public bool SourceLoaded { get; set; }
     public ScriptUpdateInfo UpdateInfo { get; set; }
     public ILogger _logger { get; set; }
     public PythonScriptable(ScriptToolSet toolSet, ILogger logger)
     {
         _logger = logger;
-        Runtime.PythonDLL = @"libpython3.12.so";
+        SourceLoaded = false;
+        Runtime.PythonDLL ??= @"libpython3.12.so";
         if (!PythonEngine.IsInitialized)
         {
             PythonEngine.Initialize();
@@ -39,36 +41,22 @@ public class PythonScriptable : IScriptable
 
     public void Init()
     {
-        int retryCounter = 0;
+        ExecFunction("init");
-        retry:
-        try
-        {
-            using (Py.GIL())
-            {
-                pyToolSet = ToolSet.ToPython();
-                scope.Set("toolset", pyToolSet);
-                scope.Exec(source);
-                scope.Exec("init(toolset)");
-            }
-        }
-        catch (Exception ex)
-        {
-            UpdateInfo = new() { DateTime = DateTime.Now, Successful = false, Exception = ex };
-            if (retryCounter < 3)
-            {
-                _logger.LogWarning("Unable to init the scriptable - retrying", [ToolSet.filePath, ex]);
-                retryCounter++;
-                goto retry;
-            }
-            _logger.LogError("Unable to init the scriptable", [ToolSet.filePath, ex]);
-            throw;
-        }
-        UpdateInfo = new() { DateTime = DateTime.Now, Successful = true };
     }
 
     public void Update(ICallbackInfos callbackInfos)
     {
-        int retryCounter = 0;
+        ExecFunction("update");
+    }
+
+    public void Stop()
+    {
+        ExecFunction("stop");
+    }
+
+    public void ExecFunction(string name, ICallbackInfos? callbackInfos = null)
+    {
+       int retryCounter = 0;
         retry:
         try
         {
@@ -77,7 +65,12 @@ public class PythonScriptable : IScriptable
                 pyToolSet = ToolSet.ToPython();
                 pyToolSet.SetAttr("callbackInfos", callbackInfos.ToPython());
                 scope.Set("toolset", pyToolSet);
-                scope.Exec("update(toolset)");
+                if (!SourceLoaded)
+                {
+                    scope.Exec(source);
+                    SourceLoaded = true;
+                }
+                scope.Exec($"{name}(toolset)");
             }
         }
         catch (Exception ex)
@@ -85,12 +78,11 @@ public class PythonScriptable : IScriptable
             UpdateInfo = new() { DateTime = DateTime.Now, Successful = false, Exception = ex };
             if (retryCounter < 3)
             {
-                _logger.LogWarning("Execution of script failed to an exception - retrying", [ToolSet.filePath, ex]);
+                _logger.LogWarning("Execution of {name} function in script {Toolset.filePath} failed to an exception {ex.Message}", [name, ToolSet.filePath, ex.Message]);
                 retryCounter++;
                 goto retry;
             }
-            _logger.LogError("Execution of script failed to an exception", [ToolSet.filePath, ex]);
+            _logger.LogError("Execution of {name} function in script {Toolset.filePath} failed to an exception {ex.Message}", [name, ToolSet.filePath, ex.Message]);
-            throw;
         }
         UpdateInfo = new() { DateTime = DateTime.Now, Successful = true };
     }
@@ -128,6 +120,8 @@ public class IntervalCallbackInfos : ICallbackInfos
 
 }
 
+public class ManualTriggerCallbackInfos : ICallbackInfos {}
+
 public struct ScriptUpdateInfo
 {
     public DateTime DateTime { get; set; }

src/Indexer/Models/Worker.cs

@@ -1,15 +1,125 @@
 using System.Diagnostics;
 using Microsoft.Extensions.Diagnostics.HealthChecks;
+using Indexer.Exceptions;
+
 namespace Indexer.Models;
 
 public class WorkerCollection
 {
-    public List<Worker> Workers;
+    public Dictionary<string, Worker> Workers;
     public List<Type> types;
-    public WorkerCollection()
+    private readonly ILogger _logger;
+    private readonly IConfiguration _configuration;
+    private readonly Client.Client client;
+
+    public WorkerCollection(ILogger<WorkerCollection> logger, IConfiguration configuration, Client.Client client)
     {
         Workers = [];
         types = [typeof(PythonScriptable)];
+        _logger = logger;
+        _configuration = configuration;
+        this.client = client;
+    }
+
+    public void InitializeWorkers()
+    {
+        _logger.LogInformation("Initializing workers");
+        // Load and configure all workers
+        var sectionMain = _configuration.GetSection("EmbeddingsearchIndexer");
+        if (!sectionMain.Exists())
+        {
+            _logger.LogCritical("Unable to load section \"EmbeddingsearchIndexer\"");
+            throw new IndexerConfigurationException("Unable to load section \"EmbeddingsearchIndexer\"");
+        }
+
+        WorkerCollectionConfig? sectionWorker = (WorkerCollectionConfig?)sectionMain.Get(typeof(WorkerCollectionConfig)); //GetValue<WorkerCollectionConfig>("Worker");
+        if (sectionWorker is not null)
+        {
+            foreach (WorkerConfig workerConfig in sectionWorker.Worker)
+            {
+                ScriptToolSet toolSet = new(workerConfig.Script, client);
+                InitializeWorker(toolSet, workerConfig);
+            }
+        }
+        else
+        {
+            _logger.LogCritical("Unable to load section \"Worker\"");
+            throw new IndexerConfigurationException("Unable to load section \"Worker\"");
+        }
+        _logger.LogInformation("Initialized workers");
+    }
+
+    public void InitializeWorker(ScriptToolSet toolSet, WorkerConfig workerConfig)
+    {
+        _logger.LogInformation("Initializing worker: {Name}", workerConfig.Name);
+        Worker worker = new(workerConfig.Name, workerConfig, GetScriptable(toolSet));
+        Workers[workerConfig.Name] = worker;
+        foreach (CallConfig callConfig in workerConfig.Calls)
+        {
+            _logger.LogInformation("Initializing call of type: {Type}", callConfig.Type);
+
+            switch (callConfig.Type)
+            {
+                case "interval":
+                    if (callConfig.Interval is null)
+                    {
+                        _logger.LogError("Interval not set for a Call in Worker \"{Name}\"", workerConfig.Name);
+                        throw new IndexerConfigurationException($"Interval not set for a Call in Worker \"{workerConfig.Name}\"");
+                    }
+                    var timer = new System.Timers.Timer((double)callConfig.Interval);
+                    timer.AutoReset = true;
+                    timer.Enabled = true;
+                    DateTime now = DateTime.Now;
+                    IntervalCall call = new(timer, worker.Scriptable, _logger, callConfig)
+                    {
+                        LastExecution = now,
+                        LastSuccessfulExecution = now
+                    };
+                    timer.Elapsed += (sender, e) =>
+                    {
+                        try
+                        {
+                            call.LastExecution = DateTime.Now;
+                            worker.Scriptable.Update(new IntervalCallbackInfos() { sender = sender, e = e });
+                            call.LastSuccessfulExecution = DateTime.Now;
+                        }
+                        catch (Exception ex)
+                        {
+                            _logger.LogError("Exception occurred in a Call of Worker \"{name}\": \"{ex}\"", worker.Name, ex.Message);
+                        }
+                    };
+                    worker.Calls.Add(call);
+                    break;
+                case "schedule": // TODO implement scheduled tasks using Quartz
+                    throw new NotImplementedException("schedule not implemented yet");
+                case "fileupdate":
+                    if (callConfig.Path is null)
+                    {
+                        _logger.LogError("Path not set for a Call in Worker \"{Name}\"", workerConfig.Name);
+                        throw new IndexerConfigurationException($"Path not set for a Call in Worker \"{workerConfig.Name}\"");
+                    }
+                    throw new NotImplementedException("fileupdate not implemented yet");
+                //break;
+                default:
+                    throw new IndexerConfigurationException($"Unknown Type specified for a Call in Worker \"{workerConfig.Name}\"");
+            }
+        }
+    }
+
+    public IScriptable GetScriptable(ScriptToolSet toolSet)
+    {
+        string fileName = toolSet.filePath;
+        foreach (Type type in types)
+        {
+            IScriptable? instance = (IScriptable?)Activator.CreateInstance(type, [toolSet, _logger]);
+            if (instance is not null && instance.IsScript(fileName))
+            {
+                return instance;
+            }
+        }
+        _logger.LogError("Unable to determine the script's language: \"{fileName}\"", fileName);
+
+        throw new UnknownScriptLanguageException(fileName);
     }
 }
 
@@ -76,6 +186,12 @@ public class CallConfig
 public interface ICall
 {
     public HealthCheckResult HealthCheck();
+    public void Start();
+    public void Stop();
+    public bool IsRunning { get; set; }
+    public CallConfig CallConfig { get; set; }
+    public DateTime? LastExecution { get; set; }
+    public DateTime? LastSuccessfulExecution { get; set; }
 }
 
 public class IntervalCall : ICall
@@ -83,11 +199,31 @@ public class IntervalCall : ICall
     public System.Timers.Timer Timer;
     public IScriptable Scriptable;
     public ILogger _logger;
-    public IntervalCall(System.Timers.Timer timer, IScriptable scriptable, ILogger logger)
+    public bool IsRunning { get; set; }
+    public CallConfig CallConfig { get; set; }
+    public DateTime? LastExecution { get; set; }
+    public DateTime? LastSuccessfulExecution { get; set; }
+
+    public IntervalCall(System.Timers.Timer timer, IScriptable scriptable, ILogger logger, CallConfig callConfig)
     {
         Timer = timer;
         Scriptable = scriptable;
         _logger = logger;
+        CallConfig = callConfig;
+        IsRunning = true;
+    }
+
+    public void Start()
+    {
+        Timer.Start();
+        IsRunning = true;
+    }
+
+    public void Stop()
+    {
+        Scriptable.Stop();
+        Timer.Stop();
+        IsRunning = false;
     }
 
     public HealthCheckResult HealthCheck()
@@ -113,6 +249,24 @@ public class IntervalCall : ICall
 
 public class ScheduleCall : ICall
 {
+    public bool IsRunning { get; set; }
+    public CallConfig CallConfig { get; set; }
+    public DateTime? LastExecution { get; set; }
+    public DateTime? LastSuccessfulExecution { get; set; }
+
+    public ScheduleCall(CallConfig callConfig)
+    {
+        CallConfig = callConfig;
+    }
+
+    public void Start()
+    {
+    }
+
+    public void Stop()
+    {
+    }
+
     public HealthCheckResult HealthCheck()
     {
         return HealthCheckResult.Unhealthy(); // Not implemented yet
@@ -121,6 +275,24 @@ public class ScheduleCall : ICall
 
 public class FileUpdateCall : ICall
 {
+    public bool IsRunning { get; set; }
+    public CallConfig CallConfig { get; set; }
+    public DateTime? LastExecution { get; set; }
+    public DateTime? LastSuccessfulExecution { get; set; }
+
+    public FileUpdateCall(CallConfig callConfig)
+    {
+        CallConfig = callConfig;
+    }
+
+    public void Start()
+    {
+    }
+
+    public void Stop()
+    {
+    }
+
     public HealthCheckResult HealthCheck()
     {
         return HealthCheckResult.Unhealthy(); // Not implemented yet

src/Indexer/Models/WorkerResults.cs

@@ -0,0 +1,68 @@
+using System.Text.Json.Serialization;
+using Microsoft.Extensions.Diagnostics.HealthChecks;
+
+namespace Indexer.Models;
+
+public class WorkerListResults
+{
+    [JsonPropertyName("WorkerList")]
+    public required List<WorkerListResult> Workers { get; set; }
+    [JsonPropertyName("Success")]
+    public required bool Success { get; set; }
+}
+
+public class WorkerListResult
+{
+    [JsonPropertyName("Name")]
+    public required string Name { get; set; }
+    [JsonPropertyName("Script")]
+    public required string Script { get; set; }
+    [JsonPropertyName("HealthStatus")]
+    public required string HealthStatus { get; set; }
+}
+
+public class CallListResults
+{
+    [JsonPropertyName("Calls")]
+    public required List<CallListResult> Calls { get; set; }
+    [JsonPropertyName("Success")]
+    public required bool Success { get; set; }
+}
+
+public class CallListResult
+{
+    [JsonPropertyName("CallConfig")]
+    public required CallConfig CallConfig { get; set; }
+    [JsonPropertyName("IsRunning")]
+    public required bool IsRunning { get; set; }
+    [JsonPropertyName("LastExecution")]
+    public required DateTime? LastExecution { get; set; }
+    [JsonPropertyName("LastSuccessfulExecution")]
+    public required DateTime? LastSuccessfulExecution { get; set; }
+    [JsonPropertyName("HealthStatus")]
+    public required string HealthStatus { get; set; }
+}
+
+public class WorkerStopResult
+{
+    [JsonPropertyName("Success")]
+    public required bool Success { get; set; }
+}
+
+public class WorkerStartResult
+{
+    [JsonPropertyName("Success")]
+    public required bool Success { get; set; }
+}
+
+public class WorkerTriggerUpdateResult
+{
+    [JsonPropertyName("Success")]
+    public required bool Success { get; set; }
+}
+
+public class WorkerReloadConfigResult
+{
+    [JsonPropertyName("Success")]
+    public required bool Success { get; set; }
+}

src/Indexer/Program.cs

@@ -3,7 +3,6 @@ using Indexer.Models;
 using Indexer.Services;
 using ElmahCore;
 using ElmahCore.Mvc;
-using Server;
 using ElmahCore.Mvc.Logger;
 using Serilog;
 
@@ -20,12 +19,12 @@ Log.Logger = new LoggerConfiguration()
     .CreateLogger();
 builder.Logging.AddSerilog();
 builder.Services.AddHttpContextAccessor();
+builder.Services.AddSingleton<IConfigurationRoot>(builder.Configuration);
 builder.Services.AddSingleton<Client.Client>();
 builder.Services.AddSingleton<WorkerCollection>();
 builder.Services.AddHostedService<IndexerService>();
 builder.Services.AddHealthChecks()
     .AddCheck<WorkerHealthCheck>("WorkerHealthCheck");
-
 builder.Services.AddElmah<XmlFileErrorLog>(Options =>
 {
     Options.LogPath = builder.Configuration.GetValue<string>("EmbeddingsearchIndexer:Elmah:LogFolder") ?? "~/logs";
@@ -67,9 +66,11 @@ if (app.Environment.IsDevelopment())
 }
 else
 {
-    app.UseMiddleware<ApiKeyMiddleware>();
+    app.UseMiddleware<Shared.ApiKeyMiddleware>();
 }
 
 // app.UseHttpsRedirection();
 
+app.MapControllers();
+
 app.Run();

src/Indexer/Services/IndexerService.cs

@@ -6,103 +6,18 @@ namespace Indexer.Services;
 
 public class IndexerService : IHostedService
 {
-    private readonly WorkerCollection workerCollection;
+    public WorkerCollection workerCollection;
-    private readonly IConfiguration _config;
-    private readonly Client.Client client;
     public ILogger<IndexerService> _logger;
 
-    public IndexerService(WorkerCollection workerCollection, IConfiguration configuration, Client.Client client, ILogger<IndexerService> logger, IHttpContextAccessor httpContextAccessor)
+    public IndexerService(WorkerCollection workerCollection, Client.Client client, ILogger<IndexerService> logger)
     {
-        this._config = configuration;
-        this.client = client;
         this.workerCollection = workerCollection;
         _logger = logger;
         _logger.LogInformation("Initializing IndexerService");
-        // Load and configure all workers
+        workerCollection.InitializeWorkers();
-        var sectionMain = _config.GetSection("EmbeddingsearchIndexer");
+        _logger.LogInformation("Initialized IndexerService");
-        if (!sectionMain.Exists())
-        {
-            _logger.LogCritical("Unable to load section \"EmbeddingsearchIndexer\"");
-            throw new IndexerConfigurationException("Unable to load section \"EmbeddingsearchIndexer\"");
-        }
-
-        WorkerCollectionConfig? sectionWorker = (WorkerCollectionConfig?)sectionMain.Get(typeof(WorkerCollectionConfig)); //GetValue<WorkerCollectionConfig>("Worker");
-        if (sectionWorker is not null)
-        {
-            foreach (WorkerConfig workerConfig in sectionWorker.Worker)
-            {
-                _logger.LogInformation("Initializing worker: {Name}", workerConfig.Name);
-                ScriptToolSet toolSet = new(workerConfig.Script, client);
-                Worker worker = new(workerConfig.Name, workerConfig, GetScriptable(toolSet));
-                workerCollection.Workers.Add(worker);
-                foreach (CallConfig callConfig in workerConfig.Calls)
-                {
-                    _logger.LogInformation("Initializing call of type: {Type}", callConfig.Type);
-                    
-                    switch (callConfig.Type)
-                    {
-                        case "interval":
-                            if (callConfig.Interval is null)
-                            {
-                                _logger.LogError("Interval not set for a Call in Worker \"{Name}\"", workerConfig.Name);
-                                throw new IndexerConfigurationException($"Interval not set for a Call in Worker \"{workerConfig.Name}\"");
-                            }
-                            var timer = new System.Timers.Timer((double)callConfig.Interval);
-                            timer.Elapsed += (sender, e) =>
-                            {
-                                try
-                                {
-                                    worker.Scriptable.Update(new IntervalCallbackInfos() { sender = sender, e = e });
-                                }
-                                catch (Exception ex)
-                                {
-                                    _logger.LogError("Exception occurred in a Call of Worker \"{name}\": \"{ex}\"", worker.Name, ex.Message);
-                                    httpContextAccessor.HttpContext.RaiseError(ex);
-                                }
-                            };
-                            timer.AutoReset = true;
-                            timer.Enabled = true;
-                            IntervalCall call = new(timer, worker.Scriptable, _logger);
-                            worker.Calls.Add(call);
-                            break;
-                        case "schedule": // TODO implement scheduled tasks using Quartz
-                            throw new NotImplementedException("schedule not implemented yet");
-                        case "fileupdate":
-                            if (callConfig.Path is null)
-                            {
-                                _logger.LogError("Path not set for a Call in Worker \"{Name}\"", workerConfig.Name);
-                                throw new IndexerConfigurationException($"Path not set for a Call in Worker \"{workerConfig.Name}\"");
-                            }
-                            throw new NotImplementedException("fileupdate not implemented yet");
-                        //break;
-                        default:
-                            throw new IndexerConfigurationException($"Unknown Type specified for a Call in Worker \"{workerConfig.Name}\"");
-                    }
-                }
-            }
-        }
-        else
-        {
-            _logger.LogCritical("Unable to load section \"Worker\"");
-            throw new IndexerConfigurationException("Unable to load section \"Worker\"");
-        }
     }
 
-    public IScriptable GetScriptable(ScriptToolSet toolSet)
-    {
-        string fileName = toolSet.filePath;
-        foreach (Type type in workerCollection.types)
-        {
-            IScriptable? instance = (IScriptable?)Activator.CreateInstance(type, [toolSet, _logger]);
-            if (instance is not null && instance.IsScript(fileName))
-            {
-                return instance;
-            }
-        }
-        _logger.LogError("Unable to determine the script's language: \"{fileName}\"", fileName);
-
-        throw new UnknownScriptLanguageException(fileName);
-    }
     public Task StartAsync(CancellationToken cancellationToken)
     {
         /*foreach (Worker worker in workerCollection.Workers)

src/Indexer/appsettings.json

@@ -13,7 +13,7 @@
       { "Name": "File", "Args": { "path": "logs/log.txt", "rollingInterval": "Day", "retainedFileCountLimit": 7 } }
     ],
     "Properties": {
-        "Application": "Indexer"
+        "Application": "Embeddingsearch.Indexer"
     }
   },
   "EmbeddingsearchIndexer": {

src/Server/AIProvider.cs

@@ -8,7 +8,7 @@ using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
 using Server.Exceptions;
 
-namespace server;
+namespace Server;
 
 public class AIProvider
 {

src/Server/Controllers/EntityController.cs

@@ -1,7 +1,8 @@
 using Microsoft.AspNetCore.Mvc;
 using System.Text.Json;
 using System.Text.Json.Nodes;
-using Server.Models;
+using Shared.Models;
+using Server.Helper;
 namespace Server.Controllers;
 
 [ApiController]

src/Server/Controllers/SearchdomainController.cs

@@ -1,6 +1,6 @@
 using ElmahCore;
 using Microsoft.AspNetCore.Mvc;
-using Server.Models;
+using Shared.Models;
 
 namespace Server.Controllers;
 

src/Server/Datapoint.cs

@@ -6,7 +6,6 @@ using System.Threading.Tasks;
 using Microsoft.Extensions.AI;
 using OllamaSharp;
 using OllamaSharp.Models;
-using server;
 
 namespace Server;
 

src/Server/Helper/SearchdomainHelper.cs

@@ -2,11 +2,9 @@ using System.Collections.Concurrent;
 using System.Security.Cryptography;
 using System.Text;
 using System.Text.Json;
-using MySql.Data.MySqlClient;
+using Shared.Models;
-using OllamaSharp;
-using server;
 
-namespace Server;
+namespace Server.Helper;
 
 public static class SearchdomainHelper
 {
@@ -85,7 +83,7 @@ public static class SearchdomainHelper
         int? preexistingEntityID = DatabaseHelper.GetEntityID(helper, jsonEntity.Name, jsonEntity.Searchdomain);
         if (preexistingEntityID is not null)
         {
-            lock (helper.connection)
+            lock (helper.connection) // TODO change this to helper and do A/B tests (i.e. before/after)
             {
                 Dictionary<string, dynamic> parameters = new()
                 {

src/Server/Program.cs

@@ -1,7 +1,6 @@
 using ElmahCore;
 using ElmahCore.Mvc;
 using Serilog;
-using server;
 using Server;
 using Server.HealthChecks;
 
@@ -72,7 +71,7 @@ if (IsDevelopment || useSwagger)
 }
 if (UseMiddleware == true && !IsDevelopment)
 {
-    app.UseMiddleware<ApiKeyMiddleware>();
+    app.UseMiddleware<Shared.ApiKeyMiddleware>();
 }
 
 app.UseAuthorization();

src/Server/Searchdomain.cs

@@ -17,11 +17,10 @@ using Mysqlx.Resultset;
 using System.Collections.Immutable;
 using System.Text.Json;
 using System.Numerics.Tensors;
-using Server;
 using System.Security.Cryptography;
 using System.Text;
 using System.Collections.Concurrent;
-using server;
+using Server.Helper;
 
 namespace Server;
 

src/Server/SearchdomainManager.cs

@@ -4,7 +4,6 @@ using OllamaSharp;
 using Microsoft.IdentityModel.Tokens;
 using Server.Exceptions;
 using Server.Migrations;
-using server;
 
 namespace Server;
 

src/Server/Server.csproj

@@ -22,4 +22,8 @@
       <PackageReference Include="System.Data.Sqlite" Version="1.0.119" />
       <PackageReference Include="System.Numerics.Tensors" Version="9.0.3" />
   </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Shared\Shared.csproj" />
+  </ItemGroup>
 </Project>

src/Server/appsettings.json

@@ -13,7 +13,7 @@
       { "Name": "File", "Args": { "path": "logs/log.txt", "rollingInterval": "Day", "retainedFileCountLimit": 7 } }
     ],
     "Properties": {
-        "Application": "Indexer"
+        "Application": "Embeddingsearch.Server"
     }
   },
   "EmbeddingsearchIndexer": {

src/Shared/ApiKeyMiddleware.cs

@@ -1,6 +1,8 @@
+using Microsoft.AspNetCore.Http;
+using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.Primitives;
 
-namespace Server;
+namespace Shared;
 
 public class ApiKeyMiddleware
 {

src/Shared/Models/EntityResults.cs

@@ -1,6 +1,6 @@
 using System.Text.Json.Serialization;
 
-namespace Server.Models;
+namespace Shared.Models;
 
 
 public class EntityQueryResults

src/Shared/Models/JSONModels.cs

@@ -1,4 +1,4 @@
-namespace Server;
+namespace Shared.Models;
 
 public class JSONEntity
 {

src/Shared/Models/SearchdomainResults.cs

@@ -1,6 +1,6 @@
 using System.Text.Json.Serialization;
 
-namespace Server.Models;
+namespace Shared.Models;
 
 public class SearchdomainListResults
 {

src/Shared/Shared.csproj

@@ -0,0 +1,11 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+  <ItemGroup>
+    <PackageReference Include="ElmahCore" Version="2.1.2" />
+  </ItemGroup>
+</Project>

src/embeddingsearch.sln

@@ -9,6 +9,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Indexer", "Indexer\Indexer.
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Client", "Client\Client.csproj", "{4232CA30-ABC0-4EB9-A796-5E217719DD88}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Shared", "Shared\Shared.csproj", "{DD8D64EC-86E9-407E-ADC7-AC3F8F9E6A7F}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -30,5 +32,9 @@ Global
 		{4232CA30-ABC0-4EB9-A796-5E217719DD88}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{4232CA30-ABC0-4EB9-A796-5E217719DD88}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{4232CA30-ABC0-4EB9-A796-5E217719DD88}.Release|Any CPU.Build.0 = Release|Any CPU
+		{DD8D64EC-86E9-407E-ADC7-AC3F8F9E6A7F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{DD8D64EC-86E9-407E-ADC7-AC3F8F9E6A7F}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{DD8D64EC-86E9-407E-ADC7-AC3F8F9E6A7F}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{DD8D64EC-86E9-407E-ADC7-AC3F8F9E6A7F}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 EndGlobal