Fixed Indexer SwaggerUI, Added worker & call management endpoints to Indexer, restructuring to improve project separation

src/Indexer/Models/Interfaces.cs

@@ -7,6 +7,8 @@ public interface IScriptable
     ILogger _logger { get; set; }
     void Init();
     void Update(ICallbackInfos callbackInfos);
+    void Stop();
+
     bool IsScript(string filePath);
 }
 

src/Indexer/Models/Script.cs

@@ -10,12 +10,14 @@ public class PythonScriptable : IScriptable
     public PyModule scope;
     public dynamic sys;
     public string source;
+    public bool SourceLoaded { get; set; }
     public ScriptUpdateInfo UpdateInfo { get; set; }
     public ILogger _logger { get; set; }
     public PythonScriptable(ScriptToolSet toolSet, ILogger logger)
     {
         _logger = logger;
-        Runtime.PythonDLL = @"libpython3.12.so";
+        SourceLoaded = false;
+        Runtime.PythonDLL ??= @"libpython3.12.so";
         if (!PythonEngine.IsInitialized)
         {
             PythonEngine.Initialize();
@@ -39,36 +41,22 @@ public class PythonScriptable : IScriptable
 
     public void Init()
     {
-        int retryCounter = 0;
-        retry:
-        try
-        {
-            using (Py.GIL())
-            {
-                pyToolSet = ToolSet.ToPython();
-                scope.Set("toolset", pyToolSet);
-                scope.Exec(source);
-                scope.Exec("init(toolset)");
-            }
-        }
-        catch (Exception ex)
-        {
-            UpdateInfo = new() { DateTime = DateTime.Now, Successful = false, Exception = ex };
-            if (retryCounter < 3)
-            {
-                _logger.LogWarning("Unable to init the scriptable - retrying", [ToolSet.filePath, ex]);
-                retryCounter++;
-                goto retry;
-            }
-            _logger.LogError("Unable to init the scriptable", [ToolSet.filePath, ex]);
-            throw;
-        }
-        UpdateInfo = new() { DateTime = DateTime.Now, Successful = true };
+        ExecFunction("init");
     }
 
     public void Update(ICallbackInfos callbackInfos)
     {
-        int retryCounter = 0;
+        ExecFunction("update");
+    }
+
+    public void Stop()
+    {
+        ExecFunction("stop");
+    }
+
+    public void ExecFunction(string name, ICallbackInfos? callbackInfos = null)
+    {
+       int retryCounter = 0;
         retry:
         try
         {
@@ -77,7 +65,12 @@ public class PythonScriptable : IScriptable
                 pyToolSet = ToolSet.ToPython();
                 pyToolSet.SetAttr("callbackInfos", callbackInfos.ToPython());
                 scope.Set("toolset", pyToolSet);
-                scope.Exec("update(toolset)");
+                if (!SourceLoaded)
+                {
+                    scope.Exec(source);
+                    SourceLoaded = true;
+                }
+                scope.Exec($"{name}(toolset)");
             }
         }
         catch (Exception ex)
@@ -85,12 +78,11 @@ public class PythonScriptable : IScriptable
             UpdateInfo = new() { DateTime = DateTime.Now, Successful = false, Exception = ex };
             if (retryCounter < 3)
             {
-                _logger.LogWarning("Execution of script failed to an exception - retrying", [ToolSet.filePath, ex]);
+                _logger.LogWarning("Execution of {name} function in script {Toolset.filePath} failed to an exception {ex.Message}", [name, ToolSet.filePath, ex.Message]);
                 retryCounter++;
                 goto retry;
             }
-            _logger.LogError("Execution of script failed to an exception", [ToolSet.filePath, ex]);
-            throw;
+            _logger.LogError("Execution of {name} function in script {Toolset.filePath} failed to an exception {ex.Message}", [name, ToolSet.filePath, ex.Message]);
         }
         UpdateInfo = new() { DateTime = DateTime.Now, Successful = true };
     }
@@ -128,6 +120,8 @@ public class IntervalCallbackInfos : ICallbackInfos
 
 }
 
+public class ManualTriggerCallbackInfos : ICallbackInfos {}
+
 public struct ScriptUpdateInfo
 {
     public DateTime DateTime { get; set; }

src/Indexer/Models/Worker.cs

@@ -1,15 +1,125 @@
 using System.Diagnostics;
 using Microsoft.Extensions.Diagnostics.HealthChecks;
+using Indexer.Exceptions;
+
 namespace Indexer.Models;
 
 public class WorkerCollection
 {
-    public List<Worker> Workers;
+    public Dictionary<string, Worker> Workers;
     public List<Type> types;
-    public WorkerCollection()
+    private readonly ILogger _logger;
+    private readonly IConfiguration _configuration;
+    private readonly Client.Client client;
+
+    public WorkerCollection(ILogger<WorkerCollection> logger, IConfiguration configuration, Client.Client client)
     {
         Workers = [];
         types = [typeof(PythonScriptable)];
+        _logger = logger;
+        _configuration = configuration;
+        this.client = client;
+    }
+
+    public void InitializeWorkers()
+    {
+        _logger.LogInformation("Initializing workers");
+        // Load and configure all workers
+        var sectionMain = _configuration.GetSection("EmbeddingsearchIndexer");
+        if (!sectionMain.Exists())
+        {
+            _logger.LogCritical("Unable to load section \"EmbeddingsearchIndexer\"");
+            throw new IndexerConfigurationException("Unable to load section \"EmbeddingsearchIndexer\"");
+        }
+
+        WorkerCollectionConfig? sectionWorker = (WorkerCollectionConfig?)sectionMain.Get(typeof(WorkerCollectionConfig)); //GetValue<WorkerCollectionConfig>("Worker");
+        if (sectionWorker is not null)
+        {
+            foreach (WorkerConfig workerConfig in sectionWorker.Worker)
+            {
+                ScriptToolSet toolSet = new(workerConfig.Script, client);
+                InitializeWorker(toolSet, workerConfig);
+            }
+        }
+        else
+        {
+            _logger.LogCritical("Unable to load section \"Worker\"");
+            throw new IndexerConfigurationException("Unable to load section \"Worker\"");
+        }
+        _logger.LogInformation("Initialized workers");
+    }
+
+    public void InitializeWorker(ScriptToolSet toolSet, WorkerConfig workerConfig)
+    {
+        _logger.LogInformation("Initializing worker: {Name}", workerConfig.Name);
+        Worker worker = new(workerConfig.Name, workerConfig, GetScriptable(toolSet));
+        Workers[workerConfig.Name] = worker;
+        foreach (CallConfig callConfig in workerConfig.Calls)
+        {
+            _logger.LogInformation("Initializing call of type: {Type}", callConfig.Type);
+
+            switch (callConfig.Type)
+            {
+                case "interval":
+                    if (callConfig.Interval is null)
+                    {
+                        _logger.LogError("Interval not set for a Call in Worker \"{Name}\"", workerConfig.Name);
+                        throw new IndexerConfigurationException($"Interval not set for a Call in Worker \"{workerConfig.Name}\"");
+                    }
+                    var timer = new System.Timers.Timer((double)callConfig.Interval);
+                    timer.AutoReset = true;
+                    timer.Enabled = true;
+                    DateTime now = DateTime.Now;
+                    IntervalCall call = new(timer, worker.Scriptable, _logger, callConfig)
+                    {
+                        LastExecution = now,
+                        LastSuccessfulExecution = now
+                    };
+                    timer.Elapsed += (sender, e) =>
+                    {
+                        try
+                        {
+                            call.LastExecution = DateTime.Now;
+                            worker.Scriptable.Update(new IntervalCallbackInfos() { sender = sender, e = e });
+                            call.LastSuccessfulExecution = DateTime.Now;
+                        }
+                        catch (Exception ex)
+                        {
+                            _logger.LogError("Exception occurred in a Call of Worker \"{name}\": \"{ex}\"", worker.Name, ex.Message);
+                        }
+                    };
+                    worker.Calls.Add(call);
+                    break;
+                case "schedule": // TODO implement scheduled tasks using Quartz
+                    throw new NotImplementedException("schedule not implemented yet");
+                case "fileupdate":
+                    if (callConfig.Path is null)
+                    {
+                        _logger.LogError("Path not set for a Call in Worker \"{Name}\"", workerConfig.Name);
+                        throw new IndexerConfigurationException($"Path not set for a Call in Worker \"{workerConfig.Name}\"");
+                    }
+                    throw new NotImplementedException("fileupdate not implemented yet");
+                //break;
+                default:
+                    throw new IndexerConfigurationException($"Unknown Type specified for a Call in Worker \"{workerConfig.Name}\"");
+            }
+        }
+    }
+
+    public IScriptable GetScriptable(ScriptToolSet toolSet)
+    {
+        string fileName = toolSet.filePath;
+        foreach (Type type in types)
+        {
+            IScriptable? instance = (IScriptable?)Activator.CreateInstance(type, [toolSet, _logger]);
+            if (instance is not null && instance.IsScript(fileName))
+            {
+                return instance;
+            }
+        }
+        _logger.LogError("Unable to determine the script's language: \"{fileName}\"", fileName);
+
+        throw new UnknownScriptLanguageException(fileName);
     }
 }
 
@@ -76,6 +186,12 @@ public class CallConfig
 public interface ICall
 {
     public HealthCheckResult HealthCheck();
+    public void Start();
+    public void Stop();
+    public bool IsRunning { get; set; }
+    public CallConfig CallConfig { get; set; }
+    public DateTime? LastExecution { get; set; }
+    public DateTime? LastSuccessfulExecution { get; set; }
 }
 
 public class IntervalCall : ICall
@@ -83,11 +199,31 @@ public class IntervalCall : ICall
     public System.Timers.Timer Timer;
     public IScriptable Scriptable;
     public ILogger _logger;
-    public IntervalCall(System.Timers.Timer timer, IScriptable scriptable, ILogger logger)
+    public bool IsRunning { get; set; }
+    public CallConfig CallConfig { get; set; }
+    public DateTime? LastExecution { get; set; }
+    public DateTime? LastSuccessfulExecution { get; set; }
+
+    public IntervalCall(System.Timers.Timer timer, IScriptable scriptable, ILogger logger, CallConfig callConfig)
     {
         Timer = timer;
         Scriptable = scriptable;
         _logger = logger;
+        CallConfig = callConfig;
+        IsRunning = true;
+    }
+
+    public void Start()
+    {
+        Timer.Start();
+        IsRunning = true;
+    }
+
+    public void Stop()
+    {
+        Scriptable.Stop();
+        Timer.Stop();
+        IsRunning = false;
     }
 
     public HealthCheckResult HealthCheck()
@@ -113,6 +249,24 @@ public class IntervalCall : ICall
 
 public class ScheduleCall : ICall
 {
+    public bool IsRunning { get; set; }
+    public CallConfig CallConfig { get; set; }
+    public DateTime? LastExecution { get; set; }
+    public DateTime? LastSuccessfulExecution { get; set; }
+
+    public ScheduleCall(CallConfig callConfig)
+    {
+        CallConfig = callConfig;
+    }
+
+    public void Start()
+    {
+    }
+
+    public void Stop()
+    {
+    }
+
     public HealthCheckResult HealthCheck()
     {
         return HealthCheckResult.Unhealthy(); // Not implemented yet
@@ -121,6 +275,24 @@ public class ScheduleCall : ICall
 
 public class FileUpdateCall : ICall
 {
+    public bool IsRunning { get; set; }
+    public CallConfig CallConfig { get; set; }
+    public DateTime? LastExecution { get; set; }
+    public DateTime? LastSuccessfulExecution { get; set; }
+
+    public FileUpdateCall(CallConfig callConfig)
+    {
+        CallConfig = callConfig;
+    }
+
+    public void Start()
+    {
+    }
+
+    public void Stop()
+    {
+    }
+
     public HealthCheckResult HealthCheck()
     {
         return HealthCheckResult.Unhealthy(); // Not implemented yet

src/Indexer/Models/WorkerResults.cs

@@ -0,0 +1,68 @@
+using System.Text.Json.Serialization;
+using Microsoft.Extensions.Diagnostics.HealthChecks;
+
+namespace Indexer.Models;
+
+public class WorkerListResults
+{
+    [JsonPropertyName("WorkerList")]
+    public required List<WorkerListResult> Workers { get; set; }
+    [JsonPropertyName("Success")]
+    public required bool Success { get; set; }
+}
+
+public class WorkerListResult
+{
+    [JsonPropertyName("Name")]
+    public required string Name { get; set; }
+    [JsonPropertyName("Script")]
+    public required string Script { get; set; }
+    [JsonPropertyName("HealthStatus")]
+    public required string HealthStatus { get; set; }
+}
+
+public class CallListResults
+{
+    [JsonPropertyName("Calls")]
+    public required List<CallListResult> Calls { get; set; }
+    [JsonPropertyName("Success")]
+    public required bool Success { get; set; }
+}
+
+public class CallListResult
+{
+    [JsonPropertyName("CallConfig")]
+    public required CallConfig CallConfig { get; set; }
+    [JsonPropertyName("IsRunning")]
+    public required bool IsRunning { get; set; }
+    [JsonPropertyName("LastExecution")]
+    public required DateTime? LastExecution { get; set; }
+    [JsonPropertyName("LastSuccessfulExecution")]
+    public required DateTime? LastSuccessfulExecution { get; set; }
+    [JsonPropertyName("HealthStatus")]
+    public required string HealthStatus { get; set; }
+}
+
+public class WorkerStopResult
+{
+    [JsonPropertyName("Success")]
+    public required bool Success { get; set; }
+}
+
+public class WorkerStartResult
+{
+    [JsonPropertyName("Success")]
+    public required bool Success { get; set; }
+}
+
+public class WorkerTriggerUpdateResult
+{
+    [JsonPropertyName("Success")]
+    public required bool Success { get; set; }
+}
+
+public class WorkerReloadConfigResult
+{
+    [JsonPropertyName("Success")]
+    public required bool Success { get; set; }
+}