Added Serilog logging, added Elmah error logging, added health checks

src/Indexer/Models/Interfaces.cs

@@ -3,6 +3,8 @@ namespace Indexer.Models;
 public interface IScriptable
 {
     ScriptToolSet ToolSet { get; set; }
+    ScriptUpdateInfo UpdateInfo { get; set; }
+    ILogger Logger { get; set; }
     void Init();
     void Update(ICallbackInfos callbackInfos);
     bool IsScript(string filePath);

src/Indexer/Models/Script.cs

@@ -1,4 +1,3 @@
-using System.Text.Json;
 using System.Timers;
 using Python.Runtime;
 
@@ -11,8 +10,11 @@ public class PythonScriptable : IScriptable
     public PyModule scope;
     public dynamic sys;
     public string source;
-    public PythonScriptable(ScriptToolSet toolSet)
+    public ScriptUpdateInfo UpdateInfo { get; set; }
+    public ILogger Logger { get; set; }
+    public PythonScriptable(ScriptToolSet toolSet, ILogger logger)
     {
+        Logger = logger;
         Runtime.PythonDLL = @"libpython3.12.so";
         if (!PythonEngine.IsInitialized)
         {
@@ -37,24 +39,60 @@ public class PythonScriptable : IScriptable
 
     public void Init()
     {
-        using (Py.GIL())
+        int retryCounter = 0;
+        retry:
+        try
         {
-            pyToolSet = ToolSet.ToPython();
-            scope.Set("toolset", pyToolSet);
-            scope.Exec(source);
-            scope.Exec("init(toolset)");
+            using (Py.GIL())
+            {
+                pyToolSet = ToolSet.ToPython();
+                scope.Set("toolset", pyToolSet);
+                scope.Exec(source);
+                scope.Exec("init(toolset)");
+            }
         }
+        catch (Exception ex)
+        {
+            UpdateInfo = new() { DateTime = DateTime.Now, Successful = false, Exception = ex };
+            if (retryCounter < 3)
+            {
+                Logger.LogWarning("Unable to init the scriptable - retrying", [ToolSet.filePath, ex]);
+                retryCounter++;
+                goto retry;
+            }
+            Logger.LogError("Unable to init the scriptable", [ToolSet.filePath, ex]);
+            throw;
+        }
+        UpdateInfo = new() { DateTime = DateTime.Now, Successful = true };
     }
 
     public void Update(ICallbackInfos callbackInfos)
     {
-        using (Py.GIL())
+        int retryCounter = 0;
+        retry:
+        try
         {
-            pyToolSet = ToolSet.ToPython();
-            pyToolSet.SetAttr("callbackInfos", callbackInfos.ToPython());
-            scope.Set("toolset", pyToolSet);
-            scope.Exec("update(toolset)");
+            using (Py.GIL())
+            {
+                pyToolSet = ToolSet.ToPython();
+                pyToolSet.SetAttr("callbackInfos", callbackInfos.ToPython());
+                scope.Set("toolset", pyToolSet);
+                scope.Exec("update(toolset)");
+            }
         }
+        catch (Exception ex)
+        {
+            UpdateInfo = new() { DateTime = DateTime.Now, Successful = false, Exception = ex };
+            if (retryCounter < 3)
+            {
+                Logger.LogWarning("Execution of script failed to an exception - retrying", [ToolSet.filePath, ex]);
+                retryCounter++;
+                goto retry;
+            }
+            Logger.LogError("Execution of script failed to an exception", [ToolSet.filePath, ex]);
+            throw;
+        }
+        UpdateInfo = new() { DateTime = DateTime.Now, Successful = true };
     }
 
     public bool IsScript(string fileName)
@@ -87,5 +125,12 @@ public class IntervalCallbackInfos : ICallbackInfos
 {
     public object? sender;
     public required ElapsedEventArgs e;
-     
+
+}
+
+public struct ScriptUpdateInfo
+{
+    public DateTime DateTime { get; set; }
+    public bool Successful { get; set; }
+    public Exception? Exception { get; set; }
 }

src/Indexer/Models/Worker.cs

@@ -1,3 +1,5 @@
+using System.Diagnostics;
+using Microsoft.Extensions.Diagnostics.HealthChecks;
 namespace Indexer.Models;
 
 public class WorkerCollection
@@ -13,13 +15,41 @@ public class WorkerCollection
 
 public class Worker
 {
+    public string Name { get; set; }
     public WorkerConfig Config { get; set; }
     public IScriptable Scriptable { get; set; }
+    public List<ICall> Calls { get; set; }
 
-    public Worker(WorkerConfig workerConfig, IScriptable scriptable)
+    public Worker(string Name, WorkerConfig workerConfig, IScriptable scriptable)
     {
+        this.Name = Name;
         this.Config = workerConfig;
         this.Scriptable = scriptable;
+        Calls = [];
+    }
+
+    public HealthCheckResult HealthCheck()
+    {
+        bool hasDegraded = false;
+        bool hasUnhealthy = false;
+        foreach (ICall call in Calls)
+        {
+            HealthCheckResult callHealth = call.HealthCheck();
+            if (callHealth.Status != HealthStatus.Healthy)
+            {
+                hasDegraded |= callHealth.Status == HealthStatus.Degraded;
+                hasUnhealthy |= callHealth.Status == HealthStatus.Unhealthy;
+            }
+        }
+        if (hasUnhealthy)
+        {
+            return HealthCheckResult.Unhealthy(); // TODO: Retrieve and forward the error message for each call
+        }
+        else if (hasDegraded)
+        {
+            return HealthCheckResult.Degraded();
+        }
+        return HealthCheckResult.Healthy();
     }
 }
 
@@ -33,13 +63,64 @@ public class WorkerConfig
     public required string Name { get; set; }
     public required List<string> Searchdomains { get; set; }
     public required string Script { get; set; }
-    public required List<Call> Calls { get; set; }
+    public required List<CallConfig> Calls { get; set; }
 }
 
-public class Call
+public class CallConfig
 {
     public required string Type { get; set; }
     public long? Interval { get; set; } // For Type: Interval
     public string? Path { get; set; } // For Type: FileSystemWatcher
 }
 
+public interface ICall
+{
+    public HealthCheckResult HealthCheck();
+}
+
+public class IntervalCall : ICall
+{
+    public System.Timers.Timer Timer;
+    public IScriptable Scriptable;
+
+    public IntervalCall(System.Timers.Timer timer, IScriptable scriptable)
+    {
+        Timer = timer;
+        Scriptable = scriptable;
+    }
+
+    public HealthCheckResult HealthCheck()
+    {
+        if (!Scriptable.UpdateInfo.Successful)
+        {
+            Debug.WriteLine(Scriptable.UpdateInfo.Exception);
+            return HealthCheckResult.Unhealthy();
+        }
+        double timerInterval = Timer.Interval; // In ms
+        DateTime lastRunDateTime = Scriptable.UpdateInfo.DateTime;
+        DateTime now = DateTime.Now;
+        double millisecondsSinceLastExecution = now.Subtract(lastRunDateTime).TotalMilliseconds;
+        if (millisecondsSinceLastExecution >= 2 * timerInterval)
+        {
+            return HealthCheckResult.Unhealthy();
+        }
+        return HealthCheckResult.Healthy();
+    }
+
+}
+
+public class ScheduleCall : ICall
+{
+    public HealthCheckResult HealthCheck()
+    {
+        return HealthCheckResult.Unhealthy(); // Not implemented yet
+    }
+}
+
+public class FileUpdateCall : ICall
+{
+    public HealthCheckResult HealthCheck()
+    {
+        return HealthCheckResult.Unhealthy(); // Not implemented yet
+    }
+}