Fixed Indexer SwaggerUI, Added worker & call management endpoints to Indexer, restructuring to improve project separation
This commit is contained in:
122
src/Indexer/Controllers/CallsController.cs
Normal file
122
src/Indexer/Controllers/CallsController.cs
Normal file
@@ -0,0 +1,122 @@
|
||||
using ElmahCore;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Indexer.Models;
|
||||
|
||||
namespace Indexer.Controllers;
|
||||
|
||||
[ApiController]
|
||||
[Route("[controller]")]
|
||||
public class CallsController : ControllerBase
|
||||
{
|
||||
private readonly ILogger<WorkerController> _logger;
|
||||
private readonly IConfiguration _config;
|
||||
private readonly IConfigurationRoot _configurationRoot;
|
||||
private readonly WorkerCollection _workerCollection;
|
||||
|
||||
public CallsController(ILogger<WorkerController> logger, IConfiguration config, IConfigurationRoot configurationRoot, WorkerCollection workerCollection)
|
||||
{
|
||||
_logger = logger;
|
||||
_config = config;
|
||||
_configurationRoot = configurationRoot;
|
||||
_workerCollection = workerCollection;
|
||||
}
|
||||
|
||||
[HttpGet("List")]
|
||||
public ActionResult<CallListResults> List(string name)
|
||||
{
|
||||
bool success = true;
|
||||
List<CallListResult> calls = [];
|
||||
var configWorkerSection = _config.GetSection("EmbeddingsearchIndexer:Worker");
|
||||
_workerCollection.Workers.TryGetValue(name, out Worker? worker);
|
||||
if (worker is null)
|
||||
{
|
||||
success = false;
|
||||
_logger.LogError("No worker found under the name {name}.", [name]);
|
||||
HttpContext.RaiseError(new Exception($"No worker found under the name {name}"));
|
||||
}
|
||||
else
|
||||
{
|
||||
foreach (ICall call in worker.Calls)
|
||||
{
|
||||
CallListResult callListResult = new()
|
||||
{
|
||||
CallConfig = call.CallConfig,
|
||||
IsRunning = call.IsRunning,
|
||||
LastExecution = call.LastExecution,
|
||||
LastSuccessfulExecution = call.LastSuccessfulExecution,
|
||||
HealthStatus = call.HealthCheck().Status.ToString()
|
||||
};
|
||||
calls.Add(callListResult);
|
||||
}
|
||||
}
|
||||
return new CallListResults() { Calls = calls, Success = success };
|
||||
}
|
||||
|
||||
[HttpGet("Enable")]
|
||||
public ActionResult<WorkerStartResult> Enable(string name)
|
||||
{
|
||||
_workerCollection.Workers.TryGetValue(name, out Worker? worker);
|
||||
if (worker is null)
|
||||
{
|
||||
_logger.LogError("Unable to start calls in worker {name} - no running worker with this name.", [name]);
|
||||
return new WorkerStartResult { Success = false };
|
||||
}
|
||||
_logger.LogInformation("Starting calls in worker {name}.", [name]);
|
||||
foreach (ICall call in worker.Calls)
|
||||
{
|
||||
call.Start();
|
||||
}
|
||||
_logger.LogInformation("Starting calls in worker {name}.", [name]);
|
||||
return new WorkerStartResult { Success = true };
|
||||
}
|
||||
|
||||
[HttpGet("Disable")]
|
||||
public ActionResult<WorkerStopResult> Disable(string name)
|
||||
{
|
||||
_workerCollection.Workers.TryGetValue(name, out Worker? worker);
|
||||
if (worker is null)
|
||||
{
|
||||
_logger.LogError("Unable to stop calls in worker {name} - no running worker with this name.", [name]);
|
||||
return new WorkerStopResult { Success = false };
|
||||
}
|
||||
_logger.LogInformation("Stopping calls in worker {name}.", [name]);
|
||||
foreach (ICall call in worker.Calls)
|
||||
{
|
||||
call.Stop();
|
||||
}
|
||||
_logger.LogInformation("Stopped calls in worker {name}.", [name]);
|
||||
return new WorkerStopResult { Success = true };
|
||||
}
|
||||
|
||||
[HttpGet("Reload")]
|
||||
public ActionResult<WorkerReloadConfigResult> Reload()
|
||||
{
|
||||
try
|
||||
{
|
||||
_logger.LogInformation("Reloading configuration");
|
||||
_configurationRoot.Reload();
|
||||
_logger.LogInformation("Reloaded configuration");
|
||||
_logger.LogInformation("Destroying workers");
|
||||
foreach (KeyValuePair<string, Worker> workerKVPair in _workerCollection.Workers)
|
||||
{
|
||||
Worker worker = workerKVPair.Value;
|
||||
foreach (ICall call in worker.Calls)
|
||||
{
|
||||
call.Stop();
|
||||
}
|
||||
_workerCollection.Workers.Remove(workerKVPair.Key);
|
||||
_logger.LogInformation("Destroyed worker {workerKVPair.Key}", [workerKVPair.Key]);
|
||||
}
|
||||
_logger.LogInformation("Destroyed workers");
|
||||
_workerCollection.InitializeWorkers();
|
||||
return new WorkerReloadConfigResult { Success = true };
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError("Exception {ex.Message} happened while trying to reload the worker configuration.", [ex.Message]);
|
||||
HttpContext.RaiseError(ex);
|
||||
return new WorkerReloadConfigResult { Success = false };
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
74
src/Indexer/Controllers/WorkerController.cs
Normal file
74
src/Indexer/Controllers/WorkerController.cs
Normal file
@@ -0,0 +1,74 @@
|
||||
using ElmahCore;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Indexer.Models;
|
||||
|
||||
namespace Indexer.Controllers;
|
||||
|
||||
[ApiController]
|
||||
[Route("[controller]")]
|
||||
public class WorkerController : ControllerBase
|
||||
{
|
||||
private readonly ILogger<WorkerController> _logger;
|
||||
private readonly IConfiguration _config;
|
||||
private readonly IConfigurationRoot _configurationRoot;
|
||||
private readonly WorkerCollection _workerCollection;
|
||||
|
||||
public WorkerController(ILogger<WorkerController> logger, IConfiguration config, IConfigurationRoot configurationRoot, WorkerCollection workerCollection)
|
||||
{
|
||||
_logger = logger;
|
||||
_config = config;
|
||||
_configurationRoot = configurationRoot;
|
||||
_workerCollection = workerCollection;
|
||||
}
|
||||
|
||||
[HttpGet("List")]
|
||||
public ActionResult<WorkerListResults> List() // List the workers (and perhaps current execution status, maybe also health status and retry count?)
|
||||
{
|
||||
bool success = true;
|
||||
List<WorkerListResult> workerListResultList = [];
|
||||
try
|
||||
{
|
||||
foreach (KeyValuePair<string, Worker> workerKVPair in _workerCollection.Workers)
|
||||
{
|
||||
Worker worker = workerKVPair.Value;
|
||||
WorkerListResult workerListResult = new()
|
||||
{
|
||||
Name = worker.Name,
|
||||
Script = worker.Config.Script,
|
||||
HealthStatus = worker.HealthCheck().Status.ToString()
|
||||
};
|
||||
workerListResultList.Add(workerListResult);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
success = false;
|
||||
_logger.LogError("Unable to list workers due to exception: {ex.Message}", [ex.Message]);
|
||||
HttpContext.RaiseError(ex);
|
||||
}
|
||||
WorkerListResults workerListResults = new()
|
||||
{
|
||||
Workers = workerListResultList,
|
||||
Success = success
|
||||
};
|
||||
return workerListResults;
|
||||
}
|
||||
|
||||
[HttpGet("TriggerUpdate")]
|
||||
public ActionResult<WorkerTriggerUpdateResult> TriggerUpdate(string name)
|
||||
{
|
||||
_workerCollection.Workers.TryGetValue(name, out Worker? worker);
|
||||
if (worker is null)
|
||||
{
|
||||
_logger.LogError("Unable to trigger worker {name} - no running worker with this name.", [name]);
|
||||
return new WorkerTriggerUpdateResult { Success = false };
|
||||
}
|
||||
_logger.LogInformation("triggering worker {name}.", [name]);
|
||||
ManualTriggerCallbackInfos callbackInfos = new();
|
||||
worker.Scriptable.Update(callbackInfos);
|
||||
_logger.LogInformation("triggered worker {name}.", [name]);
|
||||
return new WorkerTriggerUpdateResult { Success = true };
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
@@ -16,7 +16,7 @@
|
||||
<PackageReference Include="Pythonnet" Version="3.0.5" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\Server\Server.csproj" />
|
||||
<ProjectReference Include="..\Client\Client.csproj" />
|
||||
<ProjectReference Include="..\Shared\Shared.csproj" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
|
||||
@@ -17,8 +17,9 @@ public class WorkerHealthCheck : IHealthCheck
|
||||
bool hasDegraded = false;
|
||||
bool hasUnhealthy = false;
|
||||
Dictionary<string, HealthStatus> degradedWorkerList = [];
|
||||
foreach (Worker worker in _workerCollection.Workers)
|
||||
foreach (KeyValuePair<string, Worker> workerKVPair in _workerCollection.Workers)
|
||||
{
|
||||
Worker worker = workerKVPair.Value;
|
||||
HealthCheckResult workerHealth = worker.HealthCheck();
|
||||
hasDegraded |= workerHealth.Status == HealthStatus.Degraded;
|
||||
hasUnhealthy |= workerHealth.Status == HealthStatus.Unhealthy;
|
||||
|
||||
@@ -7,6 +7,8 @@ public interface IScriptable
|
||||
ILogger _logger { get; set; }
|
||||
void Init();
|
||||
void Update(ICallbackInfos callbackInfos);
|
||||
void Stop();
|
||||
|
||||
bool IsScript(string filePath);
|
||||
}
|
||||
|
||||
|
||||
@@ -10,12 +10,14 @@ public class PythonScriptable : IScriptable
|
||||
public PyModule scope;
|
||||
public dynamic sys;
|
||||
public string source;
|
||||
public bool SourceLoaded { get; set; }
|
||||
public ScriptUpdateInfo UpdateInfo { get; set; }
|
||||
public ILogger _logger { get; set; }
|
||||
public PythonScriptable(ScriptToolSet toolSet, ILogger logger)
|
||||
{
|
||||
_logger = logger;
|
||||
Runtime.PythonDLL = @"libpython3.12.so";
|
||||
SourceLoaded = false;
|
||||
Runtime.PythonDLL ??= @"libpython3.12.so";
|
||||
if (!PythonEngine.IsInitialized)
|
||||
{
|
||||
PythonEngine.Initialize();
|
||||
@@ -39,36 +41,22 @@ public class PythonScriptable : IScriptable
|
||||
|
||||
public void Init()
|
||||
{
|
||||
int retryCounter = 0;
|
||||
retry:
|
||||
try
|
||||
{
|
||||
using (Py.GIL())
|
||||
{
|
||||
pyToolSet = ToolSet.ToPython();
|
||||
scope.Set("toolset", pyToolSet);
|
||||
scope.Exec(source);
|
||||
scope.Exec("init(toolset)");
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
UpdateInfo = new() { DateTime = DateTime.Now, Successful = false, Exception = ex };
|
||||
if (retryCounter < 3)
|
||||
{
|
||||
_logger.LogWarning("Unable to init the scriptable - retrying", [ToolSet.filePath, ex]);
|
||||
retryCounter++;
|
||||
goto retry;
|
||||
}
|
||||
_logger.LogError("Unable to init the scriptable", [ToolSet.filePath, ex]);
|
||||
throw;
|
||||
}
|
||||
UpdateInfo = new() { DateTime = DateTime.Now, Successful = true };
|
||||
ExecFunction("init");
|
||||
}
|
||||
|
||||
public void Update(ICallbackInfos callbackInfos)
|
||||
{
|
||||
int retryCounter = 0;
|
||||
ExecFunction("update");
|
||||
}
|
||||
|
||||
public void Stop()
|
||||
{
|
||||
ExecFunction("stop");
|
||||
}
|
||||
|
||||
public void ExecFunction(string name, ICallbackInfos? callbackInfos = null)
|
||||
{
|
||||
int retryCounter = 0;
|
||||
retry:
|
||||
try
|
||||
{
|
||||
@@ -77,7 +65,12 @@ public class PythonScriptable : IScriptable
|
||||
pyToolSet = ToolSet.ToPython();
|
||||
pyToolSet.SetAttr("callbackInfos", callbackInfos.ToPython());
|
||||
scope.Set("toolset", pyToolSet);
|
||||
scope.Exec("update(toolset)");
|
||||
if (!SourceLoaded)
|
||||
{
|
||||
scope.Exec(source);
|
||||
SourceLoaded = true;
|
||||
}
|
||||
scope.Exec($"{name}(toolset)");
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
@@ -85,12 +78,11 @@ public class PythonScriptable : IScriptable
|
||||
UpdateInfo = new() { DateTime = DateTime.Now, Successful = false, Exception = ex };
|
||||
if (retryCounter < 3)
|
||||
{
|
||||
_logger.LogWarning("Execution of script failed to an exception - retrying", [ToolSet.filePath, ex]);
|
||||
_logger.LogWarning("Execution of {name} function in script {Toolset.filePath} failed to an exception {ex.Message}", [name, ToolSet.filePath, ex.Message]);
|
||||
retryCounter++;
|
||||
goto retry;
|
||||
}
|
||||
_logger.LogError("Execution of script failed to an exception", [ToolSet.filePath, ex]);
|
||||
throw;
|
||||
_logger.LogError("Execution of {name} function in script {Toolset.filePath} failed to an exception {ex.Message}", [name, ToolSet.filePath, ex.Message]);
|
||||
}
|
||||
UpdateInfo = new() { DateTime = DateTime.Now, Successful = true };
|
||||
}
|
||||
@@ -128,6 +120,8 @@ public class IntervalCallbackInfos : ICallbackInfos
|
||||
|
||||
}
|
||||
|
||||
public class ManualTriggerCallbackInfos : ICallbackInfos {}
|
||||
|
||||
public struct ScriptUpdateInfo
|
||||
{
|
||||
public DateTime DateTime { get; set; }
|
||||
|
||||
@@ -1,15 +1,125 @@
|
||||
using System.Diagnostics;
|
||||
using Microsoft.Extensions.Diagnostics.HealthChecks;
|
||||
using Indexer.Exceptions;
|
||||
|
||||
namespace Indexer.Models;
|
||||
|
||||
public class WorkerCollection
|
||||
{
|
||||
public List<Worker> Workers;
|
||||
public Dictionary<string, Worker> Workers;
|
||||
public List<Type> types;
|
||||
public WorkerCollection()
|
||||
private readonly ILogger _logger;
|
||||
private readonly IConfiguration _configuration;
|
||||
private readonly Client.Client client;
|
||||
|
||||
public WorkerCollection(ILogger<WorkerCollection> logger, IConfiguration configuration, Client.Client client)
|
||||
{
|
||||
Workers = [];
|
||||
types = [typeof(PythonScriptable)];
|
||||
_logger = logger;
|
||||
_configuration = configuration;
|
||||
this.client = client;
|
||||
}
|
||||
|
||||
public void InitializeWorkers()
|
||||
{
|
||||
_logger.LogInformation("Initializing workers");
|
||||
// Load and configure all workers
|
||||
var sectionMain = _configuration.GetSection("EmbeddingsearchIndexer");
|
||||
if (!sectionMain.Exists())
|
||||
{
|
||||
_logger.LogCritical("Unable to load section \"EmbeddingsearchIndexer\"");
|
||||
throw new IndexerConfigurationException("Unable to load section \"EmbeddingsearchIndexer\"");
|
||||
}
|
||||
|
||||
WorkerCollectionConfig? sectionWorker = (WorkerCollectionConfig?)sectionMain.Get(typeof(WorkerCollectionConfig)); //GetValue<WorkerCollectionConfig>("Worker");
|
||||
if (sectionWorker is not null)
|
||||
{
|
||||
foreach (WorkerConfig workerConfig in sectionWorker.Worker)
|
||||
{
|
||||
ScriptToolSet toolSet = new(workerConfig.Script, client);
|
||||
InitializeWorker(toolSet, workerConfig);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
_logger.LogCritical("Unable to load section \"Worker\"");
|
||||
throw new IndexerConfigurationException("Unable to load section \"Worker\"");
|
||||
}
|
||||
_logger.LogInformation("Initialized workers");
|
||||
}
|
||||
|
||||
public void InitializeWorker(ScriptToolSet toolSet, WorkerConfig workerConfig)
|
||||
{
|
||||
_logger.LogInformation("Initializing worker: {Name}", workerConfig.Name);
|
||||
Worker worker = new(workerConfig.Name, workerConfig, GetScriptable(toolSet));
|
||||
Workers[workerConfig.Name] = worker;
|
||||
foreach (CallConfig callConfig in workerConfig.Calls)
|
||||
{
|
||||
_logger.LogInformation("Initializing call of type: {Type}", callConfig.Type);
|
||||
|
||||
switch (callConfig.Type)
|
||||
{
|
||||
case "interval":
|
||||
if (callConfig.Interval is null)
|
||||
{
|
||||
_logger.LogError("Interval not set for a Call in Worker \"{Name}\"", workerConfig.Name);
|
||||
throw new IndexerConfigurationException($"Interval not set for a Call in Worker \"{workerConfig.Name}\"");
|
||||
}
|
||||
var timer = new System.Timers.Timer((double)callConfig.Interval);
|
||||
timer.AutoReset = true;
|
||||
timer.Enabled = true;
|
||||
DateTime now = DateTime.Now;
|
||||
IntervalCall call = new(timer, worker.Scriptable, _logger, callConfig)
|
||||
{
|
||||
LastExecution = now,
|
||||
LastSuccessfulExecution = now
|
||||
};
|
||||
timer.Elapsed += (sender, e) =>
|
||||
{
|
||||
try
|
||||
{
|
||||
call.LastExecution = DateTime.Now;
|
||||
worker.Scriptable.Update(new IntervalCallbackInfos() { sender = sender, e = e });
|
||||
call.LastSuccessfulExecution = DateTime.Now;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError("Exception occurred in a Call of Worker \"{name}\": \"{ex}\"", worker.Name, ex.Message);
|
||||
}
|
||||
};
|
||||
worker.Calls.Add(call);
|
||||
break;
|
||||
case "schedule": // TODO implement scheduled tasks using Quartz
|
||||
throw new NotImplementedException("schedule not implemented yet");
|
||||
case "fileupdate":
|
||||
if (callConfig.Path is null)
|
||||
{
|
||||
_logger.LogError("Path not set for a Call in Worker \"{Name}\"", workerConfig.Name);
|
||||
throw new IndexerConfigurationException($"Path not set for a Call in Worker \"{workerConfig.Name}\"");
|
||||
}
|
||||
throw new NotImplementedException("fileupdate not implemented yet");
|
||||
//break;
|
||||
default:
|
||||
throw new IndexerConfigurationException($"Unknown Type specified for a Call in Worker \"{workerConfig.Name}\"");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public IScriptable GetScriptable(ScriptToolSet toolSet)
|
||||
{
|
||||
string fileName = toolSet.filePath;
|
||||
foreach (Type type in types)
|
||||
{
|
||||
IScriptable? instance = (IScriptable?)Activator.CreateInstance(type, [toolSet, _logger]);
|
||||
if (instance is not null && instance.IsScript(fileName))
|
||||
{
|
||||
return instance;
|
||||
}
|
||||
}
|
||||
_logger.LogError("Unable to determine the script's language: \"{fileName}\"", fileName);
|
||||
|
||||
throw new UnknownScriptLanguageException(fileName);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -76,6 +186,12 @@ public class CallConfig
|
||||
public interface ICall
|
||||
{
|
||||
public HealthCheckResult HealthCheck();
|
||||
public void Start();
|
||||
public void Stop();
|
||||
public bool IsRunning { get; set; }
|
||||
public CallConfig CallConfig { get; set; }
|
||||
public DateTime? LastExecution { get; set; }
|
||||
public DateTime? LastSuccessfulExecution { get; set; }
|
||||
}
|
||||
|
||||
public class IntervalCall : ICall
|
||||
@@ -83,11 +199,31 @@ public class IntervalCall : ICall
|
||||
public System.Timers.Timer Timer;
|
||||
public IScriptable Scriptable;
|
||||
public ILogger _logger;
|
||||
public IntervalCall(System.Timers.Timer timer, IScriptable scriptable, ILogger logger)
|
||||
public bool IsRunning { get; set; }
|
||||
public CallConfig CallConfig { get; set; }
|
||||
public DateTime? LastExecution { get; set; }
|
||||
public DateTime? LastSuccessfulExecution { get; set; }
|
||||
|
||||
public IntervalCall(System.Timers.Timer timer, IScriptable scriptable, ILogger logger, CallConfig callConfig)
|
||||
{
|
||||
Timer = timer;
|
||||
Scriptable = scriptable;
|
||||
_logger = logger;
|
||||
CallConfig = callConfig;
|
||||
IsRunning = true;
|
||||
}
|
||||
|
||||
public void Start()
|
||||
{
|
||||
Timer.Start();
|
||||
IsRunning = true;
|
||||
}
|
||||
|
||||
public void Stop()
|
||||
{
|
||||
Scriptable.Stop();
|
||||
Timer.Stop();
|
||||
IsRunning = false;
|
||||
}
|
||||
|
||||
public HealthCheckResult HealthCheck()
|
||||
@@ -113,6 +249,24 @@ public class IntervalCall : ICall
|
||||
|
||||
public class ScheduleCall : ICall
|
||||
{
|
||||
public bool IsRunning { get; set; }
|
||||
public CallConfig CallConfig { get; set; }
|
||||
public DateTime? LastExecution { get; set; }
|
||||
public DateTime? LastSuccessfulExecution { get; set; }
|
||||
|
||||
public ScheduleCall(CallConfig callConfig)
|
||||
{
|
||||
CallConfig = callConfig;
|
||||
}
|
||||
|
||||
public void Start()
|
||||
{
|
||||
}
|
||||
|
||||
public void Stop()
|
||||
{
|
||||
}
|
||||
|
||||
public HealthCheckResult HealthCheck()
|
||||
{
|
||||
return HealthCheckResult.Unhealthy(); // Not implemented yet
|
||||
@@ -121,6 +275,24 @@ public class ScheduleCall : ICall
|
||||
|
||||
public class FileUpdateCall : ICall
|
||||
{
|
||||
public bool IsRunning { get; set; }
|
||||
public CallConfig CallConfig { get; set; }
|
||||
public DateTime? LastExecution { get; set; }
|
||||
public DateTime? LastSuccessfulExecution { get; set; }
|
||||
|
||||
public FileUpdateCall(CallConfig callConfig)
|
||||
{
|
||||
CallConfig = callConfig;
|
||||
}
|
||||
|
||||
public void Start()
|
||||
{
|
||||
}
|
||||
|
||||
public void Stop()
|
||||
{
|
||||
}
|
||||
|
||||
public HealthCheckResult HealthCheck()
|
||||
{
|
||||
return HealthCheckResult.Unhealthy(); // Not implemented yet
|
||||
|
||||
68
src/Indexer/Models/WorkerResults.cs
Normal file
68
src/Indexer/Models/WorkerResults.cs
Normal file
@@ -0,0 +1,68 @@
|
||||
using System.Text.Json.Serialization;
|
||||
using Microsoft.Extensions.Diagnostics.HealthChecks;
|
||||
|
||||
namespace Indexer.Models;
|
||||
|
||||
public class WorkerListResults
|
||||
{
|
||||
[JsonPropertyName("WorkerList")]
|
||||
public required List<WorkerListResult> Workers { get; set; }
|
||||
[JsonPropertyName("Success")]
|
||||
public required bool Success { get; set; }
|
||||
}
|
||||
|
||||
public class WorkerListResult
|
||||
{
|
||||
[JsonPropertyName("Name")]
|
||||
public required string Name { get; set; }
|
||||
[JsonPropertyName("Script")]
|
||||
public required string Script { get; set; }
|
||||
[JsonPropertyName("HealthStatus")]
|
||||
public required string HealthStatus { get; set; }
|
||||
}
|
||||
|
||||
public class CallListResults
|
||||
{
|
||||
[JsonPropertyName("Calls")]
|
||||
public required List<CallListResult> Calls { get; set; }
|
||||
[JsonPropertyName("Success")]
|
||||
public required bool Success { get; set; }
|
||||
}
|
||||
|
||||
public class CallListResult
|
||||
{
|
||||
[JsonPropertyName("CallConfig")]
|
||||
public required CallConfig CallConfig { get; set; }
|
||||
[JsonPropertyName("IsRunning")]
|
||||
public required bool IsRunning { get; set; }
|
||||
[JsonPropertyName("LastExecution")]
|
||||
public required DateTime? LastExecution { get; set; }
|
||||
[JsonPropertyName("LastSuccessfulExecution")]
|
||||
public required DateTime? LastSuccessfulExecution { get; set; }
|
||||
[JsonPropertyName("HealthStatus")]
|
||||
public required string HealthStatus { get; set; }
|
||||
}
|
||||
|
||||
public class WorkerStopResult
|
||||
{
|
||||
[JsonPropertyName("Success")]
|
||||
public required bool Success { get; set; }
|
||||
}
|
||||
|
||||
public class WorkerStartResult
|
||||
{
|
||||
[JsonPropertyName("Success")]
|
||||
public required bool Success { get; set; }
|
||||
}
|
||||
|
||||
public class WorkerTriggerUpdateResult
|
||||
{
|
||||
[JsonPropertyName("Success")]
|
||||
public required bool Success { get; set; }
|
||||
}
|
||||
|
||||
public class WorkerReloadConfigResult
|
||||
{
|
||||
[JsonPropertyName("Success")]
|
||||
public required bool Success { get; set; }
|
||||
}
|
||||
@@ -3,7 +3,6 @@ using Indexer.Models;
|
||||
using Indexer.Services;
|
||||
using ElmahCore;
|
||||
using ElmahCore.Mvc;
|
||||
using Server;
|
||||
using ElmahCore.Mvc.Logger;
|
||||
using Serilog;
|
||||
|
||||
@@ -20,12 +19,12 @@ Log.Logger = new LoggerConfiguration()
|
||||
.CreateLogger();
|
||||
builder.Logging.AddSerilog();
|
||||
builder.Services.AddHttpContextAccessor();
|
||||
builder.Services.AddSingleton<IConfigurationRoot>(builder.Configuration);
|
||||
builder.Services.AddSingleton<Client.Client>();
|
||||
builder.Services.AddSingleton<WorkerCollection>();
|
||||
builder.Services.AddHostedService<IndexerService>();
|
||||
builder.Services.AddHealthChecks()
|
||||
.AddCheck<WorkerHealthCheck>("WorkerHealthCheck");
|
||||
|
||||
builder.Services.AddElmah<XmlFileErrorLog>(Options =>
|
||||
{
|
||||
Options.LogPath = builder.Configuration.GetValue<string>("EmbeddingsearchIndexer:Elmah:LogFolder") ?? "~/logs";
|
||||
@@ -67,9 +66,11 @@ if (app.Environment.IsDevelopment())
|
||||
}
|
||||
else
|
||||
{
|
||||
app.UseMiddleware<ApiKeyMiddleware>();
|
||||
app.UseMiddleware<Shared.ApiKeyMiddleware>();
|
||||
}
|
||||
|
||||
// app.UseHttpsRedirection();
|
||||
|
||||
app.MapControllers();
|
||||
|
||||
app.Run();
|
||||
|
||||
@@ -6,103 +6,18 @@ namespace Indexer.Services;
|
||||
|
||||
public class IndexerService : IHostedService
|
||||
{
|
||||
private readonly WorkerCollection workerCollection;
|
||||
private readonly IConfiguration _config;
|
||||
private readonly Client.Client client;
|
||||
public WorkerCollection workerCollection;
|
||||
public ILogger<IndexerService> _logger;
|
||||
|
||||
public IndexerService(WorkerCollection workerCollection, IConfiguration configuration, Client.Client client, ILogger<IndexerService> logger, IHttpContextAccessor httpContextAccessor)
|
||||
public IndexerService(WorkerCollection workerCollection, Client.Client client, ILogger<IndexerService> logger)
|
||||
{
|
||||
this._config = configuration;
|
||||
this.client = client;
|
||||
this.workerCollection = workerCollection;
|
||||
_logger = logger;
|
||||
_logger.LogInformation("Initializing IndexerService");
|
||||
// Load and configure all workers
|
||||
var sectionMain = _config.GetSection("EmbeddingsearchIndexer");
|
||||
if (!sectionMain.Exists())
|
||||
{
|
||||
_logger.LogCritical("Unable to load section \"EmbeddingsearchIndexer\"");
|
||||
throw new IndexerConfigurationException("Unable to load section \"EmbeddingsearchIndexer\"");
|
||||
}
|
||||
|
||||
WorkerCollectionConfig? sectionWorker = (WorkerCollectionConfig?)sectionMain.Get(typeof(WorkerCollectionConfig)); //GetValue<WorkerCollectionConfig>("Worker");
|
||||
if (sectionWorker is not null)
|
||||
{
|
||||
foreach (WorkerConfig workerConfig in sectionWorker.Worker)
|
||||
{
|
||||
_logger.LogInformation("Initializing worker: {Name}", workerConfig.Name);
|
||||
ScriptToolSet toolSet = new(workerConfig.Script, client);
|
||||
Worker worker = new(workerConfig.Name, workerConfig, GetScriptable(toolSet));
|
||||
workerCollection.Workers.Add(worker);
|
||||
foreach (CallConfig callConfig in workerConfig.Calls)
|
||||
{
|
||||
_logger.LogInformation("Initializing call of type: {Type}", callConfig.Type);
|
||||
|
||||
switch (callConfig.Type)
|
||||
{
|
||||
case "interval":
|
||||
if (callConfig.Interval is null)
|
||||
{
|
||||
_logger.LogError("Interval not set for a Call in Worker \"{Name}\"", workerConfig.Name);
|
||||
throw new IndexerConfigurationException($"Interval not set for a Call in Worker \"{workerConfig.Name}\"");
|
||||
}
|
||||
var timer = new System.Timers.Timer((double)callConfig.Interval);
|
||||
timer.Elapsed += (sender, e) =>
|
||||
{
|
||||
try
|
||||
{
|
||||
worker.Scriptable.Update(new IntervalCallbackInfos() { sender = sender, e = e });
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError("Exception occurred in a Call of Worker \"{name}\": \"{ex}\"", worker.Name, ex.Message);
|
||||
httpContextAccessor.HttpContext.RaiseError(ex);
|
||||
}
|
||||
};
|
||||
timer.AutoReset = true;
|
||||
timer.Enabled = true;
|
||||
IntervalCall call = new(timer, worker.Scriptable, _logger);
|
||||
worker.Calls.Add(call);
|
||||
break;
|
||||
case "schedule": // TODO implement scheduled tasks using Quartz
|
||||
throw new NotImplementedException("schedule not implemented yet");
|
||||
case "fileupdate":
|
||||
if (callConfig.Path is null)
|
||||
{
|
||||
_logger.LogError("Path not set for a Call in Worker \"{Name}\"", workerConfig.Name);
|
||||
throw new IndexerConfigurationException($"Path not set for a Call in Worker \"{workerConfig.Name}\"");
|
||||
}
|
||||
throw new NotImplementedException("fileupdate not implemented yet");
|
||||
//break;
|
||||
default:
|
||||
throw new IndexerConfigurationException($"Unknown Type specified for a Call in Worker \"{workerConfig.Name}\"");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
_logger.LogCritical("Unable to load section \"Worker\"");
|
||||
throw new IndexerConfigurationException("Unable to load section \"Worker\"");
|
||||
}
|
||||
workerCollection.InitializeWorkers();
|
||||
_logger.LogInformation("Initialized IndexerService");
|
||||
}
|
||||
|
||||
public IScriptable GetScriptable(ScriptToolSet toolSet)
|
||||
{
|
||||
string fileName = toolSet.filePath;
|
||||
foreach (Type type in workerCollection.types)
|
||||
{
|
||||
IScriptable? instance = (IScriptable?)Activator.CreateInstance(type, [toolSet, _logger]);
|
||||
if (instance is not null && instance.IsScript(fileName))
|
||||
{
|
||||
return instance;
|
||||
}
|
||||
}
|
||||
_logger.LogError("Unable to determine the script's language: \"{fileName}\"", fileName);
|
||||
|
||||
throw new UnknownScriptLanguageException(fileName);
|
||||
}
|
||||
public Task StartAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
/*foreach (Worker worker in workerCollection.Workers)
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
{ "Name": "File", "Args": { "path": "logs/log.txt", "rollingInterval": "Day", "retainedFileCountLimit": 7 } }
|
||||
],
|
||||
"Properties": {
|
||||
"Application": "Indexer"
|
||||
"Application": "Embeddingsearch.Indexer"
|
||||
}
|
||||
},
|
||||
"EmbeddingsearchIndexer": {
|
||||
|
||||
Reference in New Issue
Block a user