diff --git a/.gitignore b/.gitignore index fef9f6e..d5d7fbf 100644 --- a/.gitignore +++ b/.gitignore @@ -14,4 +14,6 @@ src/Indexer/bin src/Indexer/obj src/Indexer/Scripts/__pycache__ src/Indexer/logs -src/Server/logs \ No newline at end of file +src/Server/logs +src/Shared/bin +src/Shared/obj \ No newline at end of file diff --git a/src/Client/Client.cs b/src/Client/Client.cs index fc88d56..09328e9 100644 --- a/src/Client/Client.cs +++ b/src/Client/Client.cs @@ -7,7 +7,7 @@ using System.Text.Json; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Configuration; using System.Reflection.Metadata.Ecma335; -using Server.Models; +using Shared.Models; namespace Client; @@ -86,7 +86,7 @@ public class Client return await GetUrlAndProcessJson(url); } - public async Task EntityIndexAsync(List jsonEntity) + public async Task EntityIndexAsync(List jsonEntity) { return await EntityIndexAsync(JsonSerializer.Serialize(jsonEntity)); } diff --git a/src/Client/Client.csproj b/src/Client/Client.csproj index 15f7891..e69ffbc 100644 --- a/src/Client/Client.csproj +++ b/src/Client/Client.csproj @@ -1,7 +1,7 @@  - + diff --git a/src/Indexer/Controllers/CallsController.cs b/src/Indexer/Controllers/CallsController.cs new file mode 100644 index 0000000..d048d8c --- /dev/null +++ b/src/Indexer/Controllers/CallsController.cs @@ -0,0 +1,122 @@ +using ElmahCore; +using Microsoft.AspNetCore.Mvc; +using Indexer.Models; + +namespace Indexer.Controllers; + +[ApiController] +[Route("[controller]")] +public class CallsController : ControllerBase +{ + private readonly ILogger _logger; + private readonly IConfiguration _config; + private readonly IConfigurationRoot _configurationRoot; + private readonly WorkerCollection _workerCollection; + + public CallsController(ILogger logger, IConfiguration config, IConfigurationRoot configurationRoot, WorkerCollection workerCollection) + { + _logger = logger; + _config = config; + _configurationRoot = configurationRoot; + _workerCollection = workerCollection; + } + + [HttpGet("List")] + public ActionResult List(string name) + { + bool success = true; + List calls = []; + var configWorkerSection = _config.GetSection("EmbeddingsearchIndexer:Worker"); + _workerCollection.Workers.TryGetValue(name, out Worker? worker); + if (worker is null) + { + success = false; + _logger.LogError("No worker found under the name {name}.", [name]); + HttpContext.RaiseError(new Exception($"No worker found under the name {name}")); + } + else + { + foreach (ICall call in worker.Calls) + { + CallListResult callListResult = new() + { + CallConfig = call.CallConfig, + IsRunning = call.IsRunning, + LastExecution = call.LastExecution, + LastSuccessfulExecution = call.LastSuccessfulExecution, + HealthStatus = call.HealthCheck().Status.ToString() + }; + calls.Add(callListResult); + } + } + return new CallListResults() { Calls = calls, Success = success }; + } + + [HttpGet("Enable")] + public ActionResult Enable(string name) + { + _workerCollection.Workers.TryGetValue(name, out Worker? worker); + if (worker is null) + { + _logger.LogError("Unable to start calls in worker {name} - no running worker with this name.", [name]); + return new WorkerStartResult { Success = false }; + } + _logger.LogInformation("Starting calls in worker {name}.", [name]); + foreach (ICall call in worker.Calls) + { + call.Start(); + } + _logger.LogInformation("Starting calls in worker {name}.", [name]); + return new WorkerStartResult { Success = true }; + } + + [HttpGet("Disable")] + public ActionResult Disable(string name) + { + _workerCollection.Workers.TryGetValue(name, out Worker? worker); + if (worker is null) + { + _logger.LogError("Unable to stop calls in worker {name} - no running worker with this name.", [name]); + return new WorkerStopResult { Success = false }; + } + _logger.LogInformation("Stopping calls in worker {name}.", [name]); + foreach (ICall call in worker.Calls) + { + call.Stop(); + } + _logger.LogInformation("Stopped calls in worker {name}.", [name]); + return new WorkerStopResult { Success = true }; + } + + [HttpGet("Reload")] + public ActionResult Reload() + { + try + { + _logger.LogInformation("Reloading configuration"); + _configurationRoot.Reload(); + _logger.LogInformation("Reloaded configuration"); + _logger.LogInformation("Destroying workers"); + foreach (KeyValuePair workerKVPair in _workerCollection.Workers) + { + Worker worker = workerKVPair.Value; + foreach (ICall call in worker.Calls) + { + call.Stop(); + } + _workerCollection.Workers.Remove(workerKVPair.Key); + _logger.LogInformation("Destroyed worker {workerKVPair.Key}", [workerKVPair.Key]); + } + _logger.LogInformation("Destroyed workers"); + _workerCollection.InitializeWorkers(); + return new WorkerReloadConfigResult { Success = true }; + } + catch (Exception ex) + { + _logger.LogError("Exception {ex.Message} happened while trying to reload the worker configuration.", [ex.Message]); + HttpContext.RaiseError(ex); + return new WorkerReloadConfigResult { Success = false }; + } + } + +} diff --git a/src/Indexer/Controllers/WorkerController.cs b/src/Indexer/Controllers/WorkerController.cs new file mode 100644 index 0000000..c73779a --- /dev/null +++ b/src/Indexer/Controllers/WorkerController.cs @@ -0,0 +1,74 @@ +using ElmahCore; +using Microsoft.AspNetCore.Mvc; +using Indexer.Models; + +namespace Indexer.Controllers; + +[ApiController] +[Route("[controller]")] +public class WorkerController : ControllerBase +{ + private readonly ILogger _logger; + private readonly IConfiguration _config; + private readonly IConfigurationRoot _configurationRoot; + private readonly WorkerCollection _workerCollection; + + public WorkerController(ILogger logger, IConfiguration config, IConfigurationRoot configurationRoot, WorkerCollection workerCollection) + { + _logger = logger; + _config = config; + _configurationRoot = configurationRoot; + _workerCollection = workerCollection; + } + + [HttpGet("List")] + public ActionResult List() // List the workers (and perhaps current execution status, maybe also health status and retry count?) + { + bool success = true; + List workerListResultList = []; + try + { + foreach (KeyValuePair workerKVPair in _workerCollection.Workers) + { + Worker worker = workerKVPair.Value; + WorkerListResult workerListResult = new() + { + Name = worker.Name, + Script = worker.Config.Script, + HealthStatus = worker.HealthCheck().Status.ToString() + }; + workerListResultList.Add(workerListResult); + } + } + catch (Exception ex) + { + success = false; + _logger.LogError("Unable to list workers due to exception: {ex.Message}", [ex.Message]); + HttpContext.RaiseError(ex); + } + WorkerListResults workerListResults = new() + { + Workers = workerListResultList, + Success = success + }; + return workerListResults; + } + + [HttpGet("TriggerUpdate")] + public ActionResult TriggerUpdate(string name) + { + _workerCollection.Workers.TryGetValue(name, out Worker? worker); + if (worker is null) + { + _logger.LogError("Unable to trigger worker {name} - no running worker with this name.", [name]); + return new WorkerTriggerUpdateResult { Success = false }; + } + _logger.LogInformation("triggering worker {name}.", [name]); + ManualTriggerCallbackInfos callbackInfos = new(); + worker.Scriptable.Update(callbackInfos); + _logger.LogInformation("triggered worker {name}.", [name]); + return new WorkerTriggerUpdateResult { Success = true }; + + } + +} diff --git a/src/Indexer/Indexer.csproj b/src/Indexer/Indexer.csproj index 24d5aba..61f1a7a 100644 --- a/src/Indexer/Indexer.csproj +++ b/src/Indexer/Indexer.csproj @@ -16,7 +16,7 @@ - + diff --git a/src/Indexer/IndexerHealthChecks.cs b/src/Indexer/IndexerHealthChecks.cs index 43aa9a8..66d3c31 100644 --- a/src/Indexer/IndexerHealthChecks.cs +++ b/src/Indexer/IndexerHealthChecks.cs @@ -17,8 +17,9 @@ public class WorkerHealthCheck : IHealthCheck bool hasDegraded = false; bool hasUnhealthy = false; Dictionary degradedWorkerList = []; - foreach (Worker worker in _workerCollection.Workers) + foreach (KeyValuePair workerKVPair in _workerCollection.Workers) { + Worker worker = workerKVPair.Value; HealthCheckResult workerHealth = worker.HealthCheck(); hasDegraded |= workerHealth.Status == HealthStatus.Degraded; hasUnhealthy |= workerHealth.Status == HealthStatus.Unhealthy; diff --git a/src/Indexer/Models/Interfaces.cs b/src/Indexer/Models/Interfaces.cs index a157b87..d906069 100644 --- a/src/Indexer/Models/Interfaces.cs +++ b/src/Indexer/Models/Interfaces.cs @@ -7,6 +7,8 @@ public interface IScriptable ILogger _logger { get; set; } void Init(); void Update(ICallbackInfos callbackInfos); + void Stop(); + bool IsScript(string filePath); } diff --git a/src/Indexer/Models/Script.cs b/src/Indexer/Models/Script.cs index 3471ac5..bc167f3 100644 --- a/src/Indexer/Models/Script.cs +++ b/src/Indexer/Models/Script.cs @@ -10,12 +10,14 @@ public class PythonScriptable : IScriptable public PyModule scope; public dynamic sys; public string source; + public bool SourceLoaded { get; set; } public ScriptUpdateInfo UpdateInfo { get; set; } public ILogger _logger { get; set; } public PythonScriptable(ScriptToolSet toolSet, ILogger logger) { _logger = logger; - Runtime.PythonDLL = @"libpython3.12.so"; + SourceLoaded = false; + Runtime.PythonDLL ??= @"libpython3.12.so"; if (!PythonEngine.IsInitialized) { PythonEngine.Initialize(); @@ -39,36 +41,22 @@ public class PythonScriptable : IScriptable public void Init() { - int retryCounter = 0; - retry: - try - { - using (Py.GIL()) - { - pyToolSet = ToolSet.ToPython(); - scope.Set("toolset", pyToolSet); - scope.Exec(source); - scope.Exec("init(toolset)"); - } - } - catch (Exception ex) - { - UpdateInfo = new() { DateTime = DateTime.Now, Successful = false, Exception = ex }; - if (retryCounter < 3) - { - _logger.LogWarning("Unable to init the scriptable - retrying", [ToolSet.filePath, ex]); - retryCounter++; - goto retry; - } - _logger.LogError("Unable to init the scriptable", [ToolSet.filePath, ex]); - throw; - } - UpdateInfo = new() { DateTime = DateTime.Now, Successful = true }; + ExecFunction("init"); } public void Update(ICallbackInfos callbackInfos) { - int retryCounter = 0; + ExecFunction("update"); + } + + public void Stop() + { + ExecFunction("stop"); + } + + public void ExecFunction(string name, ICallbackInfos? callbackInfos = null) + { + int retryCounter = 0; retry: try { @@ -77,7 +65,12 @@ public class PythonScriptable : IScriptable pyToolSet = ToolSet.ToPython(); pyToolSet.SetAttr("callbackInfos", callbackInfos.ToPython()); scope.Set("toolset", pyToolSet); - scope.Exec("update(toolset)"); + if (!SourceLoaded) + { + scope.Exec(source); + SourceLoaded = true; + } + scope.Exec($"{name}(toolset)"); } } catch (Exception ex) @@ -85,12 +78,11 @@ public class PythonScriptable : IScriptable UpdateInfo = new() { DateTime = DateTime.Now, Successful = false, Exception = ex }; if (retryCounter < 3) { - _logger.LogWarning("Execution of script failed to an exception - retrying", [ToolSet.filePath, ex]); + _logger.LogWarning("Execution of {name} function in script {Toolset.filePath} failed to an exception {ex.Message}", [name, ToolSet.filePath, ex.Message]); retryCounter++; goto retry; } - _logger.LogError("Execution of script failed to an exception", [ToolSet.filePath, ex]); - throw; + _logger.LogError("Execution of {name} function in script {Toolset.filePath} failed to an exception {ex.Message}", [name, ToolSet.filePath, ex.Message]); } UpdateInfo = new() { DateTime = DateTime.Now, Successful = true }; } @@ -128,6 +120,8 @@ public class IntervalCallbackInfos : ICallbackInfos } +public class ManualTriggerCallbackInfos : ICallbackInfos {} + public struct ScriptUpdateInfo { public DateTime DateTime { get; set; } diff --git a/src/Indexer/Models/Worker.cs b/src/Indexer/Models/Worker.cs index 2c0c30b..9217c27 100644 --- a/src/Indexer/Models/Worker.cs +++ b/src/Indexer/Models/Worker.cs @@ -1,15 +1,125 @@ using System.Diagnostics; using Microsoft.Extensions.Diagnostics.HealthChecks; +using Indexer.Exceptions; + namespace Indexer.Models; public class WorkerCollection { - public List Workers; + public Dictionary Workers; public List types; - public WorkerCollection() + private readonly ILogger _logger; + private readonly IConfiguration _configuration; + private readonly Client.Client client; + + public WorkerCollection(ILogger logger, IConfiguration configuration, Client.Client client) { Workers = []; types = [typeof(PythonScriptable)]; + _logger = logger; + _configuration = configuration; + this.client = client; + } + + public void InitializeWorkers() + { + _logger.LogInformation("Initializing workers"); + // Load and configure all workers + var sectionMain = _configuration.GetSection("EmbeddingsearchIndexer"); + if (!sectionMain.Exists()) + { + _logger.LogCritical("Unable to load section \"EmbeddingsearchIndexer\""); + throw new IndexerConfigurationException("Unable to load section \"EmbeddingsearchIndexer\""); + } + + WorkerCollectionConfig? sectionWorker = (WorkerCollectionConfig?)sectionMain.Get(typeof(WorkerCollectionConfig)); //GetValue("Worker"); + if (sectionWorker is not null) + { + foreach (WorkerConfig workerConfig in sectionWorker.Worker) + { + ScriptToolSet toolSet = new(workerConfig.Script, client); + InitializeWorker(toolSet, workerConfig); + } + } + else + { + _logger.LogCritical("Unable to load section \"Worker\""); + throw new IndexerConfigurationException("Unable to load section \"Worker\""); + } + _logger.LogInformation("Initialized workers"); + } + + public void InitializeWorker(ScriptToolSet toolSet, WorkerConfig workerConfig) + { + _logger.LogInformation("Initializing worker: {Name}", workerConfig.Name); + Worker worker = new(workerConfig.Name, workerConfig, GetScriptable(toolSet)); + Workers[workerConfig.Name] = worker; + foreach (CallConfig callConfig in workerConfig.Calls) + { + _logger.LogInformation("Initializing call of type: {Type}", callConfig.Type); + + switch (callConfig.Type) + { + case "interval": + if (callConfig.Interval is null) + { + _logger.LogError("Interval not set for a Call in Worker \"{Name}\"", workerConfig.Name); + throw new IndexerConfigurationException($"Interval not set for a Call in Worker \"{workerConfig.Name}\""); + } + var timer = new System.Timers.Timer((double)callConfig.Interval); + timer.AutoReset = true; + timer.Enabled = true; + DateTime now = DateTime.Now; + IntervalCall call = new(timer, worker.Scriptable, _logger, callConfig) + { + LastExecution = now, + LastSuccessfulExecution = now + }; + timer.Elapsed += (sender, e) => + { + try + { + call.LastExecution = DateTime.Now; + worker.Scriptable.Update(new IntervalCallbackInfos() { sender = sender, e = e }); + call.LastSuccessfulExecution = DateTime.Now; + } + catch (Exception ex) + { + _logger.LogError("Exception occurred in a Call of Worker \"{name}\": \"{ex}\"", worker.Name, ex.Message); + } + }; + worker.Calls.Add(call); + break; + case "schedule": // TODO implement scheduled tasks using Quartz + throw new NotImplementedException("schedule not implemented yet"); + case "fileupdate": + if (callConfig.Path is null) + { + _logger.LogError("Path not set for a Call in Worker \"{Name}\"", workerConfig.Name); + throw new IndexerConfigurationException($"Path not set for a Call in Worker \"{workerConfig.Name}\""); + } + throw new NotImplementedException("fileupdate not implemented yet"); + //break; + default: + throw new IndexerConfigurationException($"Unknown Type specified for a Call in Worker \"{workerConfig.Name}\""); + } + } + } + + public IScriptable GetScriptable(ScriptToolSet toolSet) + { + string fileName = toolSet.filePath; + foreach (Type type in types) + { + IScriptable? instance = (IScriptable?)Activator.CreateInstance(type, [toolSet, _logger]); + if (instance is not null && instance.IsScript(fileName)) + { + return instance; + } + } + _logger.LogError("Unable to determine the script's language: \"{fileName}\"", fileName); + + throw new UnknownScriptLanguageException(fileName); } } @@ -76,6 +186,12 @@ public class CallConfig public interface ICall { public HealthCheckResult HealthCheck(); + public void Start(); + public void Stop(); + public bool IsRunning { get; set; } + public CallConfig CallConfig { get; set; } + public DateTime? LastExecution { get; set; } + public DateTime? LastSuccessfulExecution { get; set; } } public class IntervalCall : ICall @@ -83,11 +199,31 @@ public class IntervalCall : ICall public System.Timers.Timer Timer; public IScriptable Scriptable; public ILogger _logger; - public IntervalCall(System.Timers.Timer timer, IScriptable scriptable, ILogger logger) + public bool IsRunning { get; set; } + public CallConfig CallConfig { get; set; } + public DateTime? LastExecution { get; set; } + public DateTime? LastSuccessfulExecution { get; set; } + + public IntervalCall(System.Timers.Timer timer, IScriptable scriptable, ILogger logger, CallConfig callConfig) { Timer = timer; Scriptable = scriptable; _logger = logger; + CallConfig = callConfig; + IsRunning = true; + } + + public void Start() + { + Timer.Start(); + IsRunning = true; + } + + public void Stop() + { + Scriptable.Stop(); + Timer.Stop(); + IsRunning = false; } public HealthCheckResult HealthCheck() @@ -113,6 +249,24 @@ public class IntervalCall : ICall public class ScheduleCall : ICall { + public bool IsRunning { get; set; } + public CallConfig CallConfig { get; set; } + public DateTime? LastExecution { get; set; } + public DateTime? LastSuccessfulExecution { get; set; } + + public ScheduleCall(CallConfig callConfig) + { + CallConfig = callConfig; + } + + public void Start() + { + } + + public void Stop() + { + } + public HealthCheckResult HealthCheck() { return HealthCheckResult.Unhealthy(); // Not implemented yet @@ -121,6 +275,24 @@ public class ScheduleCall : ICall public class FileUpdateCall : ICall { + public bool IsRunning { get; set; } + public CallConfig CallConfig { get; set; } + public DateTime? LastExecution { get; set; } + public DateTime? LastSuccessfulExecution { get; set; } + + public FileUpdateCall(CallConfig callConfig) + { + CallConfig = callConfig; + } + + public void Start() + { + } + + public void Stop() + { + } + public HealthCheckResult HealthCheck() { return HealthCheckResult.Unhealthy(); // Not implemented yet diff --git a/src/Indexer/Models/WorkerResults.cs b/src/Indexer/Models/WorkerResults.cs new file mode 100644 index 0000000..8defcf3 --- /dev/null +++ b/src/Indexer/Models/WorkerResults.cs @@ -0,0 +1,68 @@ +using System.Text.Json.Serialization; +using Microsoft.Extensions.Diagnostics.HealthChecks; + +namespace Indexer.Models; + +public class WorkerListResults +{ + [JsonPropertyName("WorkerList")] + public required List Workers { get; set; } + [JsonPropertyName("Success")] + public required bool Success { get; set; } +} + +public class WorkerListResult +{ + [JsonPropertyName("Name")] + public required string Name { get; set; } + [JsonPropertyName("Script")] + public required string Script { get; set; } + [JsonPropertyName("HealthStatus")] + public required string HealthStatus { get; set; } +} + +public class CallListResults +{ + [JsonPropertyName("Calls")] + public required List Calls { get; set; } + [JsonPropertyName("Success")] + public required bool Success { get; set; } +} + +public class CallListResult +{ + [JsonPropertyName("CallConfig")] + public required CallConfig CallConfig { get; set; } + [JsonPropertyName("IsRunning")] + public required bool IsRunning { get; set; } + [JsonPropertyName("LastExecution")] + public required DateTime? LastExecution { get; set; } + [JsonPropertyName("LastSuccessfulExecution")] + public required DateTime? LastSuccessfulExecution { get; set; } + [JsonPropertyName("HealthStatus")] + public required string HealthStatus { get; set; } +} + +public class WorkerStopResult +{ + [JsonPropertyName("Success")] + public required bool Success { get; set; } +} + +public class WorkerStartResult +{ + [JsonPropertyName("Success")] + public required bool Success { get; set; } +} + +public class WorkerTriggerUpdateResult +{ + [JsonPropertyName("Success")] + public required bool Success { get; set; } +} + +public class WorkerReloadConfigResult +{ + [JsonPropertyName("Success")] + public required bool Success { get; set; } +} diff --git a/src/Indexer/Program.cs b/src/Indexer/Program.cs index 552a335..a91eee0 100644 --- a/src/Indexer/Program.cs +++ b/src/Indexer/Program.cs @@ -3,7 +3,6 @@ using Indexer.Models; using Indexer.Services; using ElmahCore; using ElmahCore.Mvc; -using Server; using ElmahCore.Mvc.Logger; using Serilog; @@ -20,12 +19,12 @@ Log.Logger = new LoggerConfiguration() .CreateLogger(); builder.Logging.AddSerilog(); builder.Services.AddHttpContextAccessor(); +builder.Services.AddSingleton(builder.Configuration); builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddHostedService(); builder.Services.AddHealthChecks() .AddCheck("WorkerHealthCheck"); - builder.Services.AddElmah(Options => { Options.LogPath = builder.Configuration.GetValue("EmbeddingsearchIndexer:Elmah:LogFolder") ?? "~/logs"; @@ -67,9 +66,11 @@ if (app.Environment.IsDevelopment()) } else { - app.UseMiddleware(); + app.UseMiddleware(); } // app.UseHttpsRedirection(); +app.MapControllers(); + app.Run(); diff --git a/src/Indexer/Services/IndexerService.cs b/src/Indexer/Services/IndexerService.cs index 0d49f9a..22c09bf 100644 --- a/src/Indexer/Services/IndexerService.cs +++ b/src/Indexer/Services/IndexerService.cs @@ -6,103 +6,18 @@ namespace Indexer.Services; public class IndexerService : IHostedService { - private readonly WorkerCollection workerCollection; - private readonly IConfiguration _config; - private readonly Client.Client client; + public WorkerCollection workerCollection; public ILogger _logger; - public IndexerService(WorkerCollection workerCollection, IConfiguration configuration, Client.Client client, ILogger logger, IHttpContextAccessor httpContextAccessor) + public IndexerService(WorkerCollection workerCollection, Client.Client client, ILogger logger) { - this._config = configuration; - this.client = client; this.workerCollection = workerCollection; _logger = logger; _logger.LogInformation("Initializing IndexerService"); - // Load and configure all workers - var sectionMain = _config.GetSection("EmbeddingsearchIndexer"); - if (!sectionMain.Exists()) - { - _logger.LogCritical("Unable to load section \"EmbeddingsearchIndexer\""); - throw new IndexerConfigurationException("Unable to load section \"EmbeddingsearchIndexer\""); - } - - WorkerCollectionConfig? sectionWorker = (WorkerCollectionConfig?)sectionMain.Get(typeof(WorkerCollectionConfig)); //GetValue("Worker"); - if (sectionWorker is not null) - { - foreach (WorkerConfig workerConfig in sectionWorker.Worker) - { - _logger.LogInformation("Initializing worker: {Name}", workerConfig.Name); - ScriptToolSet toolSet = new(workerConfig.Script, client); - Worker worker = new(workerConfig.Name, workerConfig, GetScriptable(toolSet)); - workerCollection.Workers.Add(worker); - foreach (CallConfig callConfig in workerConfig.Calls) - { - _logger.LogInformation("Initializing call of type: {Type}", callConfig.Type); - - switch (callConfig.Type) - { - case "interval": - if (callConfig.Interval is null) - { - _logger.LogError("Interval not set for a Call in Worker \"{Name}\"", workerConfig.Name); - throw new IndexerConfigurationException($"Interval not set for a Call in Worker \"{workerConfig.Name}\""); - } - var timer = new System.Timers.Timer((double)callConfig.Interval); - timer.Elapsed += (sender, e) => - { - try - { - worker.Scriptable.Update(new IntervalCallbackInfos() { sender = sender, e = e }); - } - catch (Exception ex) - { - _logger.LogError("Exception occurred in a Call of Worker \"{name}\": \"{ex}\"", worker.Name, ex.Message); - httpContextAccessor.HttpContext.RaiseError(ex); - } - }; - timer.AutoReset = true; - timer.Enabled = true; - IntervalCall call = new(timer, worker.Scriptable, _logger); - worker.Calls.Add(call); - break; - case "schedule": // TODO implement scheduled tasks using Quartz - throw new NotImplementedException("schedule not implemented yet"); - case "fileupdate": - if (callConfig.Path is null) - { - _logger.LogError("Path not set for a Call in Worker \"{Name}\"", workerConfig.Name); - throw new IndexerConfigurationException($"Path not set for a Call in Worker \"{workerConfig.Name}\""); - } - throw new NotImplementedException("fileupdate not implemented yet"); - //break; - default: - throw new IndexerConfigurationException($"Unknown Type specified for a Call in Worker \"{workerConfig.Name}\""); - } - } - } - } - else - { - _logger.LogCritical("Unable to load section \"Worker\""); - throw new IndexerConfigurationException("Unable to load section \"Worker\""); - } + workerCollection.InitializeWorkers(); + _logger.LogInformation("Initialized IndexerService"); } - public IScriptable GetScriptable(ScriptToolSet toolSet) - { - string fileName = toolSet.filePath; - foreach (Type type in workerCollection.types) - { - IScriptable? instance = (IScriptable?)Activator.CreateInstance(type, [toolSet, _logger]); - if (instance is not null && instance.IsScript(fileName)) - { - return instance; - } - } - _logger.LogError("Unable to determine the script's language: \"{fileName}\"", fileName); - - throw new UnknownScriptLanguageException(fileName); - } public Task StartAsync(CancellationToken cancellationToken) { /*foreach (Worker worker in workerCollection.Workers) diff --git a/src/Indexer/appsettings.json b/src/Indexer/appsettings.json index 54557c5..1ccb58f 100644 --- a/src/Indexer/appsettings.json +++ b/src/Indexer/appsettings.json @@ -13,7 +13,7 @@ { "Name": "File", "Args": { "path": "logs/log.txt", "rollingInterval": "Day", "retainedFileCountLimit": 7 } } ], "Properties": { - "Application": "Indexer" + "Application": "Embeddingsearch.Indexer" } }, "EmbeddingsearchIndexer": { diff --git a/src/Server/AIProvider.cs b/src/Server/AIProvider.cs index a84ef6b..4196f32 100644 --- a/src/Server/AIProvider.cs +++ b/src/Server/AIProvider.cs @@ -8,7 +8,7 @@ using Newtonsoft.Json; using Newtonsoft.Json.Linq; using Server.Exceptions; -namespace server; +namespace Server; public class AIProvider { diff --git a/src/Server/Controllers/EntityController.cs b/src/Server/Controllers/EntityController.cs index d896956..0985b26 100644 --- a/src/Server/Controllers/EntityController.cs +++ b/src/Server/Controllers/EntityController.cs @@ -1,7 +1,8 @@ using Microsoft.AspNetCore.Mvc; using System.Text.Json; using System.Text.Json.Nodes; -using Server.Models; +using Shared.Models; +using Server.Helper; namespace Server.Controllers; [ApiController] diff --git a/src/Server/Controllers/SearchdomainController.cs b/src/Server/Controllers/SearchdomainController.cs index 784f01c..d3e54ef 100644 --- a/src/Server/Controllers/SearchdomainController.cs +++ b/src/Server/Controllers/SearchdomainController.cs @@ -1,6 +1,6 @@ using ElmahCore; using Microsoft.AspNetCore.Mvc; -using Server.Models; +using Shared.Models; namespace Server.Controllers; diff --git a/src/Server/Datapoint.cs b/src/Server/Datapoint.cs index 5e54841..5d7567b 100644 --- a/src/Server/Datapoint.cs +++ b/src/Server/Datapoint.cs @@ -6,7 +6,6 @@ using System.Threading.Tasks; using Microsoft.Extensions.AI; using OllamaSharp; using OllamaSharp.Models; -using server; namespace Server; diff --git a/src/Server/Helper/SearchdomainHelper.cs b/src/Server/Helper/SearchdomainHelper.cs index 334401d..79edea3 100644 --- a/src/Server/Helper/SearchdomainHelper.cs +++ b/src/Server/Helper/SearchdomainHelper.cs @@ -2,11 +2,9 @@ using System.Collections.Concurrent; using System.Security.Cryptography; using System.Text; using System.Text.Json; -using MySql.Data.MySqlClient; -using OllamaSharp; -using server; +using Shared.Models; -namespace Server; +namespace Server.Helper; public static class SearchdomainHelper { @@ -41,7 +39,7 @@ public static class SearchdomainHelper } return null; } - + public static List? EntitiesFromJSON(List entityCache, Dictionary> embeddingCache, AIProvider aIProvider, SQLHelper helper, ILogger logger, string json) { List? jsonEntities = JsonSerializer.Deserialize>(json); @@ -78,14 +76,14 @@ public static class SearchdomainHelper }); return [.. retVal]; } - + public static Entity? EntityFromJSON(List entityCache, Dictionary> embeddingCache, AIProvider aIProvider, SQLHelper helper, ILogger logger, JSONEntity jsonEntity) //string json) { Dictionary> embeddingsLUT = []; int? preexistingEntityID = DatabaseHelper.GetEntityID(helper, jsonEntity.Name, jsonEntity.Searchdomain); if (preexistingEntityID is not null) { - lock (helper.connection) + lock (helper.connection) // TODO change this to helper and do A/B tests (i.e. before/after) { Dictionary parameters = new() { diff --git a/src/Server/Program.cs b/src/Server/Program.cs index 4f1ad18..fe97a97 100644 --- a/src/Server/Program.cs +++ b/src/Server/Program.cs @@ -1,7 +1,6 @@ using ElmahCore; using ElmahCore.Mvc; using Serilog; -using server; using Server; using Server.HealthChecks; @@ -72,7 +71,7 @@ if (IsDevelopment || useSwagger) } if (UseMiddleware == true && !IsDevelopment) { - app.UseMiddleware(); + app.UseMiddleware(); } app.UseAuthorization(); diff --git a/src/Server/Searchdomain.cs b/src/Server/Searchdomain.cs index 8b076b0..0f1f9bb 100644 --- a/src/Server/Searchdomain.cs +++ b/src/Server/Searchdomain.cs @@ -17,11 +17,10 @@ using Mysqlx.Resultset; using System.Collections.Immutable; using System.Text.Json; using System.Numerics.Tensors; -using Server; using System.Security.Cryptography; using System.Text; using System.Collections.Concurrent; -using server; +using Server.Helper; namespace Server; diff --git a/src/Server/SearchdomainManager.cs b/src/Server/SearchdomainManager.cs index b4f1623..f5282f3 100644 --- a/src/Server/SearchdomainManager.cs +++ b/src/Server/SearchdomainManager.cs @@ -4,7 +4,6 @@ using OllamaSharp; using Microsoft.IdentityModel.Tokens; using Server.Exceptions; using Server.Migrations; -using server; namespace Server; diff --git a/src/Server/Server.csproj b/src/Server/Server.csproj index e1ceed5..178f4c0 100644 --- a/src/Server/Server.csproj +++ b/src/Server/Server.csproj @@ -22,4 +22,8 @@ + + + + diff --git a/src/Server/appsettings.json b/src/Server/appsettings.json index 54557c5..657168d 100644 --- a/src/Server/appsettings.json +++ b/src/Server/appsettings.json @@ -13,7 +13,7 @@ { "Name": "File", "Args": { "path": "logs/log.txt", "rollingInterval": "Day", "retainedFileCountLimit": 7 } } ], "Properties": { - "Application": "Indexer" + "Application": "Embeddingsearch.Server" } }, "EmbeddingsearchIndexer": { diff --git a/src/Server/ApiKeyMiddleware.cs b/src/Shared/ApiKeyMiddleware.cs similarity index 89% rename from src/Server/ApiKeyMiddleware.cs rename to src/Shared/ApiKeyMiddleware.cs index 1379cde..007a24b 100644 --- a/src/Server/ApiKeyMiddleware.cs +++ b/src/Shared/ApiKeyMiddleware.cs @@ -1,37 +1,39 @@ -using Microsoft.Extensions.Primitives; - -namespace Server; - -public class ApiKeyMiddleware -{ - private readonly RequestDelegate _next; - private readonly IConfiguration _configuration; - - public ApiKeyMiddleware(RequestDelegate next, IConfiguration configuration) - { - _next = next; - _configuration = configuration; - } - - public async Task InvokeAsync(HttpContext context) - { - if (!context.Request.Headers.TryGetValue("X-API-KEY", out StringValues extractedApiKey)) - { - context.Response.StatusCode = 401; - await context.Response.WriteAsync("API Key is missing."); - return; - } - - var validApiKeys = _configuration.GetSection("Embeddingsearch").GetSection("ApiKeys").Get>(); -#pragma warning disable CS8604 - if (validApiKeys == null || !validApiKeys.Contains(extractedApiKey)) // CS8604 extractedApiKey is not null here, but the compiler still thinks that it might be. - { - context.Response.StatusCode = 403; - await context.Response.WriteAsync("Invalid API Key."); - return; - } -#pragma warning restore CS8604 - - await _next(context); - } +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Primitives; + +namespace Shared; + +public class ApiKeyMiddleware +{ + private readonly RequestDelegate _next; + private readonly IConfiguration _configuration; + + public ApiKeyMiddleware(RequestDelegate next, IConfiguration configuration) + { + _next = next; + _configuration = configuration; + } + + public async Task InvokeAsync(HttpContext context) + { + if (!context.Request.Headers.TryGetValue("X-API-KEY", out StringValues extractedApiKey)) + { + context.Response.StatusCode = 401; + await context.Response.WriteAsync("API Key is missing."); + return; + } + + var validApiKeys = _configuration.GetSection("Embeddingsearch").GetSection("ApiKeys").Get>(); +#pragma warning disable CS8604 + if (validApiKeys == null || !validApiKeys.Contains(extractedApiKey)) // CS8604 extractedApiKey is not null here, but the compiler still thinks that it might be. + { + context.Response.StatusCode = 403; + await context.Response.WriteAsync("Invalid API Key."); + return; + } +#pragma warning restore CS8604 + + await _next(context); + } } \ No newline at end of file diff --git a/src/Server/Models/EntityResults.cs b/src/Shared/Models/EntityResults.cs similarity index 98% rename from src/Server/Models/EntityResults.cs rename to src/Shared/Models/EntityResults.cs index 3bf0636..1ad54bf 100644 --- a/src/Server/Models/EntityResults.cs +++ b/src/Shared/Models/EntityResults.cs @@ -1,6 +1,6 @@ using System.Text.Json.Serialization; -namespace Server.Models; +namespace Shared.Models; public class EntityQueryResults diff --git a/src/Server/JSONModels.cs b/src/Shared/Models/JSONModels.cs similarity index 95% rename from src/Server/JSONModels.cs rename to src/Shared/Models/JSONModels.cs index c47827a..d74e4ac 100644 --- a/src/Server/JSONModels.cs +++ b/src/Shared/Models/JSONModels.cs @@ -1,4 +1,4 @@ -namespace Server; +namespace Shared.Models; public class JSONEntity { diff --git a/src/Server/Models/SearchdomainResults.cs b/src/Shared/Models/SearchdomainResults.cs similarity index 97% rename from src/Server/Models/SearchdomainResults.cs rename to src/Shared/Models/SearchdomainResults.cs index f870e66..f98a3d2 100644 --- a/src/Server/Models/SearchdomainResults.cs +++ b/src/Shared/Models/SearchdomainResults.cs @@ -1,6 +1,6 @@ using System.Text.Json.Serialization; -namespace Server.Models; +namespace Shared.Models; public class SearchdomainListResults { diff --git a/src/Shared/Shared.csproj b/src/Shared/Shared.csproj new file mode 100644 index 0000000..9d59b4a --- /dev/null +++ b/src/Shared/Shared.csproj @@ -0,0 +1,11 @@ + + + + net8.0 + enable + enable + + + + + diff --git a/src/embeddingsearch.sln b/src/embeddingsearch.sln index ed571b7..46bd2b2 100644 --- a/src/embeddingsearch.sln +++ b/src/embeddingsearch.sln @@ -9,6 +9,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Indexer", "Indexer\Indexer. EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Client", "Client\Client.csproj", "{4232CA30-ABC0-4EB9-A796-5E217719DD88}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Shared", "Shared\Shared.csproj", "{DD8D64EC-86E9-407E-ADC7-AC3F8F9E6A7F}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -30,5 +32,9 @@ Global {4232CA30-ABC0-4EB9-A796-5E217719DD88}.Debug|Any CPU.Build.0 = Debug|Any CPU {4232CA30-ABC0-4EB9-A796-5E217719DD88}.Release|Any CPU.ActiveCfg = Release|Any CPU {4232CA30-ABC0-4EB9-A796-5E217719DD88}.Release|Any CPU.Build.0 = Release|Any CPU + {DD8D64EC-86E9-407E-ADC7-AC3F8F9E6A7F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {DD8D64EC-86E9-407E-ADC7-AC3F8F9E6A7F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {DD8D64EC-86E9-407E-ADC7-AC3F8F9E6A7F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {DD8D64EC-86E9-407E-ADC7-AC3F8F9E6A7F}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection EndGlobal