lucretia/embeddingsearch
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added CSharp scripting using Roslyn, Cleanup, Moved Scriptables to appropriate folder

Browse Source
This commit is contained in:
LD-Reborn
2025-08-30 23:32:14 +02:00
parent a47d1dca3a
commit d33b2d594f
11 changed files with 379 additions and 129 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
src/Indexer/Models/Interfaces.cs
View File

@@ -1,15 +1,22 @@
namespace Indexer.Models;
public interface IScript
{
int Init(ScriptToolSet toolSet);
int Update(ICallbackInfos callbackInfos);
int Stop();
}
public interface IScriptable
{
ScriptToolSet ToolSet { get; set; }
ScriptUpdateInfo UpdateInfo { get; set; }
ILogger _logger { get; set; }
void Init();
void Update(ICallbackInfos callbackInfos);
void Stop();
int Init();
int Update(ICallbackInfos callbackInfos);
int Stop();
bool IsScript(string filePath);
abstract static bool IsScript(string filePath);
}
public interface ICallbackInfos { }

116
src/Indexer/Models/Script.cs
View File

@@ -1,115 +1,23 @@
using System.Timers;
using Python.Runtime;
namespace Indexer.Models;
public class PythonScriptable : IScriptable
{
public ScriptToolSet ToolSet { get; set; }
public PyObject? pyToolSet;
public PyModule scope;
public dynamic sys;
public string source;
public bool SourceLoaded { get; set; }
public ScriptUpdateInfo UpdateInfo { get; set; }
public ILogger _logger { get; set; }
public PythonScriptable(ScriptToolSet toolSet, ILogger logger)
{
_logger = logger;
SourceLoaded = false;
Runtime.PythonDLL ??= @"libpython3.12.so";
if (!PythonEngine.IsInitialized)
{
PythonEngine.Initialize();
PythonEngine.BeginAllowThreads();
}
ToolSet = toolSet;
source = File.ReadAllText(ToolSet.filePath);
string fullPath = Path.GetFullPath(ToolSet.filePath);
string? scriptDir = Path.GetDirectoryName(fullPath);
using (Py.GIL())
{
scope = Py.CreateScope();
sys = Py.Import("sys");
if (scriptDir is not null)
{
sys.path.append(scriptDir);
}
}
Init();
}
public void Init()
{
ExecFunction("init");
}
public void Update(ICallbackInfos callbackInfos)
{
ExecFunction("update");
}
public void Stop()
{
ExecFunction("stop");
}
public void ExecFunction(string name, ICallbackInfos? callbackInfos = null)
{
int retryCounter = 0;
retry:
try
{
using (Py.GIL())
{
pyToolSet = ToolSet.ToPython();
pyToolSet.SetAttr("callbackInfos", callbackInfos.ToPython());
scope.Set("toolset", pyToolSet);
if (!SourceLoaded)
{
scope.Exec(source);
SourceLoaded = true;
}
scope.Exec($"{name}(toolset)");
}
}
catch (Exception ex)
{
UpdateInfo = new() { DateTime = DateTime.Now, Successful = false, Exception = ex };
if (retryCounter < 3)
{
_logger.LogWarning("Execution of {name} function in script {Toolset.filePath} failed to an exception {ex.Message}", [name, ToolSet.filePath, ex.Message]);
retryCounter++;
goto retry;
}
_logger.LogError("Execution of {name} function in script {Toolset.filePath} failed to an exception {ex.Message}", [name, ToolSet.filePath, ex.Message]);
}
UpdateInfo = new() { DateTime = DateTime.Now, Successful = true };
}
public bool IsScript(string fileName)
{
return fileName.EndsWith(".py");
}
}
/*
TODO Add the following languages
- Javascript
- Golang (reconsider)
*/
public class ScriptToolSet
{
public string filePath;
public Client.Client client;
public ICallbackInfos? callbackInfos;
public string FilePath;
public Client.Client Client;
public ILogger Logger;
public ICallbackInfos? CallbackInfos;
public IConfiguration Configuration;
public string Name;
// IConfiguration - Access to connection strings, ollama, etc. maybe?
public ScriptToolSet(string filePath, Client.Client client)
public ScriptToolSet(string filePath, Client.Client client, ILogger logger, IConfiguration configuration, string name)
{
this.filePath = filePath;
this.client = client;
Configuration = configuration;
Name = name;
FilePath = filePath;
Client = client;
Logger = logger;
}
}

33
src/Indexer/Models/Worker.cs
View File

@@ -1,4 +1,5 @@
using Microsoft.Extensions.Diagnostics.HealthChecks;
using Indexer.Scriptables;
using Indexer.Exceptions;
using Quartz;
using Quartz.Impl;
@@ -16,7 +17,7 @@ public class WorkerCollection
public WorkerCollection(ILogger<WorkerCollection> logger, IConfiguration configuration, Client.Client client)
{
Workers = [];
types = [typeof(PythonScriptable)];
types = [typeof(PythonScriptable), typeof(CSharpScriptable)];
_logger = logger;
_configuration = configuration;
this.client = client;
@@ -38,7 +39,7 @@ public class WorkerCollection
{
foreach (WorkerConfig workerConfig in sectionWorker.Worker)
{
ScriptToolSet toolSet = new(workerConfig.Script, client);
ScriptToolSet toolSet = new(workerConfig.Script, client, _logger, _configuration, workerConfig.Name);
InitializeWorker(toolSet, workerConfig);
}
}
@@ -153,17 +154,23 @@ public class WorkerCollection
public IScriptable GetScriptable(ScriptToolSet toolSet)
{
string fileName = toolSet.filePath;
string fileName = toolSet.FilePath ?? throw new IndexerConfigurationException($"\"Script\" not set for Worker \"{toolSet.Name}\"");
foreach (Type type in types)
{
IScriptable? instance = (IScriptable?)Activator.CreateInstance(type, [toolSet, _logger]);
if (instance is not null && instance.IsScript(fileName))
{
return instance;
System.Reflection.MethodInfo? method = type.GetMethod("IsScript");
bool? isInstance = method is not null ? (bool?)method.Invoke(null, [fileName]) : null;
if (isInstance == true)
{
IScriptable? instance = (IScriptable?)Activator.CreateInstance(type, [toolSet, _logger]);
if (instance is null)
{
_logger.LogError("Unable to initialize script: \"{fileName}\"", fileName);
throw new Exception($"Unable to initialize script: \"{fileName}\"");
}
return instance;
}
}
}
_logger.LogError("Unable to determine the script's language: \"{fileName}\"", fileName);
throw new UnknownScriptLanguageException(fileName);
}
}
@@ -346,8 +353,8 @@ public class IntervalCall : ICall
{
if (!Scriptable.UpdateInfo.Successful)
{
_logger.LogWarning("HealthCheck revealed: The last execution of \"{name}\" was not successful", Scriptable.ToolSet.filePath);
return HealthCheckResult.Unhealthy($"HealthCheck revealed: The last execution of \"{Scriptable.ToolSet.filePath}\" was not successful");
_logger.LogWarning("HealthCheck revealed: The last execution of \"{name}\" was not successful", Scriptable.ToolSet.FilePath);
return HealthCheckResult.Unhealthy($"HealthCheck revealed: The last execution of \"{Scriptable.ToolSet.FilePath}\" was not successful");
}
double timerInterval = Timer.Interval; // In ms
DateTime lastRunDateTime = Scriptable.UpdateInfo.DateTime;
@@ -355,8 +362,8 @@ public class IntervalCall : ICall
double millisecondsSinceLastExecution = now.Subtract(lastRunDateTime).TotalMilliseconds;
if (millisecondsSinceLastExecution >= 2 * timerInterval)
{
_logger.LogWarning("HealthCheck revealed: Since the last execution of \"{name}\" more than twice the interval has passed", Scriptable.ToolSet.filePath);
return HealthCheckResult.Unhealthy($"HealthCheck revealed: Since the last execution of \"{Scriptable.ToolSet.filePath}\" more than twice the interval has passed");
_logger.LogWarning("HealthCheck revealed: Since the last execution of \"{name}\" more than twice the interval has passed", Scriptable.ToolSet.FilePath);
return HealthCheckResult.Unhealthy($"HealthCheck revealed: Since the last execution of \"{Scriptable.ToolSet.FilePath}\" more than twice the interval has passed");
}
return HealthCheckResult.Healthy();
}