lucretia/embeddingsearch
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added CSharp scripting using Roslyn, Cleanup, Moved Scriptables to appropriate folder

Browse Source
This commit is contained in:
LD-Reborn
2025-08-30 23:32:14 +02:00
parent a47d1dca3a
commit d33b2d594f
11 changed files with 379 additions and 129 deletions
Download Patch File Download Diff File Expand all files Collapse all files

68
docs/Indexer.md
View File

@@ -177,8 +177,10 @@ probmethod_entity = "DictionaryWeightedAverage:{\"title\": 2, \"filename\": 0.1,
To ease scripting, tools.py contains all definitions of the .NET objects passed to the script. This includes attributes and methods.
These are not yet defined in a way that makes them 100% interactible with the Dotnet CLR, meaning some methods that require anything more than strings or other simple data types to be passed are not yet supported. (WIP)
### Required elements
Here is an overview of required elements by example:
### Supported file extensions
- .py
### Code elements
Here is an overview of code elements by example:
```python
from tools import * # Import all tools that are provided for ease of scripting
@@ -197,6 +199,68 @@ Currently, `Toolset`, as provided by the IndexerService to the Python script, co
1. (only for `update`, not `init`) `callbackInfos` - an object that provides all information regarding the callback. (e.g. what file was updated)
2. `client` - a .NET object that has the functions as described in `src/Indexer/Scripts/tools.py`. It's the client that - according to the configuration - communicates with the search server and executes the API calls.
3. `filePath` - the path to the script, as specified in the configuration
## C# (Roslyn)
### Supported file extensions
- .csx
### Code elements
**important hint:** As shown in the last two lines of the example code, simply declaring the class is **not** enough. One must also return an object of said class!
```csharp
// #load directives are disregarded at compile time. Its use is currently for syntax highlighting only
#load "../../Client/Client.cs"
#load "../Models/Script.cs"
#load "../Models/Interfaces.cs"
#load "../Models/WorkerResults.cs"
#load "../../Shared/Models/SearchdomainResults.cs"
#load "../../Shared/Models/JSONModels.cs"
#load "../../Shared/Models/EntityResults.cs"
using Shared.Models;
using System.Collections.Generic;
using System.Linq;
using Microsoft.Extensions.Logging;
// Required: a class that extends Indexer.Models.IScript
public class ExampleScript : Indexer.Models.IScript
{
public Indexer.Models.ScriptToolSet ToolSet;
public Client.Client client;
// Optional: constructor
public ExampleScript()
{
//System.Console.WriteLine("DEBUG@example.cs - Constructor"); // logger not passed here yet
}
// Required: Init method as required to extend IScript
public int Init(Indexer.Models.ScriptToolSet toolSet)
{
ToolSet = toolSet;
ToolSet.Logger.LogInformation("DEBUG@example.csx - Init");
return 0; // Required: int error value return
}
// Required: Updaet method as required to extend IScript
public int Update(Indexer.Models.ICallbackInfos callbackInfos)
{
ToolSet.Logger.LogInformation("DEBUG@example.csx - Update");
EntityQueryResults test = ToolSet.Client.EntityQueryAsync(defaultSearchdomain, "DNA").Result;
var firstResult = test.Results.ToArray()[0];
ToolSet.Logger.LogInformation(firstResult.Name);
ToolSet.Logger.LogInformation(firstResult.Value.ToString());
return 0; // Required: int error value return
}
// Required: int error value return
public int Stop()
{
ToolSet.Logger.LogInformation("DEBUG@example.csx - Stop");
return 0; // Required: int error value return
}
}
// Required: return an instance of your IScript-extending class
return new ExampleScript();
```
## Golang
TODO
## Javascript

1
src/Indexer/Indexer.csproj
View File

@@ -9,6 +9,7 @@
<ItemGroup>
<PackageReference Include="ElmahCore" Version="2.1.2" />
<PackageReference Include="Microsoft.AspNetCore.OpenApi" Version="8.0.14" />
<PackageReference Include="Microsoft.CodeAnalysis.CSharp.Scripting" Version="4.14.0" />
<PackageReference Include="Microsoft.Extensions.Hosting" Version="9.0.8" />
<PackageReference Include="Quartz.Extensions.Hosting" Version="3.15.0" />
<PackageReference Include="Serilog.AspNetCore" Version="9.0.0" />

15
src/Indexer/Models/Interfaces.cs
View File

@@ -1,15 +1,22 @@
namespace Indexer.Models;
public interface IScript
{
int Init(ScriptToolSet toolSet);
int Update(ICallbackInfos callbackInfos);
int Stop();
}
public interface IScriptable
{
ScriptToolSet ToolSet { get; set; }
ScriptUpdateInfo UpdateInfo { get; set; }
ILogger _logger { get; set; }
void Init();
void Update(ICallbackInfos callbackInfos);
void Stop();
int Init();
int Update(ICallbackInfos callbackInfos);
int Stop();
bool IsScript(string filePath);
abstract static bool IsScript(string filePath);
}
public interface ICallbackInfos { }

116
src/Indexer/Models/Script.cs
View File

@@ -1,115 +1,23 @@
using System.Timers;
using Python.Runtime;
namespace Indexer.Models;
public class PythonScriptable : IScriptable
{
public ScriptToolSet ToolSet { get; set; }
public PyObject? pyToolSet;
public PyModule scope;
public dynamic sys;
public string source;
public bool SourceLoaded { get; set; }
public ScriptUpdateInfo UpdateInfo { get; set; }
public ILogger _logger { get; set; }
public PythonScriptable(ScriptToolSet toolSet, ILogger logger)
{
_logger = logger;
SourceLoaded = false;
Runtime.PythonDLL ??= @"libpython3.12.so";
if (!PythonEngine.IsInitialized)
{
PythonEngine.Initialize();
PythonEngine.BeginAllowThreads();
}
ToolSet = toolSet;
source = File.ReadAllText(ToolSet.filePath);
string fullPath = Path.GetFullPath(ToolSet.filePath);
string? scriptDir = Path.GetDirectoryName(fullPath);
using (Py.GIL())
{
scope = Py.CreateScope();
sys = Py.Import("sys");
if (scriptDir is not null)
{
sys.path.append(scriptDir);
}
}
Init();
}
public void Init()
{
ExecFunction("init");
}
public void Update(ICallbackInfos callbackInfos)
{
ExecFunction("update");
}
public void Stop()
{
ExecFunction("stop");
}
public void ExecFunction(string name, ICallbackInfos? callbackInfos = null)
{
int retryCounter = 0;
retry:
try
{
using (Py.GIL())
{
pyToolSet = ToolSet.ToPython();
pyToolSet.SetAttr("callbackInfos", callbackInfos.ToPython());
scope.Set("toolset", pyToolSet);
if (!SourceLoaded)
{
scope.Exec(source);
SourceLoaded = true;
}
scope.Exec($"{name}(toolset)");
}
}
catch (Exception ex)
{
UpdateInfo = new() { DateTime = DateTime.Now, Successful = false, Exception = ex };
if (retryCounter < 3)
{
_logger.LogWarning("Execution of {name} function in script {Toolset.filePath} failed to an exception {ex.Message}", [name, ToolSet.filePath, ex.Message]);
retryCounter++;
goto retry;
}
_logger.LogError("Execution of {name} function in script {Toolset.filePath} failed to an exception {ex.Message}", [name, ToolSet.filePath, ex.Message]);
}
UpdateInfo = new() { DateTime = DateTime.Now, Successful = true };
}
public bool IsScript(string fileName)
{
return fileName.EndsWith(".py");
}
}
/*
TODO Add the following languages
- Javascript
- Golang (reconsider)
*/
public class ScriptToolSet
{
public string filePath;
public Client.Client client;
public ICallbackInfos? callbackInfos;
public string FilePath;
public Client.Client Client;
public ILogger Logger;
public ICallbackInfos? CallbackInfos;
public IConfiguration Configuration;
public string Name;
// IConfiguration - Access to connection strings, ollama, etc. maybe?
public ScriptToolSet(string filePath, Client.Client client)
public ScriptToolSet(string filePath, Client.Client client, ILogger logger, IConfiguration configuration, string name)
{
this.filePath = filePath;
this.client = client;
Configuration = configuration;
Name = name;
FilePath = filePath;
Client = client;
Logger = logger;
}
}

27
src/Indexer/Models/Worker.cs
View File

@@ -1,4 +1,5 @@
using Microsoft.Extensions.Diagnostics.HealthChecks;
using Indexer.Scriptables;
using Indexer.Exceptions;
using Quartz;
using Quartz.Impl;
@@ -16,7 +17,7 @@ public class WorkerCollection
public WorkerCollection(ILogger<WorkerCollection> logger, IConfiguration configuration, Client.Client client)
{
Workers = [];
types = [typeof(PythonScriptable)];
types = [typeof(PythonScriptable), typeof(CSharpScriptable)];
_logger = logger;
_configuration = configuration;
this.client = client;
@@ -38,7 +39,7 @@ public class WorkerCollection
{
foreach (WorkerConfig workerConfig in sectionWorker.Worker)
{
ScriptToolSet toolSet = new(workerConfig.Script, client);
ScriptToolSet toolSet = new(workerConfig.Script, client, _logger, _configuration, workerConfig.Name);
InitializeWorker(toolSet, workerConfig);
}
}
@@ -153,17 +154,23 @@ public class WorkerCollection
public IScriptable GetScriptable(ScriptToolSet toolSet)
{
string fileName = toolSet.filePath;
string fileName = toolSet.FilePath ?? throw new IndexerConfigurationException($"\"Script\" not set for Worker \"{toolSet.Name}\"");
foreach (Type type in types)
{
IScriptable? instance = (IScriptable?)Activator.CreateInstance(type, [toolSet, _logger]);
if (instance is not null && instance.IsScript(fileName))
System.Reflection.MethodInfo? method = type.GetMethod("IsScript");
bool? isInstance = method is not null ? (bool?)method.Invoke(null, [fileName]) : null;
if (isInstance == true)
{
IScriptable? instance = (IScriptable?)Activator.CreateInstance(type, [toolSet, _logger]);
if (instance is null)
{
_logger.LogError("Unable to initialize script: \"{fileName}\"", fileName);
throw new Exception($"Unable to initialize script: \"{fileName}\"");
}
return instance;
}
}
_logger.LogError("Unable to determine the script's language: \"{fileName}\"", fileName);
throw new UnknownScriptLanguageException(fileName);
}
}
@@ -346,8 +353,8 @@ public class IntervalCall : ICall
{
if (!Scriptable.UpdateInfo.Successful)
{
_logger.LogWarning("HealthCheck revealed: The last execution of \"{name}\" was not successful", Scriptable.ToolSet.filePath);
return HealthCheckResult.Unhealthy($"HealthCheck revealed: The last execution of \"{Scriptable.ToolSet.filePath}\" was not successful");
_logger.LogWarning("HealthCheck revealed: The last execution of \"{name}\" was not successful", Scriptable.ToolSet.FilePath);
return HealthCheckResult.Unhealthy($"HealthCheck revealed: The last execution of \"{Scriptable.ToolSet.FilePath}\" was not successful");
}
double timerInterval = Timer.Interval; // In ms
DateTime lastRunDateTime = Scriptable.UpdateInfo.DateTime;
@@ -355,8 +362,8 @@ public class IntervalCall : ICall
double millisecondsSinceLastExecution = now.Subtract(lastRunDateTime).TotalMilliseconds;
if (millisecondsSinceLastExecution >= 2 * timerInterval)
{
_logger.LogWarning("HealthCheck revealed: Since the last execution of \"{name}\" more than twice the interval has passed", Scriptable.ToolSet.filePath);
return HealthCheckResult.Unhealthy($"HealthCheck revealed: Since the last execution of \"{Scriptable.ToolSet.filePath}\" more than twice the interval has passed");
_logger.LogWarning("HealthCheck revealed: Since the last execution of \"{name}\" more than twice the interval has passed", Scriptable.ToolSet.FilePath);
return HealthCheckResult.Unhealthy($"HealthCheck revealed: Since the last execution of \"{Scriptable.ToolSet.FilePath}\" more than twice the interval has passed");
}
return HealthCheckResult.Healthy();
}

75
src/Indexer/Scriptables/CSharpScriptable.cs Normal file
View File

@@ -0,0 +1,75 @@
using Microsoft.CodeAnalysis;
using Microsoft.CodeAnalysis.CSharp.Scripting;
using Microsoft.CodeAnalysis.Scripting;
using Indexer.Models;
namespace Indexer.Scriptables;
public class CSharpScriptable : IScriptable
{
public ScriptToolSet ToolSet { get; set; }
public ScriptUpdateInfo UpdateInfo { get; set; }
public ILogger _logger { get; set; }
public IScript script;
public CSharpScriptable(ScriptToolSet toolSet, ILogger logger)
{
_logger = logger;
ToolSet = toolSet;
try
{
script = LoadScriptAsync(ToolSet).Result;
Init();
}
catch (Exception ex)
{
_logger.LogCritical("Exception loading the script {ToolSet.filePath} CSharpScriptable {ex}", [ToolSet.FilePath, ex.StackTrace]);
throw;
}
}
public int Init()
{
return script.Init(ToolSet);
}
public int Update(ICallbackInfos callbackInfos)
{
return script.Update(callbackInfos);
}
public int Stop()
{
return script.Stop();
}
public async Task<IScript> LoadScriptAsync(ScriptToolSet toolSet)
{
string path = toolSet.FilePath;
var fileText = File.ReadAllText(path);
var code = string.Join("\n", fileText.Split("\n").Select(line => line.StartsWith("#load") ? "// " + line : line)); // CRUTCH! enables syntax highlighting via "#load" directive
var options = ScriptOptions.Default
.WithReferences(typeof(IScript).Assembly)
.WithImports("System")
.WithImports("System.Linq")
.WithImports("System.Console")
.WithImports("System.Collections")
.WithImports("Indexer.Models");
try
{
return await CSharpScript.EvaluateAsync<IScript>(code, options);
}
catch (Exception ex)
{
_logger.LogCritical("Exception loading the script {ToolSet.filePath} CSharpScriptable {ex.Message} {ex.StackTrace}", [ToolSet.FilePath, ex.Message, ex.StackTrace]);
throw;
}
}
public static bool IsScript(string fileName)
{
return fileName.EndsWith(".cs") || fileName.EndsWith(".csx");
}
}

101
src/Indexer/Scriptables/PythonScriptable.cs Normal file
View File

@@ -0,0 +1,101 @@
using Python.Runtime;
using Indexer.Models;
namespace Indexer.Scriptables;
public class PythonScriptable : IScriptable
{
public ScriptToolSet ToolSet { get; set; }
public PyObject? pyToolSet;
public PyModule scope;
public dynamic sys;
public string source;
public bool SourceLoaded { get; set; }
public ScriptUpdateInfo UpdateInfo { get; set; }
public ILogger _logger { get; set; }
public PythonScriptable(ScriptToolSet toolSet, ILogger logger)
{
string? runtime = toolSet.Configuration.GetValue<string>("EmbeddingsearchIndexer:PythonRuntime");
if (runtime is not null)
{
Runtime.PythonDLL ??= runtime;
}
_logger = logger;
SourceLoaded = false;
if (!PythonEngine.IsInitialized)
{
PythonEngine.Initialize();
PythonEngine.BeginAllowThreads();
}
ToolSet = toolSet;
source = File.ReadAllText(ToolSet.FilePath);
string fullPath = Path.GetFullPath(ToolSet.FilePath);
string? scriptDir = Path.GetDirectoryName(fullPath);
using (Py.GIL())
{
scope = Py.CreateScope();
sys = Py.Import("sys");
if (scriptDir is not null)
{
sys.path.append(scriptDir);
}
}
Init();
}
public int Init()
{
return ExecFunction("init");
}
public int Update(ICallbackInfos callbackInfos)
{
return ExecFunction("update");
}
public int Stop()
{
return ExecFunction("stop");
}
public int ExecFunction(string name, ICallbackInfos? callbackInfos = null)
{
int error = 0;
int retryCounter = 0;
retry:
try
{
using (Py.GIL())
{
pyToolSet = ToolSet.ToPython();
pyToolSet.SetAttr("callbackInfos", callbackInfos.ToPython());
scope.Set("toolset", pyToolSet);
if (!SourceLoaded)
{
scope.Exec(source);
SourceLoaded = true;
}
scope.Exec($"{name}(toolset)");
}
}
catch (Exception ex)
{
UpdateInfo = new() { DateTime = DateTime.Now, Successful = false, Exception = ex };
if (retryCounter < 3)
{
_logger.LogWarning("Execution of {name} function in script {Toolset.filePath} failed to an exception {ex.Message}", [name, ToolSet.FilePath, ex.Message]);
retryCounter++;
goto retry;
}
_logger.LogError("Execution of {name} function in script {Toolset.filePath} failed to an exception {ex.Message}", [name, ToolSet.FilePath, ex.Message]);
error = 1;
}
UpdateInfo = new() { DateTime = DateTime.Now, Successful = true };
return error;
}
public static bool IsScript(string fileName)
{
return fileName.EndsWith(".py");
}
}

69
src/Indexer/Scripts/example.csx Normal file
View File

@@ -0,0 +1,69 @@
#load "../../Client/Client.cs"
#load "../Models/Script.cs"
#load "../Models/Interfaces.cs"
#load "../Models/WorkerResults.cs"
#load "../../Shared/Models/SearchdomainResults.cs"
#load "../../Shared/Models/JSONModels.cs"
#load "../../Shared/Models/EntityResults.cs"
using Shared.Models;
using System.Collections.Generic;
using System.Linq;
using Microsoft.Extensions.Logging;
public class ExampleScript : Indexer.Models.IScript
{
public Indexer.Models.ScriptToolSet ToolSet;
public Client.Client client;
string defaultSearchdomain;
string exampleContent;
string probMethod;
string similarityMethod;
string exampleSearchdomain;
int exampleCounter;
List<string> models;
string probmethodDatapoint;
string probmethodEntity;
public ExampleScript()
{
//System.Console.WriteLine("DEBUG@example.cs - Constructor"); // logger not passed here yet
exampleContent = "./Scripts/example_content";
probMethod = "HVEWAvg";
similarityMethod = "Cosine";
exampleSearchdomain = "example_" + probMethod;
exampleCounter = 0;
models = ["ollama:bge-m3", "ollama:mxbai-embed-large"];
probmethodDatapoint = probMethod;
probmethodEntity = probMethod;
}
public int Init(Indexer.Models.ScriptToolSet toolSet)
{
ToolSet = toolSet;
ToolSet.Logger.LogInformation("DEBUG@example.csx - Init");
SearchdomainListResults searchdomains = ToolSet.Client.SearchdomainListAsync().Result;
defaultSearchdomain = searchdomains.Searchdomains.First();
var searchdomainList = string.Join("\n", searchdomains.Searchdomains);
ToolSet.Logger.LogInformation(searchdomainList);
return 0;
}
public int Update(Indexer.Models.ICallbackInfos callbackInfos)
{
ToolSet.Logger.LogInformation("DEBUG@example.csx - Update");
EntityQueryResults test = ToolSet.Client.EntityQueryAsync(defaultSearchdomain, "DNA").Result;
var firstResult = test.Results.ToArray()[0];
ToolSet.Logger.LogInformation(firstResult.Name);
ToolSet.Logger.LogInformation(firstResult.Value.ToString());
return 0;
}
public int Stop()
{
ToolSet.Logger.LogInformation("DEBUG@example.csx - Stop");
return 0;
}
}
return new ExampleScript();

10
src/Indexer/Scripts/example.py
View File

@@ -21,10 +21,10 @@ def init(toolset: Toolset):
print("Py-DEBUG@init")
print("This is the init function from the python example script")
print(f"example_counter: {example_counter}")
searchdomainlist:SearchdomainListResults = toolset.client.SearchdomainListAsync().Result
searchdomainlist:SearchdomainListResults = toolset.Client.SearchdomainListAsync().Result
if example_searchdomain not in searchdomainlist.Searchdomains:
toolset.client.SearchdomainCreateAsync(example_searchdomain).Result
searchdomainlist = toolset.client.SearchdomainListAsync().Result
toolset.Client.SearchdomainCreateAsync(example_searchdomain).Result
searchdomainlist = toolset.Client.SearchdomainListAsync().Result
print("Currently these searchdomains exist:")
for searchdomain in searchdomainlist.Searchdomains:
print(f" - {searchdomain}")
@@ -34,7 +34,7 @@ def update(toolset: Toolset):
global example_counter
print("Py-DEBUG@update")
print("This is the update function from the python example script")
callbackInfos:ICallbackInfos = toolset.callbackInfos
callbackInfos:ICallbackInfos = toolset.CallbackInfos
if (str(callbackInfos) == "Indexer.Models.IntervalCallbackInfos"):
print("It was called via an interval callback")
else:
@@ -59,6 +59,6 @@ def index_files(toolset: Toolset):
jsonEntities.append(jsonEntity)
jsonstring = json.dumps(jsonEntities)
timer_start = time.time()
result:EntityIndexResult = toolset.client.EntityIndexAsync(jsonstring).Result
result:EntityIndexResult = toolset.Client.EntityIndexAsync(jsonstring).Result
timer_end = time.time()
print(f"Update was successful: {result.Success} - and was done in {timer_end - timer_start} seconds.")

17
src/Indexer/appsettings.Development.json
View File

@@ -20,6 +20,23 @@
"Interval": 30000
}
]
},
{
"Name": "csharpExample",
"Script": "Scripts/example.csx",
"Calls": [
{
"Type": "runonce"
},
{
"Type": "schedule",
"Schedule": "0 0/5 * * * ?"
},
{
"Type": "fileupdate",
"Path": "./Scripts/example_content"
}
]
}
]
}

3
src/Indexer/appsettings.json
View File

@@ -23,7 +23,8 @@
"::1"
],
"LogFolder": "./logs"
}
},
"PythonRuntime": "libpython3.12.so"
},
"AllowedHosts": "*"
}