Implemented indexer, cleanup

src/Indexer/Exceptions/IndexerExceptions.cs

@@ -0,0 +1,14 @@
+namespace Indexer.Exceptions;
+
+public class IndexerConfigurationException : Exception
+{
+    public IndexerConfigurationException()
+        : base("Configuration is incomplete or was set up incorrectly")
+    {
+    }
+
+    public IndexerConfigurationException(string message)
+        : base(message)
+    {
+    }
+}

src/Indexer/Exceptions/WorkerExceptions.cs

@@ -0,0 +1,18 @@
+namespace Indexer.Exceptions;
+
+public class UnknownScriptLanguageException : Exception
+{
+    public string? FileName { get; }
+
+    public UnknownScriptLanguageException(string? fileName = null)
+        : base("Unable to determine script language")
+    {
+        FileName = fileName;
+    }
+
+    public UnknownScriptLanguageException(string message, string? fileName = null)
+        : base(message)
+    {
+        FileName = fileName;
+    }
+}

src/Indexer/Indexer.csproj

@@ -0,0 +1,19 @@
+<Project Sdk="Microsoft.NET.Sdk.Web">
+
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <Nullable>enable</Nullable>
+    <ImplicitUsings>enable</ImplicitUsings>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.AspNetCore.OpenApi" Version="8.0.14" />
+    <PackageReference Include="Swashbuckle.AspNetCore" Version="6.6.2" />
+    <PackageReference Include="Python" Version="3.13.3" />
+    <PackageReference Include="Pythonnet" Version="3.0.5" />
+  </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\server\server.csproj" />
+    <ProjectReference Include="..\Client\Client.csproj" />
+  </ItemGroup>
+</Project>

src/Indexer/Models/Interfaces.cs

@@ -0,0 +1,12 @@
+namespace Indexer.Models;
+
+public interface IScriptable
+{
+    ScriptToolSet ToolSet { get; set; }
+    void Init();
+    void Update(ICallbackInfos callbackInfos);
+    bool IsScript(string filePath);
+}
+
+public interface ICallbackInfos { }
+

src/Indexer/Models/Script.cs

@@ -0,0 +1,94 @@
+using System.Text.Json;
+using System.Timers;
+using embeddingsearch;
+using Python.Runtime;
+
+namespace Indexer.Models;
+
+public class PythonScriptable : IScriptable
+{
+    public ScriptToolSet ToolSet { get; set; }
+    public PyObject? pyToolSet;
+    public PyModule scope;
+    public dynamic sys;
+    public string source;
+    public PythonScriptable(ScriptToolSet toolSet)
+    {
+        Runtime.PythonDLL = @"libpython3.12.so";
+        if (!PythonEngine.IsInitialized)
+        {
+            PythonEngine.Initialize();
+            PythonEngine.BeginAllowThreads();
+        }
+        ToolSet = toolSet;
+        source = File.ReadAllText(ToolSet.filePath);
+        string fullPath = Path.GetFullPath(ToolSet.filePath);
+        string? scriptDir = Path.GetDirectoryName(fullPath);
+        using (Py.GIL())
+        {
+            scope = Py.CreateScope();
+            sys = Py.Import("sys");
+            if (scriptDir is not null)
+            {
+                sys.path.append(scriptDir);
+            }
+        }
+        Init();
+    }
+    
+    public void Init()
+    {
+        using (Py.GIL())
+        {
+            pyToolSet = ToolSet.ToPython();
+            scope.Set("toolset", pyToolSet);
+            scope.Exec(source);
+            scope.Exec("init(toolset)");
+        }
+    }
+
+    public void Update(ICallbackInfos callbackInfos)
+    {
+        PythonEngine.Initialize();
+        using (Py.GIL())
+        {
+            ToolSet.callbackInfos = callbackInfos;
+            pyToolSet = ToolSet.ToPython();
+            scope.Set("toolset", pyToolSet);
+            scope.Exec("update(toolset)");
+        }
+        PythonEngine.Shutdown();
+    }
+
+    public bool IsScript(string fileName)
+    {
+        return fileName.EndsWith(".py");
+    }
+}
+
+/*
+    TODO Add the following languages
+    - Javascript
+    - Golang (reconsider)
+*/
+
+public class ScriptToolSet
+{
+    public string filePath;
+    public Client.Client client;
+    public ICallbackInfos? callbackInfos;
+
+    // IConfiguration - Access to connection strings, ollama, etc. maybe?
+    public ScriptToolSet(string filePath, Client.Client client)
+    {
+        this.filePath = filePath;
+        this.client = client;
+    }
+}
+
+public class IntervalCallbackInfos : ICallbackInfos
+{
+    public object? sender;
+    public required ElapsedEventArgs e;
+     
+}

src/Indexer/Models/Worker.cs

@@ -0,0 +1,45 @@
+namespace Indexer.Models;
+
+public class WorkerCollection
+{
+    public List<Worker> Workers;
+    public List<Type> types;
+    public WorkerCollection()
+    {
+        Workers = [];
+        types = [typeof(PythonScriptable)];
+    }
+}
+
+public class Worker
+{
+    public WorkerConfig Config { get; set; }
+    public IScriptable Scriptable { get; set; }
+
+    public Worker(WorkerConfig workerConfig, IScriptable scriptable)
+    {
+        this.Config = workerConfig;
+        this.Scriptable = scriptable;
+    }
+}
+
+public class WorkerCollectionConfig
+{
+    public required List<WorkerConfig> Worker { get; set; }
+}
+
+public class WorkerConfig
+{
+    public required string Name { get; set; }
+    public required List<string> Searchdomains { get; set; }
+    public required string Script { get; set; }
+    public required List<Call> Calls { get; set; }
+}
+
+public class Call
+{
+    public required string Type { get; set; }
+    public long? Interval { get; set; } // For Type: Interval
+    public string? Path { get; set; } // For Type: FileSystemWatcher
+}
+

src/Indexer/Program.cs

@@ -0,0 +1,31 @@
+using Indexer.Models;
+using Indexer.Services;
+using server;
+
+var builder = WebApplication.CreateBuilder(args);
+
+// Add services to the container.
+
+builder.Services.AddControllers();
+// Learn more about configuring Swagger/OpenAPI at https://aka.ms/aspnetcore/swashbuckle
+builder.Services.AddEndpointsApiExplorer();
+builder.Services.AddSwaggerGen();
+builder.Services.AddSingleton<Client.Client>();
+builder.Services.AddSingleton<WorkerCollection>();
+builder.Services.AddHostedService<IndexerService>();
+var app = builder.Build();
+
+// Configure the HTTP request pipeline.
+if (app.Environment.IsDevelopment())
+{
+    app.UseSwagger();
+    app.UseSwaggerUI();
+}
+else
+{
+    app.UseMiddleware<ApiKeyMiddleware>();
+}
+
+// app.UseHttpsRedirection();
+
+app.Run();

src/Indexer/Properties/launchSettings.json

@@ -0,0 +1,41 @@
+{
+  "$schema": "http://json.schemastore.org/launchsettings.json",
+  "iisSettings": {
+    "windowsAuthentication": false,
+    "anonymousAuthentication": true,
+    "iisExpress": {
+      "applicationUrl": "http://localhost:36687",
+      "sslPort": 44337
+    }
+  },
+  "profiles": {
+    "http": {
+      "commandName": "Project",
+      "dotnetRunMessages": true,
+      "launchBrowser": true,
+      "launchUrl": "swagger",
+      "applicationUrl": "http://localhost:5210",
+      "environmentVariables": {
+        "ASPNETCORE_ENVIRONMENT": "Development"
+      }
+    },
+    "https": {
+      "commandName": "Project",
+      "dotnetRunMessages": true,
+      "launchBrowser": true,
+      "launchUrl": "swagger",
+      "applicationUrl": "https://localhost:7282;http://localhost:5210",
+      "environmentVariables": {
+        "ASPNETCORE_ENVIRONMENT": "Development"
+      }
+    },
+    "IIS Express": {
+      "commandName": "IISExpress",
+      "launchBrowser": true,
+      "launchUrl": "swagger",
+      "environmentVariables": {
+        "ASPNETCORE_ENVIRONMENT": "Development"
+      }
+    }
+  }
+}

src/Indexer/Scripts/example.py

@@ -0,0 +1,50 @@
+import os
+from tools import *
+import json
+from dataclasses import asdict
+
+example_content = "./Scripts/example_content"
+example_searchdomain = "example"
+example_counter = 0
+models = ["bge-m3", "mxbai-embed-large"]
+
+def init(toolset: Toolset):
+    global example_counter
+    print("Py-DEBUG@init")
+    print("This is the init function from the python example script")
+    print(f"example_counter: {example_counter}")
+    searchdomainlist:SearchdomainListResults = toolset.client.SearchdomainListAsync().Result
+    print("Currently these searchdomains exist")
+    for searchdomain in searchdomainlist.Searchdomains:
+        print(f" - {searchdomain}")
+    index_files(toolset)
+
+def update(toolset: Toolset):
+    global example_counter
+    print("Py-DEBUG@update")
+    print("This is the update function from the python example script")
+    callbackInfos:ICallbackInfos = toolset.callbackInfos
+    if (callbackInfos is IntervalCallbackInfos):
+        print("It was called via an interval callback")
+    example_counter += 1
+    mycounter = example_counter
+    print(f"example_counter: {example_counter}")
+    index_files(toolset)
+
+def index_files(toolset: Toolset):
+    jsonEntities:list = []
+    for filename in os.listdir(example_content):
+        qualified_filepath = example_content + "/" + filename
+        with open(qualified_filepath, "r") as file:
+            title = file.readline()
+            text = file.read()
+        datapoints:list = [
+            JSONDatapoint("filename", qualified_filepath, "wavg", models),
+            JSONDatapoint("title", title, "wavg", models),
+            JSONDatapoint("text", text, "wavg", models)
+        ]
+        jsonEntity:dict = asdict(JSONEntity(qualified_filepath, "wavg", example_searchdomain, {}, datapoints))
+        jsonEntities.append(jsonEntity)
+    jsonstring = json.dumps(jsonEntities)
+    result:EntityIndexResult = toolset.client.EntityIndexAsync(jsonstring).Result
+    print(f"Update was successful: {result.Success}")

src/Indexer/Scripts/generate_example_content.py

@@ -0,0 +1,122 @@
+#!/usr/bin/env python3
+"""
+Generate ten brief-overview files for a given topic using an Ollama model.
+
+▪ The directory ./files is used as a mini knowledge-base.
+▪ Two Python functions are exposed to the model as *tools*:
+      • list_files()   – return [{name, title}, …] for everything in ./files
+      • create_file()  – create/overwrite a file and write the content supplied
+▪ The model is instructed to call create_file() ten times (one per sub-topic)
+  and put the title on the first line of each file.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+from pathlib import Path
+from typing import List, Dict, Any
+
+import ollama   # pip install -U ollama
+
+
+# ---------- constant configuration ------------------------------------------------
+
+FILES_DIR = Path(__file__).parent / "example_content"
+FILES_DIR.mkdir(exist_ok=True)
+
+
+# ---------- tool functions ---------------------------------------------------------
+
+def list_files() -> List[Dict[str, str]]:
+    """
+    List every regular file in ./example_content together with its first line (title).
+
+    Returns
+    -------
+    list[dict]
+        Each element has:  {"name": "<filename>", "title": "<first line or ''>"}
+    """
+    results: List[Dict[str, str]] = []
+    for path in FILES_DIR.iterdir():
+        if path.is_file():
+            with path.open("r", encoding="utf-8", errors="ignore") as fh:
+                title = fh.readline().rstrip("\n")
+            results.append({"name": path.name, "title": title})
+    return results
+
+
+def create_file(filename: str, content: str) -> str:
+    """
+    Create (or overwrite) a file inside ./files.
+
+    Parameters
+    ----------
+    filename : str
+        A simple name like "quantum_entanglement.md".  Any path components
+        beyond the basename are stripped for safety.
+    content : str
+        The full text to write – the first line *must* be the title.
+
+    Returns
+    -------
+    str
+        Absolute path of the file that was written.
+    """
+    safe_name = os.path.basename(filename)
+    if not safe_name:
+        raise ValueError("filename cannot be empty")
+    path = FILES_DIR / safe_name
+    path.write_text(content, encoding="utf-8")
+    return str(path.resolve())
+
+
+# ---------- main driver ------------------------------------------------------------
+
+def run(topic: str, *, model: str = "qwen3:latest", temperature: float = 0.2) -> None:
+    """Ask the model to create ten overview files about *topic*."""
+
+    system_prompt = (
+        "You are a file-writing assistant.  For each of ten distinct sub-topics "
+        "related to the given topic you will call the `create_file` tool to "
+        "write a Markdown file that contains at least 5 sentences.  Use a short, "
+        "snake_case filename ending with '.md'.  The very first line of the "
+        "file **must** be the title (capitalized).  After you have created all "
+        "ten files, reply with only the single word DONE."
+    )
+
+    messages: List[Dict[str, Any]] = [
+        {"role": "system", "content": system_prompt},
+        {"role": "user", "content": f"Topic: {topic}"},
+    ]
+
+    tools = [list_files, create_file]
+    available = {f.__name__: f for f in tools}
+
+    # initial call
+    response = ollama.chat(model=model,
+                           messages=messages,
+                           tools=tools,
+                           options={"temperature": temperature})
+
+    for call in response.message.tool_calls or []:
+        fn_name = call.function.name
+        fn_args = call.function.arguments
+        result = available[fn_name](**fn_args) # Run tool calls
+        messages.append({"role": "tool",
+                            "name": fn_name,
+                            "content": json.dumps(result, ensure_ascii=False)})
+
+# ---------- CLI entry-point --------------------------------------------------------
+
+if __name__ == "__main__":
+    import argparse
+
+    parser = argparse.ArgumentParser(
+        description="Generate ten overview files for a topic using Ollama + tool calling")
+    parser.add_argument("topic", help="Main subject area, e.g. 'quantum computing'")
+    parser.add_argument("--model", default="qwen3:latest",
+                        help="Local Ollama model to use (default: qwen3:latest)")
+    args = parser.parse_args()
+
+    run(args.topic, model=args.model)

src/Indexer/Scripts/tools.py

@@ -0,0 +1,130 @@
+from dataclasses import dataclass
+import array
+from typing import Optional
+
+@dataclass
+class JSONDatapoint:
+    Name:str
+    Text:str
+    Probmethod_embedding:str
+    Model:list[str]
+
+@dataclass
+class JSONEntity:
+    Name:str
+    Probmethod:str
+    Searchdomain:str
+    Attributes:dict
+    Datapoints:array.array[JSONDatapoint]
+
+#Model - Searchdomain
+@dataclass
+class SearchdomainListResults:
+    Searchdomains:list[str]
+
+@dataclass
+class SearchdomainCreateResults:
+    Success:bool
+    id:int|None
+
+@dataclass
+class SearchdomainUpdateResults:
+    Success:bool
+
+@dataclass
+class SearchdomainDeleteResults:
+    Success:bool
+    DeletedEntities:int
+
+#Model - Entity
+@dataclass
+class EntityQueryResult:
+    name:str
+    ValueError:float
+
+@dataclass
+class EntityQueryResults:
+    Results:list[EntityQueryResult]
+
+@dataclass
+class EntityIndexResult:
+    Success:bool
+
+@dataclass
+class AttributeResult:
+    Name:str
+    Value:str
+
+@dataclass
+class EmbeddingResult:
+    Model:str
+    Embeddings:array.array[float]
+
+@dataclass
+class DatapointResult:
+    Name:str
+    ProbMethod:str
+    Embeddings:list[EmbeddingResult]|None
+
+@dataclass
+class EntityListResults:
+    Name:str
+    Attributes:list[AttributeResult]
+    Datapoints:list[DatapointResult]
+
+@dataclass
+class EntityDeleteResults:
+    Success:bool
+
+# Model - Client
+@dataclass
+class Client:
+    baseUri:str
+    apiKey:str
+    searchdomain:str
+    async def SearchdomainListAsync() -> SearchdomainListResults:
+        pass
+    async def SearchdomainDeleteAsync() -> SearchdomainDeleteResults:
+        pass
+    async def SearchdomainCreateAsync() -> SearchdomainCreateResults:
+        pass
+    async def SearchdomainCreateAsync(searchdomain:str) -> SearchdomainCreateResults:
+        pass
+    async def SearchdomainUpdateAsync(newName:str, settings:str) -> SearchdomainUpdateResults:
+        pass
+    async def SearchdomainUpdateAsync(searchdomain:str, newName:str, settings:str) -> SearchdomainUpdateResults:
+        pass
+    async def EntityQueryAsync(query:str) -> EntityQueryResults:
+        pass
+    async def EntityQueryAsync(searchdomain:str, query:str) -> EntityQueryResults:
+        pass
+    #async def EntityIndexAsync(jsonEntity): # -> EntityIndexResult:#:NetList[JSONEntity]) -> EntityIndexResult: #TODO fix clr issues, i.e. make this work
+    #    pass
+    #async def EntityIndexAsync(searchdomain:str, jsonEntity:list[JSONEntity]) -> EntityIndexResult:
+    #    pass
+    async def EntityIndexAsync(jsonEntity:str) -> EntityIndexResult:
+        pass
+    async def EntityIndexAsync(searchdomain:str, jsonEntity:str) -> EntityIndexResult:
+        pass
+    async def EntityListAsync(returnEmbeddings:bool = False) -> EntityListResults:
+        pass
+    async def EntityListAsync(searchdomain:str, returnEmbeddings:bool = False) -> EntityListResults:
+        pass
+    async def EntityDeleteAsync(searchdomain:str, entityName:str) -> EntityDeleteResults:
+        pass
+class ICallbackInfos:
+    pass
+
+
+@dataclass
+class IntervalCallbackInfos(ICallbackInfos):
+    sender: Optional[object]
+    e: object  
+
+@dataclass
+class Toolset:
+    filePath:str
+    client:Client
+    callbackInfos: Optional[ICallbackInfos] = None
+    
+

src/Indexer/Services/IndexerService.cs

@@ -0,0 +1,102 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+using Indexer.Exceptions;
+using Indexer.Models;
+using System.Timers;
+using Microsoft.AspNetCore.Http.HttpResults;
+using embeddingsearch;
+using Python.Runtime;
+
+namespace Indexer.Services;
+
+public class IndexerService : IHostedService
+{
+    private readonly WorkerCollection workerCollection;
+    private readonly IConfiguration _config;
+    private readonly Client.Client client;
+
+    public IndexerService(WorkerCollection workerCollection, IConfiguration configuration, Client.Client client)
+    {
+        this._config = configuration;
+        this.client = client;
+        this.workerCollection = workerCollection;
+        // Load and configure all workers
+        var sectionMain = _config.GetSection("EmbeddingsearchIndexer");
+
+        WorkerCollectionConfig? sectionWorker = (WorkerCollectionConfig?) sectionMain.Get(typeof(WorkerCollectionConfig)); //GetValue<WorkerCollectionConfig>("Worker");
+        if (sectionWorker is not null)
+        {
+            foreach (WorkerConfig workerConfig in sectionWorker.Worker)
+            {
+                if (client.searchdomain == "" && workerConfig.Searchdomains.Count >= 1)
+                {
+                    client.searchdomain = workerConfig.Searchdomains.First();
+                }
+                ScriptToolSet toolSet = new(workerConfig.Script, client);
+                Worker worker = new(workerConfig, GetScriptable(toolSet));
+                workerCollection.Workers.Add(worker);
+                foreach (Call call in workerConfig.Calls)
+                {
+                    switch (call.Type)
+                    {
+                        case "interval":
+                            if (call.Interval is null)
+                            {
+                                throw new IndexerConfigurationException($"Interval not set for a Call in Worker \"{workerConfig.Name}\"");
+                            }
+                            var timer = new System.Timers.Timer((double)call.Interval);
+                            timer.Elapsed += (sender, e) => worker.Scriptable.Update(new IntervalCallbackInfos() { sender = sender, e = e });
+                            timer.AutoReset = true;
+                            timer.Enabled = true;
+                            break;
+                        case "schedule": // TODO implement scheduled tasks using Quartz
+                            throw new NotImplementedException("schedule not implemented yet");
+                        case "fileupdate":
+                            if (call.Path is null)
+                            {
+                                throw new IndexerConfigurationException($"Path not set for a Call in Worker \"{workerConfig.Name}\"");
+                            }
+                            throw new NotImplementedException("fileupdate not implemented yet");
+                        //break;
+                        default:
+                            throw new IndexerConfigurationException($"Unknown Type specified for a Call in Worker \"{workerConfig.Name}\"");
+                    }
+                }
+            }
+        }
+        else
+        {
+            throw new IndexerConfigurationException("Unable to find section \"Worker\"");
+        }
+    }
+
+    public IScriptable GetScriptable(ScriptToolSet toolSet)
+    {
+        string fileName = toolSet.filePath;
+        foreach (Type type in workerCollection.types)
+        {
+            IScriptable? instance = (IScriptable?)Activator.CreateInstance(type, toolSet);
+            if (instance is not null && instance.IsScript(fileName))
+            {
+                return instance;
+            }
+        }
+
+        throw new UnknownScriptLanguageException(fileName);
+    }
+    public Task StartAsync(CancellationToken cancellationToken)
+    {
+        /*foreach (Worker worker in workerCollection.Workers)
+        {
+            worker.Scriptable.Init();
+        }*/
+        return Task.CompletedTask;
+    }
+
+    public Task StopAsync(CancellationToken cancellationToken)
+    {
+        return Task.CompletedTask;
+    }
+}

src/Indexer/appsettings.Development.json

@@ -0,0 +1,29 @@
+{
+  "Logging": {
+    "LogLevel": {
+      "Default": "Information",
+      "Microsoft.AspNetCore": "Warning"
+    }
+  },
+  "Embeddingsearch": {
+    "BaseUri": "http://localhost:5146"
+  },
+  "EmbeddingsearchIndexer": {
+    "Worker":
+    [
+      {
+        "Name": "example",
+        "Searchdomains": [
+          "example"
+        ],
+        "Script": "Scripts/example.py",
+        "Calls": [
+          {
+            "Type": "interval",
+            "Interval": 10000
+          }
+        ]
+      }
+    ]
+  }
+}

src/Indexer/appsettings.json

@@ -0,0 +1,9 @@
+{
+  "Logging": {
+    "LogLevel": {
+      "Default": "Information",
+      "Microsoft.AspNetCore": "Warning"
+    }
+  },
+  "AllowedHosts": "*"
+}