lucretia/embeddingsearch
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Removed CLI project, added database migrations, added SQLHelper

Browse Source
This commit is contained in:
LD-Reborn
2025-06-06 00:29:38 +02:00
parent 89545b0d71
commit ad599433bf
10 changed files with 152 additions and 555 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@@ -82,6 +82,8 @@ All commands, parameters and examples are documented here: [docs/CLI.md](docs/CL
- Remove the `id` collumns from the database tables where the table is actually identified (and should be unique by) the name, which should become the new primary key. - Remove the `id` collumns from the database tables where the table is actually identified (and should be unique by) the name, which should become the new primary key.
- Improve performance & latency (Create ready-to-go processes where each contain an n'th share of the entity cache, ready to perform a query. Prepare it after creating the entity cache.) - Improve performance & latency (Create ready-to-go processes where each contain an n'th share of the entity cache, ready to perform a query. Prepare it after creating the entity cache.)
- Make the API server (and indexer, once it is done) a docker container - Make the API server (and indexer, once it is done) a docker container
- Implement dynamic invocation based database migrations
- Remove remaining DRY violations using the SQLHelper
# Future features # Future features
- Support for other database types (MSSQL, SQLite) - Support for other database types (MSSQL, SQLite)

10
embeddingsearch.sln
View File

@@ -7,10 +7,6 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{6AA0A9E0-A36
EndProject EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Server", "src\Server\Server.csproj", "{643CB1D1-02F6-4BCC-A1CC-6E403D78C442}" Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Server", "src\Server\Server.csproj", "{643CB1D1-02F6-4BCC-A1CC-6E403D78C442}"
EndProject EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "cli", "cli", "{BC4F3063-B921-4C4A-A7CE-11FAF5B73D50}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "cli", "src\cli\cli.csproj", "{D61A2C50-B46C-42BA-B75D-E84D8FA28C29}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Client", "src\Client\Client.csproj", "{A8EBB748-5BBA-47EB-840D-E398365C52A2}" Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Client", "src\Client\Client.csproj", "{A8EBB748-5BBA-47EB-840D-E398365C52A2}"
EndProject EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Indexer", "src\Indexer\Indexer.csproj", "{5361FD10-E85C-496C-9BEF-9232F767F904}" Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Indexer", "src\Indexer\Indexer.csproj", "{5361FD10-E85C-496C-9BEF-9232F767F904}"
@@ -25,10 +21,6 @@ Global
{643CB1D1-02F6-4BCC-A1CC-6E403D78C442}.Debug|Any CPU.Build.0 = Debug|Any CPU {643CB1D1-02F6-4BCC-A1CC-6E403D78C442}.Debug|Any CPU.Build.0 = Debug|Any CPU
{643CB1D1-02F6-4BCC-A1CC-6E403D78C442}.Release|Any CPU.ActiveCfg = Release|Any CPU {643CB1D1-02F6-4BCC-A1CC-6E403D78C442}.Release|Any CPU.ActiveCfg = Release|Any CPU
{643CB1D1-02F6-4BCC-A1CC-6E403D78C442}.Release|Any CPU.Build.0 = Release|Any CPU {643CB1D1-02F6-4BCC-A1CC-6E403D78C442}.Release|Any CPU.Build.0 = Release|Any CPU
{D61A2C50-B46C-42BA-B75D-E84D8FA28C29}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{D61A2C50-B46C-42BA-B75D-E84D8FA28C29}.Debug|Any CPU.Build.0 = Debug|Any CPU
{D61A2C50-B46C-42BA-B75D-E84D8FA28C29}.Release|Any CPU.ActiveCfg = Release|Any CPU
{D61A2C50-B46C-42BA-B75D-E84D8FA28C29}.Release|Any CPU.Build.0 = Release|Any CPU
{A8EBB748-5BBA-47EB-840D-E398365C52A2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {A8EBB748-5BBA-47EB-840D-E398365C52A2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{A8EBB748-5BBA-47EB-840D-E398365C52A2}.Debug|Any CPU.Build.0 = Debug|Any CPU {A8EBB748-5BBA-47EB-840D-E398365C52A2}.Debug|Any CPU.Build.0 = Debug|Any CPU
{A8EBB748-5BBA-47EB-840D-E398365C52A2}.Release|Any CPU.ActiveCfg = Release|Any CPU {A8EBB748-5BBA-47EB-840D-E398365C52A2}.Release|Any CPU.ActiveCfg = Release|Any CPU
@@ -43,8 +35,6 @@ Global
EndGlobalSection EndGlobalSection
GlobalSection(NestedProjects) = preSolution GlobalSection(NestedProjects) = preSolution
{643CB1D1-02F6-4BCC-A1CC-6E403D78C442} = {6AA0A9E0-A361-4E86-BA02-D5F6779C6DEF} {643CB1D1-02F6-4BCC-A1CC-6E403D78C442} = {6AA0A9E0-A361-4E86-BA02-D5F6779C6DEF}
{BC4F3063-B921-4C4A-A7CE-11FAF5B73D50} = {6AA0A9E0-A361-4E86-BA02-D5F6779C6DEF}
{D61A2C50-B46C-42BA-B75D-E84D8FA28C29} = {BC4F3063-B921-4C4A-A7CE-11FAF5B73D50}
{A8EBB748-5BBA-47EB-840D-E398365C52A2} = {6AA0A9E0-A361-4E86-BA02-D5F6779C6DEF} {A8EBB748-5BBA-47EB-840D-E398365C52A2} = {6AA0A9E0-A361-4E86-BA02-D5F6779C6DEF}
{5361FD10-E85C-496C-9BEF-9232F767F904} = {6AA0A9E0-A361-4E86-BA02-D5F6779C6DEF} {5361FD10-E85C-496C-9BEF-9232F767F904} = {6AA0A9E0-A361-4E86-BA02-D5F6779C6DEF}
EndGlobalSection EndGlobalSection

14
src/Server/Exceptions/DatabaseExceptions.cs Normal file
View File

@@ -0,0 +1,14 @@
namespace Server.Exceptions;
public class DatabaseVersionException : Exception
{
public DatabaseVersionException()
: base("DatabaseVersion could not be parsed as integer. Please ensure the DatabaseVersion can be parsed as an integer.")
{
}
public DatabaseVersionException(string message)
: base(message)
{
}
}

69
src/Server/Migrations/DatabaseMigrations.cs Normal file
View File

@@ -0,0 +1,69 @@
using System.Data.Common;
using Server.Exceptions;
namespace Server.Migrations;
public static class DatabaseMigrations
{
public static void Migrate(SQLHelper helper)
{
int databaseVersion = DatabaseGetVersion(helper);
switch (databaseVersion)
{
case 0:
databaseVersion = Create(helper);
goto case 1; // Here lies a dead braincell.
case 1:
databaseVersion = UpdateFrom1(helper); // TODO: Implement reflection based dynamic invocation.
goto case 2;
case 2:
default:
break;
}
}
public static int DatabaseGetVersion(SQLHelper helper)
{
DbDataReader reader = helper.ExecuteSQLCommand("show tables", []);
bool hasTables = reader.Read();
reader.Close();
if (!hasTables)
{
return 0;
}
reader = helper.ExecuteSQLCommand("show tables like '%settings%'", []);
bool hasSystemTable = reader.Read();
reader.Close();
if (!hasSystemTable)
{
return 1;
}
reader = helper.ExecuteSQLCommand("SELECT value FROM settings WHERE name=\"DatabaseVersion\"", []);
reader.Read();
string rawVersion = reader.GetString(0);
reader.Close();
bool success = int.TryParse(rawVersion, out int version);
if (!success)
{
throw new DatabaseVersionException();
}
return version;
}
public static int Create(SQLHelper helper)
{
helper.ExecuteSQLNonQuery("CREATE TABLE searchdomain (id int PRIMARY KEY auto_increment, name varchar(512), settings JSON);", []);
helper.ExecuteSQLNonQuery("CREATE TABLE entity (id int PRIMARY KEY auto_increment, name varchar(512), probmethod varchar(128), id_searchdomain int, FOREIGN KEY (id_searchdomain) REFERENCES searchdomain(id));", []);
helper.ExecuteSQLNonQuery("CREATE TABLE attribute (id int PRIMARY KEY auto_increment, id_entity int, attribute varchar(512), value longtext, FOREIGN KEY (id_entity) REFERENCES entity(id));", []);
helper.ExecuteSQLNonQuery("CREATE TABLE datapoint (id int PRIMARY KEY auto_increment, name varchar(512), probmethod_embedding varchar(512), id_entity int, FOREIGN KEY (id_entity) REFERENCES entity(id));", []);
helper.ExecuteSQLNonQuery("CREATE TABLE embedding (id int PRIMARY KEY auto_increment, id_datapoint int, model varchar(512), embedding blob, FOREIGN KEY (id_datapoint) REFERENCES datapoint(id));", []);
return 1;
}
public static int UpdateFrom1(SQLHelper helper)
{
helper.ExecuteSQLNonQuery("CREATE TABLE settings (name varchar(512), value varchar(8192));", []);
helper.ExecuteSQLNonQuery("INSERT INTO settings (name, value) VALUES (\"DatabaseVersion\", \"2\");", []);
return 2;
}
}

49
src/Server/SQLHelper.cs Normal file
View File

@@ -0,0 +1,49 @@
using System.Data.Common;
using MySql.Data.MySqlClient;
namespace Server;
public class SQLHelper
{
public MySqlConnection connection;
public SQLHelper(MySqlConnection connection)
{
this.connection = connection;
}
public DbDataReader ExecuteSQLCommand(string query, Dictionary<string, dynamic> parameters)
{
using MySqlCommand command = connection.CreateCommand();
command.CommandText = query;
foreach (KeyValuePair<string, dynamic> parameter in parameters)
{
command.Parameters.AddWithValue($"@{parameter.Key}", parameter.Value);
}
return command.ExecuteReader();
}
public void ExecuteSQLNonQuery(string query, Dictionary<string, dynamic> parameters)
{
using MySqlCommand command = connection.CreateCommand();
command.CommandText = query;
foreach (KeyValuePair<string, dynamic> parameter in parameters)
{
command.Parameters.AddWithValue($"@{parameter.Key}", parameter.Value);
}
command.ExecuteNonQuery();
}
public int ExecuteSQLCommandGetInsertedID(string query, Dictionary<string, dynamic> parameters)
{
using MySqlCommand command = connection.CreateCommand();
command.CommandText = query;
foreach (KeyValuePair<string, dynamic> parameter in parameters)
{
command.Parameters.AddWithValue($"@{parameter.Key}", parameter.Value);
}
command.ExecuteNonQuery();
command.CommandText = "SELECT LAST_INSERT_ID();";
return Convert.ToInt32(command.ExecuteScalar());
}
}

19
src/Server/SearchdomainManager.cs
View File

@@ -3,6 +3,7 @@ using System.Data.Common;
using OllamaSharp; using OllamaSharp;
using Microsoft.IdentityModel.Tokens; using Microsoft.IdentityModel.Tokens;
using Server.Exceptions; using Server.Exceptions;
using Server.Migrations;
namespace Server; namespace Server;
@@ -29,8 +30,11 @@ public class SearchdomainManager
client = new(new Uri(ollamaURL)); client = new(new Uri(ollamaURL));
connection = new MySqlConnection(connectionString); connection = new MySqlConnection(connectionString);
connection.Open(); connection.Open();
DatabaseMigrations.Migrate(new SQLHelper(connection));
} }
public Searchdomain GetSearchdomain(string searchdomain) public Searchdomain GetSearchdomain(string searchdomain)
{ {
if (searchdomains.TryGetValue(searchdomain, out Searchdomain? value)) if (searchdomains.TryGetValue(searchdomain, out Searchdomain? value))
@@ -40,7 +44,8 @@ public class SearchdomainManager
try try
{ {
return SetSearchdomain(searchdomain, new Searchdomain(searchdomain, connectionString, client)); return SetSearchdomain(searchdomain, new Searchdomain(searchdomain, connectionString, client));
} catch (MySqlException) }
catch (MySqlException)
{ {
_logger.LogError("Unable to find the searchdomain {searchdomain}", searchdomain); _logger.LogError("Unable to find the searchdomain {searchdomain}", searchdomain);
throw new Exception($"Unable to find the searchdomain {searchdomain}"); throw new Exception($"Unable to find the searchdomain {searchdomain}");
@@ -105,6 +110,18 @@ public class SearchdomainManager
return command.ExecuteReader(); return command.ExecuteReader();
} }
public void ExecuteSQLNonQuery(string query, Dictionary<string, dynamic> parameters)
{
using MySqlCommand command = connection.CreateCommand();
command.CommandText = query;
foreach (KeyValuePair<string, dynamic> parameter in parameters)
{
command.Parameters.AddWithValue($"@{parameter.Key}", parameter.Value);
}
command.ExecuteNonQuery();
}
public int ExecuteSQLCommandGetInsertedID(string query, Dictionary<string, dynamic> parameters) public int ExecuteSQLCommandGetInsertedID(string query, Dictionary<string, dynamic> parameters)
{ {
using MySqlCommand command = connection.CreateCommand(); using MySqlCommand command = connection.CreateCommand();

5
src/Server/Server.csproj
View File

@@ -18,9 +18,4 @@
<PackageReference Include="System.Data.Sqlite" Version="1.0.119" /> <PackageReference Include="System.Data.Sqlite" Version="1.0.119" />
<PackageReference Include="System.Numerics.Tensors" Version="9.0.3" /> <PackageReference Include="System.Numerics.Tensors" Version="9.0.3" />
</ItemGroup> </ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Models\Models.csproj" />
</ItemGroup>
</Project> </Project>

171
src/cli/Options.cs
View File

@@ -1,171 +0,0 @@
using CommandLine;
namespace cli;
public class OptionsCommand
{
[Option("database", Required = false, HelpText = "Do things related to the database")] // Create database / ensure it is set up correctly
public bool IsDatabase { get; set; }
[Option("searchdomain", Required = false, HelpText = "Execute CRUD on searchdomains")]
public bool IsSearchdomain { get; set; }
[Option("entity", Required = false, HelpText = "Execute CRUD on entities")]
public bool IsEntity { get; set; }
[Option('h', "host", Required = true, HelpText = "Host IP address (e.g. 192.168.0.75)")]
public required string IP { get; set; }
[Option('p', "port", Required = true, HelpText = "Host port (e.g. 3306)")]
public required int Port { get; set; }
[Option('U', "username", Required = true, HelpText = "Username for the MySQL database")]
public required string Username { get; set; }
[Option('P', "password", Required = true, HelpText = "Password for the MySQL database")]
public required string Password { get; set; }
}
public class OptionsDatabase : OptionsCommand
{
[Option("setup", Required = false, HelpText = "Ensure the database is set up correctly")]
public bool SetupDatabase { get; set; }
}
public class OptionsSearchdomain : OptionsCommand
{
[Option("create", Required = false, HelpText = "Create a searchdomain")]
public bool IsCreate { get; set; }
[Option("list", Required = false, HelpText = "Lists the searchdomains")]
public bool IsList { get; set; }
[Option("update", Required = false, HelpText = "Update a searchdomain (settings, name)")]
public bool IsUpdate { get; set; }
[Option("delete", Required = false, HelpText = "Delete a searchdomain")]
public bool IsDelete { get; set; }
}
public class OptionsSearchdomainCreate : OptionsSearchdomain
{
[Option('s', Required = true, HelpText = "Name of the searchdomain to create")]
public required string Searchdomain { get; set; }
}
public class OptionsSearchdomainList : OptionsSearchdomain
{
// The cleanest piece of code in this project
}
public class OptionsSearchdomainUpdate : OptionsSearchdomain
{
[Option('s', Required = true, HelpText = "Name of the searchdomain to update")]
public required string Searchdomain { get; set; }
[Option('n', Required = false, HelpText = "New name to set")]
public string? Name { get; set; }
[Option('S', Required = false, HelpText = "New Settings (as json)")]
public string? Settings { get; set; }
}
public class OptionsSearchdomainDelete : OptionsSearchdomain
{
[Option('s', Required = true, HelpText = "Name of the searchdomain to delete")]
public required string Searchdomain { get; set; }
}
public class OptionsEntity : OptionsCommand
{
[Option("evaluate", Required = false, HelpText = "Evaluate a query")]
public bool IsEvaluate { get; set; }
[Option("index", Required = false, HelpText = "Create or update an entity from a JSON string")]
public bool IsIndex { get; set; }
[Option("remove", Required = false, HelpText = "Remove an entity")]
public bool IsDelete { get; set; }
[Option("list", Required = false, HelpText = "List all entities")]
public bool IsList { get; set; }
}
public class OptionsEntityQuery : OptionsEntity
{
[Option('s', Required = true, HelpText = "Searchdomain to be searched")]
public required string Searchdomain { get; set; }
[Option('q', "query", Required = true, HelpText = "Query string to evaluate the entities against")]
public required string Query { get; set; }
[Option('o', "ollama", Required = true, HelpText = "Ollama URL")]
public required string OllamaURL { get; set; }
[Option('n', "num", Required = false, HelpText = "(Maximum) number of results to output", Default = 5)]
public int Num { get; set; }
}
public class OptionsEntityIndex : OptionsEntity // Example: -i -e {"name": "myfile.txt", "probmethod": "weighted_average", "searchdomain": "mysearchdomain", "attributes": {"mimetype": "text-plain"}, "datapoints": [{"name": "text", "text": "this is the full text", "probmethod_embedding": "weighted_average", "model": ["bge-m3", "nomic-embed-text", "paraphrase-multilingual"]}, {"name": "filepath", "text": "/home/myuser/myfile.txt", "probmethod_embedding": "weighted_average", "model": ["bge-m3", "nomic-embed-text", "paraphrase-multilingual"]}]}
{
[Option('s', Required = true, HelpText = "Searchdomain the entity belongs to")]
public required string Searchdomain { get; set; }
[Option('e', Required = false, HelpText = "Entity (as JSON) to be inserted")]
public string? EntityJSON { get; set; }
/* Example for an entity:
{
"name": "myfile.txt",
"probmethod": "weighted_average",
"searchdomain": "mysearchdomain",
"attributes": {
"mimetype": "text-plain"
},
"datapoints": [
{
"name": "text",
"text": "this is the full text",
"probmethod_embedding": "weighted_average",
"model": [
"bge-m3",
"nomic-embed-text",
"paraphrase-multilingual"
]
},
{
"name": "filepath",
"text": "/home/myuser/myfile.txt",
"probmethod_embedding": "weighted_average",
"model": [
"bge-m3",
"nomic-embed-text",
"paraphrase-multilingual"
]
}
]
}
*/
[Option('o', "ollama", Required = true, HelpText = "Ollama URL")]
public required string OllamaURL { get; set; }
}
public class OptionsEntityRemove : OptionsEntity
{
[Option('s', Required = true, HelpText = "Searchdomain the entity belongs to")]
public required string Searchdomain { get; set; }
[Option('n', Required = true, HelpText = "Name of the entity")]
public required string Name { get; set; }
}
public class OptionsEntityList : OptionsEntity
{
[Option('s', Required = true, HelpText = "Searchdomain the entity belongs to")]
public required string Searchdomain { get; set; }
}

349
src/cli/Program.cs
View File

@@ -1,349 +0,0 @@
using System.Drawing.Printing;
using Microsoft.Extensions.AI;
using OllamaSharp;
using OllamaSharp.Models;
using CommandLine;
using cli;
using System.ComponentModel.DataAnnotations;
using System.ComponentModel;
using Org.BouncyCastle.Asn1.X509.Qualified;
using Microsoft.Identity.Client;
using System.Text.Json.Serialization;
using System.Text.Json;
using Server;
// ParserSettings parserSettings = new()
// {
// IgnoreUnknownArguments = true
// };
Parser parser = new(settings =>
{
settings.HelpWriter = Console.Error;
settings.IgnoreUnknownArguments = true;
});
int retval = 0;
parser.ParseArguments<OptionsCommand>(args).WithParsed<OptionsCommand>(opts =>
{
if (opts.IsDatabase)
{
parser.ParseArguments<OptionsDatabase>(args).WithParsed<OptionsDatabase>(opts =>
{
Searchdomain searchdomain = GetSearchdomain("http://localhost", "", opts.IP, opts.Username, opts.Password, true); // http://localhost is merely a placeholder.
Dictionary<string, dynamic> parameters = [];
System.Data.Common.DbDataReader reader = searchdomain.ExecuteSQLCommand("show tables", parameters);
bool hasTables = reader.Read();
if (!hasTables)
{
reader.Close();
Console.WriteLine("Your database has no tables.");
if (opts.SetupDatabase)
{
Console.WriteLine("Setting up tables.");
searchdomain.ExecuteSQLNonQuery("CREATE TABLE searchdomain (id int PRIMARY KEY auto_increment, name varchar(512), settings JSON);", parameters);
searchdomain.ExecuteSQLNonQuery("CREATE TABLE entity (id int PRIMARY KEY auto_increment, name varchar(512), probmethod varchar(128), id_searchdomain int, FOREIGN KEY (id_searchdomain) REFERENCES searchdomain(id));", parameters);
searchdomain.ExecuteSQLNonQuery("CREATE TABLE attribute (id int PRIMARY KEY auto_increment, id_entity int, attribute varchar(512), value longtext, FOREIGN KEY (id_entity) REFERENCES entity(id));", parameters);
searchdomain.ExecuteSQLNonQuery("CREATE TABLE datapoint (id int PRIMARY KEY auto_increment, name varchar(512), probmethod_embedding varchar(512), id_entity int, FOREIGN KEY (id_entity) REFERENCES entity(id));", parameters);
searchdomain.ExecuteSQLNonQuery("CREATE TABLE embedding (id int PRIMARY KEY auto_increment, id_datapoint int, model varchar(512), embedding blob, FOREIGN KEY (id_datapoint) REFERENCES datapoint(id));", parameters);
Console.WriteLine("Your database is ready to use.");
} else
{
Console.WriteLine("Add the parameter `--setup` if you want the tables to be created for you.");
}
} else
{
List<string> tables = ["attribute", "datapoint", "embedding", "entity", "searchdomain"];
Console.WriteLine("Your database is read-accessible and has the following tables:");
while (hasTables)
{
string table = reader.GetString(0);
Console.WriteLine($" - {table}");
try
{
tables.Remove(table);
} catch (Exception) {}
hasTables = reader.Read();
}
if (tables.Count == 0)
{
Console.WriteLine("It looks like all necessary tables are there.");
}
Console.WriteLine("There is no check in place (yet) as to whether each table is formatted correctly and the data is consistent. Also this does not test write access. Good luck.");
}
})
.WithNotParsed<OptionsDatabase>(action =>
{
PrintErrorMissingParameters("database");
retval = 1;
});
} else if (opts.IsSearchdomain)
{
parser.ParseArguments<OptionsSearchdomain>(args).WithParsed<OptionsSearchdomain>(opts =>
{
if (opts.IsCreate)
{
parser.ParseArguments<OptionsSearchdomainCreate>(args).WithParsed<OptionsSearchdomainCreate>(opts =>
{
Searchdomain searchdomain = GetSearchdomain("http://localhost", "", opts.IP, opts.Username, opts.Password, true); // http://localhost is merely a placeholder. // TODO implement a cleaner workaround
int id = searchdomain.DatabaseInsertSearchdomain(opts.Searchdomain);
Console.WriteLine($"The searchdomain was created under the following ID: {id}");
})
.WithNotParsed<OptionsSearchdomainCreate>(action =>
{
PrintErrorMissingParameters("searchdomain --create");
retval = 1;
});
} else if (opts.IsList)
{
parser.ParseArguments<OptionsSearchdomainList>(args).WithParsed<OptionsSearchdomainList>(opts =>
{
Searchdomain searchdomain = GetSearchdomain("http://localhost", "", opts.IP, opts.Username, opts.Password, true);
System.Data.Common.DbDataReader search = searchdomain.ExecuteSQLCommand("SELECT name FROM searchdomain", []);
Console.WriteLine("Searchdomains:");
while (search.Read())
{
Console.WriteLine($" - {search.GetString(0)}");
}
search.Close();
})
.WithNotParsed<OptionsSearchdomainList>(action =>
{
PrintErrorMissingParameters("searchdomain --list");
retval = 1;
});
} else if (opts.IsUpdate)
{
parser.ParseArguments<OptionsSearchdomainUpdate>(args).WithParsed<OptionsSearchdomainUpdate>(opts =>
{
if (opts.Name is null && opts.Settings is null)
{
Console.WriteLine("Warning: You did not specify either a new name or new settings. This will run, but with no effects."); // TODO add settings so this actually does not have any effect
}
Searchdomain searchdomainDry = GetSearchdomain("http://localhost:11434", "", opts.IP, opts.Username, opts.Password, true);
var search = searchdomainDry.ExecuteSQLCommand("SELECT * FROM searchdomain where name = @name", new() {{"name", opts.Searchdomain}});
bool hasSearchdomain = search.Read();
search.Close();
if (hasSearchdomain)
{
Searchdomain searchdomain = GetSearchdomain("http://localhost:11434", opts.Searchdomain, opts.IP, opts.Username, opts.Password);
Dictionary<string, dynamic> parameters = new()
{
{"name", opts.Name ?? opts.Searchdomain},
{"settings", opts.Settings ?? "{}"}, // TODO add settings.
{"id", searchdomain.id}
};
searchdomain.ExecuteSQLNonQuery("UPDATE searchdomain set name = @name, settings = @settings WHERE id = @id", parameters);
Console.WriteLine("Updated the searchdomain.");
} else
{
Console.WriteLine("No searchdomain under this name found.");
retval = 1;
}
})
.WithNotParsed<OptionsSearchdomainUpdate>(action =>
{
PrintErrorMissingParameters("searchdomain --list");
retval = 1;
});
} else if (opts.IsDelete)
{
parser.ParseArguments<OptionsSearchdomainDelete>(args).WithParsed<OptionsSearchdomainDelete>(opts =>
{
Searchdomain searchdomain = GetSearchdomain("http://localhost:11434", opts.Searchdomain, opts.IP, opts.Username, opts.Password);
int counter = 0;
foreach (Entity entity in searchdomain.entityCache)
{
searchdomain.RemoveEntity(entity.name);
counter += 1;
}
Console.WriteLine($"Number of entities deleted as part of deleting the searchdomain: {counter}");
searchdomain.ExecuteSQLNonQuery("DELETE FROM entity WHERE id_searchdomain = @id", new() {{"id", searchdomain.id}}); // Cleanup // TODO add rows affected
searchdomain.ExecuteSQLNonQuery("DELETE FROM searchdomain WHERE name = @name", new() {{"name", opts.Searchdomain}});
Console.WriteLine("Searchdomain has been successfully removed.");
})
.WithNotParsed<OptionsSearchdomainDelete>(action =>
{
PrintErrorMissingParameters("searchdomain --list");
retval = 1;
});
}
})
.WithNotParsed<OptionsSearchdomain>(action =>
{
PrintErrorMissingParameters("searchdomain");
retval = 1;
});
} else if (opts.IsEntity)
{
parser.ParseArguments<OptionsEntity>(args).WithParsed<OptionsEntity>(opts =>
{
if (opts.IsEvaluate)
{
parser.ParseArguments<OptionsEntityQuery>(args).WithParsed<OptionsEntityQuery>(opts =>
{
Console.WriteLine("The results:");
var search = Search(opts);
int max = opts.Num;
if (max > search.Count)
{
max = search.Count;
}
for (int i = 0; i < max; i++)
{
Console.WriteLine($"{search[i].Item1} {search[i].Item2}");
}
})
.WithNotParsed<OptionsEntityQuery>(action =>
{
PrintErrorUndeterminedAction("entity");
retval = 1;
});
} else if (opts.IsIndex)
{
parser.ParseArguments<OptionsEntityIndex>(args).WithParsed<OptionsEntityIndex>(opts =>
{
if (opts.EntityJSON is null)
{
opts.EntityJSON = Console.In.ReadToEnd();
}
Searchdomain searchdomain = GetSearchdomain(opts.OllamaURL, opts.Searchdomain, opts.IP, opts.Username, opts.Password);
try
{
if (opts.EntityJSON.StartsWith('[')) // multiple entities
{
List<JSONEntity>? jsonEntities = JsonSerializer.Deserialize<List<JSONEntity>?>(opts.EntityJSON);
if (jsonEntities is not null)
{
List<Entity>? entities = searchdomain.EntitiesFromJSON(opts.EntityJSON);
if (entities is not null)
{
Console.WriteLine("Successfully created/updated the entity");
} else
{
Console.Error.WriteLine("Unable to create the entity using the provided JSON.");
retval = 1;
}
}
} else
{
Entity? entity = searchdomain.EntityFromJSON(opts.EntityJSON);
if (entity is not null)
{
Console.WriteLine("Successfully created/updated the entity");
} else
{
Console.Error.WriteLine("Unable to create the entity using the provided JSON.");
retval = 1;
}
}
} catch (Exception e)
{
Console.Error.WriteLine($"Unable to create the entity using the provided JSON.\nException: {e}");
retval = 1;
}
})
.WithNotParsed<OptionsEntityIndex>(action =>
{
PrintErrorMissingParameters("entity --index");
retval = 1;
});
} else if (opts.IsDelete)
{
parser.ParseArguments<OptionsEntityRemove>(args).WithParsed<OptionsEntityRemove>(opts =>
{
Searchdomain searchdomain = GetSearchdomain("http://localhost:11434", opts.Searchdomain, opts.IP, opts.Username, opts.Password);
bool hasEntity = searchdomain.HasEntity(opts.Name);
if (hasEntity)
{
searchdomain.RemoveEntity(opts.Name);
} else
{
Console.Error.WriteLine($"No entity with the name {opts.Name} has been found.");
retval = 1;
}
})
.WithNotParsed<OptionsEntityRemove>(action =>
{
PrintErrorMissingParameters("entity --remove");
retval = 1;
});
} else if (opts.IsList)
{
parser.ParseArguments<OptionsEntityList>(args).WithParsed<OptionsEntityList>(opts =>
{
Searchdomain searchdomain = GetSearchdomain("http://localhost:11434", opts.Searchdomain, opts.IP, opts.Username, opts.Password);
Console.WriteLine("Entities:");
foreach (Entity entity in searchdomain.entityCache)
{
Dictionary<string, string> datapointNames = [];
foreach (Datapoint datapoint in entity.datapoints)
{
datapointNames[datapoint.name] = datapoint.probMethod.Method.Name;
}
Console.WriteLine($"- {entity.name} | {JsonSerializer.Serialize(entity.attributes)} | {JsonSerializer.Serialize(datapointNames)}");
}
})
.WithNotParsed<OptionsEntityList>(action =>
{
PrintErrorMissingParameters("entity --list");
retval = 1;
});
} else
{
PrintErrorUndeterminedAction("entity");
retval = 1;
}
});
} else
{
Console.Error.WriteLine($"Unable to parse {args[0]}. Needs to be \"database\", \"searchdomain\", or \"entity\".");
}
});
return retval;
static List<(float, string)> Search(OptionsEntityQuery optionsEntityIndex)
{
var searchdomain = GetSearchdomain(optionsEntityIndex.OllamaURL, optionsEntityIndex.Searchdomain, optionsEntityIndex.IP, optionsEntityIndex.Username, optionsEntityIndex.Password);
List<(float, string)> results = searchdomain.Search(optionsEntityIndex.Query);
return results;
}
static Searchdomain GetSearchdomain(string ollamaURL, string searchdomain, string ip, string username, string password, bool runEmpty = false)
{
string connectionString = $"server={ip};database=embeddingsearch;uid={username};pwd={password};";
// var ollamaConfig = new OllamaApiClient.Configuration
// {
// Uri = new Uri(ollamaURL)
// };
var httpClient = new HttpClient
{
BaseAddress = new Uri(ollamaURL),
Timeout = TimeSpan.FromSeconds(36000) //.MaxValue //FromSeconds(timeout)
};
var ollama = new OllamaApiClient(httpClient);
return new Searchdomain(searchdomain, connectionString, ollama, "sqlserver", runEmpty);
}
static void PrintErrorUndeterminedAction(string prefix)
{
PrintErrorReferToHelp("Unable to determine an action", prefix);
}
static void PrintErrorMissingParameters(string prefix)
{
PrintErrorReferToHelp("Not all required parameters were specified", prefix);
}
static void PrintErrorReferToHelp(string text, string prefix) // TODO make this not static and set retval to not zero
{
Console.Error.WriteLine($"{text}. Please use `{prefix} --help` for more info");
}

19
src/cli/cli.csproj
View File

@@ -1,19 +0,0 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="commandlineparser">
<Version>2.9.1</Version>
</PackageReference>
<PackageReference Include="OllamaSharp">
<Version>5.1.13</Version>
</PackageReference>
</ItemGroup>
<ItemGroup>
<ProjectReference Include="../Server/Server.csproj" />
</ItemGroup>
</Project>