From ad599433bf0664d2f906979799792c46a22207c9 Mon Sep 17 00:00:00 2001 From: LD-Reborn Date: Fri, 6 Jun 2025 00:29:38 +0200 Subject: [PATCH] Removed CLI project, added database migrations, added SQLHelper --- README.md | 2 + embeddingsearch.sln | 10 - src/Server/Exceptions/DatabaseExceptions.cs | 14 + src/Server/Migrations/DatabaseMigrations.cs | 69 ++++ src/Server/SQLHelper.cs | 49 +++ src/Server/SearchdomainManager.cs | 19 +- src/Server/Server.csproj | 5 - src/cli/Options.cs | 171 ---------- src/cli/Program.cs | 349 -------------------- src/cli/cli.csproj | 19 -- 10 files changed, 152 insertions(+), 555 deletions(-) create mode 100644 src/Server/Exceptions/DatabaseExceptions.cs create mode 100644 src/Server/Migrations/DatabaseMigrations.cs create mode 100644 src/Server/SQLHelper.cs delete mode 100644 src/cli/Options.cs delete mode 100644 src/cli/Program.cs delete mode 100644 src/cli/cli.csproj diff --git a/README.md b/README.md index da511cf..328ac97 100644 --- a/README.md +++ b/README.md @@ -82,6 +82,8 @@ All commands, parameters and examples are documented here: [docs/CLI.md](docs/CL - Remove the `id` collumns from the database tables where the table is actually identified (and should be unique by) the name, which should become the new primary key. - Improve performance & latency (Create ready-to-go processes where each contain an n'th share of the entity cache, ready to perform a query. Prepare it after creating the entity cache.) - Make the API server (and indexer, once it is done) a docker container +- Implement dynamic invocation based database migrations +- Remove remaining DRY violations using the SQLHelper # Future features - Support for other database types (MSSQL, SQLite) diff --git a/embeddingsearch.sln b/embeddingsearch.sln index 9061e57..d214497 100644 --- a/embeddingsearch.sln +++ b/embeddingsearch.sln @@ -7,10 +7,6 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{6AA0A9E0-A36 EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Server", "src\Server\Server.csproj", "{643CB1D1-02F6-4BCC-A1CC-6E403D78C442}" EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "cli", "cli", "{BC4F3063-B921-4C4A-A7CE-11FAF5B73D50}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "cli", "src\cli\cli.csproj", "{D61A2C50-B46C-42BA-B75D-E84D8FA28C29}" -EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Client", "src\Client\Client.csproj", "{A8EBB748-5BBA-47EB-840D-E398365C52A2}" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Indexer", "src\Indexer\Indexer.csproj", "{5361FD10-E85C-496C-9BEF-9232F767F904}" @@ -25,10 +21,6 @@ Global {643CB1D1-02F6-4BCC-A1CC-6E403D78C442}.Debug|Any CPU.Build.0 = Debug|Any CPU {643CB1D1-02F6-4BCC-A1CC-6E403D78C442}.Release|Any CPU.ActiveCfg = Release|Any CPU {643CB1D1-02F6-4BCC-A1CC-6E403D78C442}.Release|Any CPU.Build.0 = Release|Any CPU - {D61A2C50-B46C-42BA-B75D-E84D8FA28C29}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {D61A2C50-B46C-42BA-B75D-E84D8FA28C29}.Debug|Any CPU.Build.0 = Debug|Any CPU - {D61A2C50-B46C-42BA-B75D-E84D8FA28C29}.Release|Any CPU.ActiveCfg = Release|Any CPU - {D61A2C50-B46C-42BA-B75D-E84D8FA28C29}.Release|Any CPU.Build.0 = Release|Any CPU {A8EBB748-5BBA-47EB-840D-E398365C52A2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {A8EBB748-5BBA-47EB-840D-E398365C52A2}.Debug|Any CPU.Build.0 = Debug|Any CPU {A8EBB748-5BBA-47EB-840D-E398365C52A2}.Release|Any CPU.ActiveCfg = Release|Any CPU @@ -43,8 +35,6 @@ Global EndGlobalSection GlobalSection(NestedProjects) = preSolution {643CB1D1-02F6-4BCC-A1CC-6E403D78C442} = {6AA0A9E0-A361-4E86-BA02-D5F6779C6DEF} - {BC4F3063-B921-4C4A-A7CE-11FAF5B73D50} = {6AA0A9E0-A361-4E86-BA02-D5F6779C6DEF} - {D61A2C50-B46C-42BA-B75D-E84D8FA28C29} = {BC4F3063-B921-4C4A-A7CE-11FAF5B73D50} {A8EBB748-5BBA-47EB-840D-E398365C52A2} = {6AA0A9E0-A361-4E86-BA02-D5F6779C6DEF} {5361FD10-E85C-496C-9BEF-9232F767F904} = {6AA0A9E0-A361-4E86-BA02-D5F6779C6DEF} EndGlobalSection diff --git a/src/Server/Exceptions/DatabaseExceptions.cs b/src/Server/Exceptions/DatabaseExceptions.cs new file mode 100644 index 0000000..46149bd --- /dev/null +++ b/src/Server/Exceptions/DatabaseExceptions.cs @@ -0,0 +1,14 @@ +namespace Server.Exceptions; + +public class DatabaseVersionException : Exception +{ + public DatabaseVersionException() + : base("DatabaseVersion could not be parsed as integer. Please ensure the DatabaseVersion can be parsed as an integer.") + { + } + + public DatabaseVersionException(string message) + : base(message) + { + } +} \ No newline at end of file diff --git a/src/Server/Migrations/DatabaseMigrations.cs b/src/Server/Migrations/DatabaseMigrations.cs new file mode 100644 index 0000000..a6b60c1 --- /dev/null +++ b/src/Server/Migrations/DatabaseMigrations.cs @@ -0,0 +1,69 @@ +using System.Data.Common; +using Server.Exceptions; + +namespace Server.Migrations; + +public static class DatabaseMigrations +{ + public static void Migrate(SQLHelper helper) + { + int databaseVersion = DatabaseGetVersion(helper); + switch (databaseVersion) + { + case 0: + databaseVersion = Create(helper); + goto case 1; // Here lies a dead braincell. + case 1: + databaseVersion = UpdateFrom1(helper); // TODO: Implement reflection based dynamic invocation. + goto case 2; + case 2: + default: + break; + } + } + public static int DatabaseGetVersion(SQLHelper helper) + { + DbDataReader reader = helper.ExecuteSQLCommand("show tables", []); + bool hasTables = reader.Read(); + reader.Close(); + if (!hasTables) + { + return 0; + } + + reader = helper.ExecuteSQLCommand("show tables like '%settings%'", []); + bool hasSystemTable = reader.Read(); + reader.Close(); + if (!hasSystemTable) + { + return 1; + } + reader = helper.ExecuteSQLCommand("SELECT value FROM settings WHERE name=\"DatabaseVersion\"", []); + reader.Read(); + string rawVersion = reader.GetString(0); + reader.Close(); + bool success = int.TryParse(rawVersion, out int version); + if (!success) + { + throw new DatabaseVersionException(); + } + return version; + } + + public static int Create(SQLHelper helper) + { + helper.ExecuteSQLNonQuery("CREATE TABLE searchdomain (id int PRIMARY KEY auto_increment, name varchar(512), settings JSON);", []); + helper.ExecuteSQLNonQuery("CREATE TABLE entity (id int PRIMARY KEY auto_increment, name varchar(512), probmethod varchar(128), id_searchdomain int, FOREIGN KEY (id_searchdomain) REFERENCES searchdomain(id));", []); + helper.ExecuteSQLNonQuery("CREATE TABLE attribute (id int PRIMARY KEY auto_increment, id_entity int, attribute varchar(512), value longtext, FOREIGN KEY (id_entity) REFERENCES entity(id));", []); + helper.ExecuteSQLNonQuery("CREATE TABLE datapoint (id int PRIMARY KEY auto_increment, name varchar(512), probmethod_embedding varchar(512), id_entity int, FOREIGN KEY (id_entity) REFERENCES entity(id));", []); + helper.ExecuteSQLNonQuery("CREATE TABLE embedding (id int PRIMARY KEY auto_increment, id_datapoint int, model varchar(512), embedding blob, FOREIGN KEY (id_datapoint) REFERENCES datapoint(id));", []); + return 1; + } + + public static int UpdateFrom1(SQLHelper helper) + { + helper.ExecuteSQLNonQuery("CREATE TABLE settings (name varchar(512), value varchar(8192));", []); + helper.ExecuteSQLNonQuery("INSERT INTO settings (name, value) VALUES (\"DatabaseVersion\", \"2\");", []); + return 2; + } +} \ No newline at end of file diff --git a/src/Server/SQLHelper.cs b/src/Server/SQLHelper.cs new file mode 100644 index 0000000..8d493a5 --- /dev/null +++ b/src/Server/SQLHelper.cs @@ -0,0 +1,49 @@ +using System.Data.Common; +using MySql.Data.MySqlClient; + +namespace Server; + +public class SQLHelper +{ + public MySqlConnection connection; + public SQLHelper(MySqlConnection connection) + { + this.connection = connection; + } + public DbDataReader ExecuteSQLCommand(string query, Dictionary parameters) + { + using MySqlCommand command = connection.CreateCommand(); + command.CommandText = query; + foreach (KeyValuePair parameter in parameters) + { + command.Parameters.AddWithValue($"@{parameter.Key}", parameter.Value); + } + return command.ExecuteReader(); + } + + public void ExecuteSQLNonQuery(string query, Dictionary parameters) + { + using MySqlCommand command = connection.CreateCommand(); + + command.CommandText = query; + foreach (KeyValuePair parameter in parameters) + { + command.Parameters.AddWithValue($"@{parameter.Key}", parameter.Value); + } + command.ExecuteNonQuery(); + } + + public int ExecuteSQLCommandGetInsertedID(string query, Dictionary parameters) + { + using MySqlCommand command = connection.CreateCommand(); + + command.CommandText = query; + foreach (KeyValuePair parameter in parameters) + { + command.Parameters.AddWithValue($"@{parameter.Key}", parameter.Value); + } + command.ExecuteNonQuery(); + command.CommandText = "SELECT LAST_INSERT_ID();"; + return Convert.ToInt32(command.ExecuteScalar()); + } +} \ No newline at end of file diff --git a/src/Server/SearchdomainManager.cs b/src/Server/SearchdomainManager.cs index 30fb0c3..0a31838 100644 --- a/src/Server/SearchdomainManager.cs +++ b/src/Server/SearchdomainManager.cs @@ -3,6 +3,7 @@ using System.Data.Common; using OllamaSharp; using Microsoft.IdentityModel.Tokens; using Server.Exceptions; +using Server.Migrations; namespace Server; @@ -29,8 +30,11 @@ public class SearchdomainManager client = new(new Uri(ollamaURL)); connection = new MySqlConnection(connectionString); connection.Open(); + DatabaseMigrations.Migrate(new SQLHelper(connection)); } + + public Searchdomain GetSearchdomain(string searchdomain) { if (searchdomains.TryGetValue(searchdomain, out Searchdomain? value)) @@ -40,7 +44,8 @@ public class SearchdomainManager try { return SetSearchdomain(searchdomain, new Searchdomain(searchdomain, connectionString, client)); - } catch (MySqlException) + } + catch (MySqlException) { _logger.LogError("Unable to find the searchdomain {searchdomain}", searchdomain); throw new Exception($"Unable to find the searchdomain {searchdomain}"); @@ -105,6 +110,18 @@ public class SearchdomainManager return command.ExecuteReader(); } + public void ExecuteSQLNonQuery(string query, Dictionary parameters) + { + using MySqlCommand command = connection.CreateCommand(); + + command.CommandText = query; + foreach (KeyValuePair parameter in parameters) + { + command.Parameters.AddWithValue($"@{parameter.Key}", parameter.Value); + } + command.ExecuteNonQuery(); + } + public int ExecuteSQLCommandGetInsertedID(string query, Dictionary parameters) { using MySqlCommand command = connection.CreateCommand(); diff --git a/src/Server/Server.csproj b/src/Server/Server.csproj index 49be5bf..2d00256 100644 --- a/src/Server/Server.csproj +++ b/src/Server/Server.csproj @@ -18,9 +18,4 @@ - - - - - diff --git a/src/cli/Options.cs b/src/cli/Options.cs deleted file mode 100644 index 5268224..0000000 --- a/src/cli/Options.cs +++ /dev/null @@ -1,171 +0,0 @@ -using CommandLine; -namespace cli; - -public class OptionsCommand -{ - [Option("database", Required = false, HelpText = "Do things related to the database")] // Create database / ensure it is set up correctly - public bool IsDatabase { get; set; } - - [Option("searchdomain", Required = false, HelpText = "Execute CRUD on searchdomains")] - public bool IsSearchdomain { get; set; } - - [Option("entity", Required = false, HelpText = "Execute CRUD on entities")] - public bool IsEntity { get; set; } - - [Option('h', "host", Required = true, HelpText = "Host IP address (e.g. 192.168.0.75)")] - public required string IP { get; set; } - - [Option('p', "port", Required = true, HelpText = "Host port (e.g. 3306)")] - public required int Port { get; set; } - - [Option('U', "username", Required = true, HelpText = "Username for the MySQL database")] - public required string Username { get; set; } - - [Option('P', "password", Required = true, HelpText = "Password for the MySQL database")] - public required string Password { get; set; } - -} - -public class OptionsDatabase : OptionsCommand -{ - [Option("setup", Required = false, HelpText = "Ensure the database is set up correctly")] - public bool SetupDatabase { get; set; } -} - - -public class OptionsSearchdomain : OptionsCommand -{ - [Option("create", Required = false, HelpText = "Create a searchdomain")] - public bool IsCreate { get; set; } - - [Option("list", Required = false, HelpText = "Lists the searchdomains")] - public bool IsList { get; set; } - - [Option("update", Required = false, HelpText = "Update a searchdomain (settings, name)")] - public bool IsUpdate { get; set; } - - [Option("delete", Required = false, HelpText = "Delete a searchdomain")] - public bool IsDelete { get; set; } -} - - -public class OptionsSearchdomainCreate : OptionsSearchdomain -{ - [Option('s', Required = true, HelpText = "Name of the searchdomain to create")] - public required string Searchdomain { get; set; } -} - -public class OptionsSearchdomainList : OptionsSearchdomain -{ - // The cleanest piece of code in this project -} - -public class OptionsSearchdomainUpdate : OptionsSearchdomain -{ - [Option('s', Required = true, HelpText = "Name of the searchdomain to update")] - public required string Searchdomain { get; set; } - - [Option('n', Required = false, HelpText = "New name to set")] - public string? Name { get; set; } - - [Option('S', Required = false, HelpText = "New Settings (as json)")] - public string? Settings { get; set; } -} - -public class OptionsSearchdomainDelete : OptionsSearchdomain -{ - [Option('s', Required = true, HelpText = "Name of the searchdomain to delete")] - public required string Searchdomain { get; set; } -} - - -public class OptionsEntity : OptionsCommand -{ - [Option("evaluate", Required = false, HelpText = "Evaluate a query")] - public bool IsEvaluate { get; set; } - - [Option("index", Required = false, HelpText = "Create or update an entity from a JSON string")] - public bool IsIndex { get; set; } - - [Option("remove", Required = false, HelpText = "Remove an entity")] - public bool IsDelete { get; set; } - - [Option("list", Required = false, HelpText = "List all entities")] - public bool IsList { get; set; } - -} - -public class OptionsEntityQuery : OptionsEntity -{ - [Option('s', Required = true, HelpText = "Searchdomain to be searched")] - public required string Searchdomain { get; set; } - - [Option('q', "query", Required = true, HelpText = "Query string to evaluate the entities against")] - public required string Query { get; set; } - - [Option('o', "ollama", Required = true, HelpText = "Ollama URL")] - public required string OllamaURL { get; set; } - - [Option('n', "num", Required = false, HelpText = "(Maximum) number of results to output", Default = 5)] - public int Num { get; set; } -} - -public class OptionsEntityIndex : OptionsEntity // Example: -i -e {"name": "myfile.txt", "probmethod": "weighted_average", "searchdomain": "mysearchdomain", "attributes": {"mimetype": "text-plain"}, "datapoints": [{"name": "text", "text": "this is the full text", "probmethod_embedding": "weighted_average", "model": ["bge-m3", "nomic-embed-text", "paraphrase-multilingual"]}, {"name": "filepath", "text": "/home/myuser/myfile.txt", "probmethod_embedding": "weighted_average", "model": ["bge-m3", "nomic-embed-text", "paraphrase-multilingual"]}]} -{ - [Option('s', Required = true, HelpText = "Searchdomain the entity belongs to")] - public required string Searchdomain { get; set; } - - [Option('e', Required = false, HelpText = "Entity (as JSON) to be inserted")] - public string? EntityJSON { get; set; } - - /* Example for an entity: - { - "name": "myfile.txt", - "probmethod": "weighted_average", - "searchdomain": "mysearchdomain", - "attributes": { - "mimetype": "text-plain" - }, - "datapoints": [ - { - "name": "text", - "text": "this is the full text", - "probmethod_embedding": "weighted_average", - "model": [ - "bge-m3", - "nomic-embed-text", - "paraphrase-multilingual" - ] - }, - { - "name": "filepath", - "text": "/home/myuser/myfile.txt", - "probmethod_embedding": "weighted_average", - "model": [ - "bge-m3", - "nomic-embed-text", - "paraphrase-multilingual" - ] - } - ] - } - */ - - [Option('o', "ollama", Required = true, HelpText = "Ollama URL")] - public required string OllamaURL { get; set; } -} - -public class OptionsEntityRemove : OptionsEntity -{ - [Option('s', Required = true, HelpText = "Searchdomain the entity belongs to")] - public required string Searchdomain { get; set; } - - [Option('n', Required = true, HelpText = "Name of the entity")] - public required string Name { get; set; } -} - -public class OptionsEntityList : OptionsEntity -{ - [Option('s', Required = true, HelpText = "Searchdomain the entity belongs to")] - public required string Searchdomain { get; set; } -} \ No newline at end of file diff --git a/src/cli/Program.cs b/src/cli/Program.cs deleted file mode 100644 index 3b7124a..0000000 --- a/src/cli/Program.cs +++ /dev/null @@ -1,349 +0,0 @@ -using System.Drawing.Printing; -using Microsoft.Extensions.AI; -using OllamaSharp; -using OllamaSharp.Models; -using CommandLine; -using cli; -using System.ComponentModel.DataAnnotations; -using System.ComponentModel; -using Org.BouncyCastle.Asn1.X509.Qualified; -using Microsoft.Identity.Client; -using System.Text.Json.Serialization; -using System.Text.Json; -using Server; - -// ParserSettings parserSettings = new() -// { -// IgnoreUnknownArguments = true -// }; - -Parser parser = new(settings => -{ - settings.HelpWriter = Console.Error; - settings.IgnoreUnknownArguments = true; -}); - -int retval = 0; - -parser.ParseArguments(args).WithParsed(opts => -{ - if (opts.IsDatabase) - { - parser.ParseArguments(args).WithParsed(opts => - { - Searchdomain searchdomain = GetSearchdomain("http://localhost", "", opts.IP, opts.Username, opts.Password, true); // http://localhost is merely a placeholder. - - Dictionary parameters = []; - System.Data.Common.DbDataReader reader = searchdomain.ExecuteSQLCommand("show tables", parameters); - bool hasTables = reader.Read(); - if (!hasTables) - { - reader.Close(); - Console.WriteLine("Your database has no tables."); - if (opts.SetupDatabase) - { - Console.WriteLine("Setting up tables."); - searchdomain.ExecuteSQLNonQuery("CREATE TABLE searchdomain (id int PRIMARY KEY auto_increment, name varchar(512), settings JSON);", parameters); - searchdomain.ExecuteSQLNonQuery("CREATE TABLE entity (id int PRIMARY KEY auto_increment, name varchar(512), probmethod varchar(128), id_searchdomain int, FOREIGN KEY (id_searchdomain) REFERENCES searchdomain(id));", parameters); - searchdomain.ExecuteSQLNonQuery("CREATE TABLE attribute (id int PRIMARY KEY auto_increment, id_entity int, attribute varchar(512), value longtext, FOREIGN KEY (id_entity) REFERENCES entity(id));", parameters); - searchdomain.ExecuteSQLNonQuery("CREATE TABLE datapoint (id int PRIMARY KEY auto_increment, name varchar(512), probmethod_embedding varchar(512), id_entity int, FOREIGN KEY (id_entity) REFERENCES entity(id));", parameters); - searchdomain.ExecuteSQLNonQuery("CREATE TABLE embedding (id int PRIMARY KEY auto_increment, id_datapoint int, model varchar(512), embedding blob, FOREIGN KEY (id_datapoint) REFERENCES datapoint(id));", parameters); - Console.WriteLine("Your database is ready to use."); - } else - { - Console.WriteLine("Add the parameter `--setup` if you want the tables to be created for you."); - } - } else - { - List tables = ["attribute", "datapoint", "embedding", "entity", "searchdomain"]; - Console.WriteLine("Your database is read-accessible and has the following tables:"); - while (hasTables) - { - string table = reader.GetString(0); - Console.WriteLine($" - {table}"); - try - { - tables.Remove(table); - } catch (Exception) {} - hasTables = reader.Read(); - } - if (tables.Count == 0) - { - Console.WriteLine("It looks like all necessary tables are there."); - } - Console.WriteLine("There is no check in place (yet) as to whether each table is formatted correctly and the data is consistent. Also this does not test write access. Good luck."); - } - }) - .WithNotParsed(action => - { - PrintErrorMissingParameters("database"); - retval = 1; - }); - } else if (opts.IsSearchdomain) - { - parser.ParseArguments(args).WithParsed(opts => - { - if (opts.IsCreate) - { - parser.ParseArguments(args).WithParsed(opts => - { - Searchdomain searchdomain = GetSearchdomain("http://localhost", "", opts.IP, opts.Username, opts.Password, true); // http://localhost is merely a placeholder. // TODO implement a cleaner workaround - int id = searchdomain.DatabaseInsertSearchdomain(opts.Searchdomain); - Console.WriteLine($"The searchdomain was created under the following ID: {id}"); - }) - .WithNotParsed(action => - { - PrintErrorMissingParameters("searchdomain --create"); - retval = 1; - }); - } else if (opts.IsList) - { - parser.ParseArguments(args).WithParsed(opts => - { - Searchdomain searchdomain = GetSearchdomain("http://localhost", "", opts.IP, opts.Username, opts.Password, true); - System.Data.Common.DbDataReader search = searchdomain.ExecuteSQLCommand("SELECT name FROM searchdomain", []); - Console.WriteLine("Searchdomains:"); - while (search.Read()) - { - Console.WriteLine($" - {search.GetString(0)}"); - } - search.Close(); - }) - .WithNotParsed(action => - { - PrintErrorMissingParameters("searchdomain --list"); - retval = 1; - }); - } else if (opts.IsUpdate) - { - parser.ParseArguments(args).WithParsed(opts => - { - if (opts.Name is null && opts.Settings is null) - { - Console.WriteLine("Warning: You did not specify either a new name or new settings. This will run, but with no effects."); // TODO add settings so this actually does not have any effect - } - Searchdomain searchdomainDry = GetSearchdomain("http://localhost:11434", "", opts.IP, opts.Username, opts.Password, true); - var search = searchdomainDry.ExecuteSQLCommand("SELECT * FROM searchdomain where name = @name", new() {{"name", opts.Searchdomain}}); - bool hasSearchdomain = search.Read(); - search.Close(); - if (hasSearchdomain) - { - Searchdomain searchdomain = GetSearchdomain("http://localhost:11434", opts.Searchdomain, opts.IP, opts.Username, opts.Password); - Dictionary parameters = new() - { - {"name", opts.Name ?? opts.Searchdomain}, - {"settings", opts.Settings ?? "{}"}, // TODO add settings. - {"id", searchdomain.id} - }; - searchdomain.ExecuteSQLNonQuery("UPDATE searchdomain set name = @name, settings = @settings WHERE id = @id", parameters); - Console.WriteLine("Updated the searchdomain."); - } else - { - Console.WriteLine("No searchdomain under this name found."); - retval = 1; - } - }) - .WithNotParsed(action => - { - PrintErrorMissingParameters("searchdomain --list"); - retval = 1; - }); - } else if (opts.IsDelete) - { - parser.ParseArguments(args).WithParsed(opts => - { - Searchdomain searchdomain = GetSearchdomain("http://localhost:11434", opts.Searchdomain, opts.IP, opts.Username, opts.Password); - int counter = 0; - foreach (Entity entity in searchdomain.entityCache) - { - searchdomain.RemoveEntity(entity.name); - counter += 1; - } - Console.WriteLine($"Number of entities deleted as part of deleting the searchdomain: {counter}"); - searchdomain.ExecuteSQLNonQuery("DELETE FROM entity WHERE id_searchdomain = @id", new() {{"id", searchdomain.id}}); // Cleanup // TODO add rows affected - searchdomain.ExecuteSQLNonQuery("DELETE FROM searchdomain WHERE name = @name", new() {{"name", opts.Searchdomain}}); - Console.WriteLine("Searchdomain has been successfully removed."); - }) - .WithNotParsed(action => - { - PrintErrorMissingParameters("searchdomain --list"); - retval = 1; - }); - } - }) - .WithNotParsed(action => - { - PrintErrorMissingParameters("searchdomain"); - retval = 1; - }); - - } else if (opts.IsEntity) - { - parser.ParseArguments(args).WithParsed(opts => - { - if (opts.IsEvaluate) - { - parser.ParseArguments(args).WithParsed(opts => - { - Console.WriteLine("The results:"); - var search = Search(opts); - int max = opts.Num; - if (max > search.Count) - { - max = search.Count; - } - for (int i = 0; i < max; i++) - { - Console.WriteLine($"{search[i].Item1} {search[i].Item2}"); - } - }) - .WithNotParsed(action => - { - PrintErrorUndeterminedAction("entity"); - retval = 1; - }); - } else if (opts.IsIndex) - { - parser.ParseArguments(args).WithParsed(opts => - { - if (opts.EntityJSON is null) - { - opts.EntityJSON = Console.In.ReadToEnd(); - } - Searchdomain searchdomain = GetSearchdomain(opts.OllamaURL, opts.Searchdomain, opts.IP, opts.Username, opts.Password); - try - { - if (opts.EntityJSON.StartsWith('[')) // multiple entities - { - List? jsonEntities = JsonSerializer.Deserialize?>(opts.EntityJSON); - if (jsonEntities is not null) - { - - List? entities = searchdomain.EntitiesFromJSON(opts.EntityJSON); - if (entities is not null) - { - Console.WriteLine("Successfully created/updated the entity"); - } else - { - Console.Error.WriteLine("Unable to create the entity using the provided JSON."); - retval = 1; - } - } - } else - { - Entity? entity = searchdomain.EntityFromJSON(opts.EntityJSON); - if (entity is not null) - { - Console.WriteLine("Successfully created/updated the entity"); - } else - { - Console.Error.WriteLine("Unable to create the entity using the provided JSON."); - retval = 1; - } - } - } catch (Exception e) - { - Console.Error.WriteLine($"Unable to create the entity using the provided JSON.\nException: {e}"); - retval = 1; - } - }) - .WithNotParsed(action => - { - PrintErrorMissingParameters("entity --index"); - retval = 1; - }); - } else if (opts.IsDelete) - { - parser.ParseArguments(args).WithParsed(opts => - { - Searchdomain searchdomain = GetSearchdomain("http://localhost:11434", opts.Searchdomain, opts.IP, opts.Username, opts.Password); - bool hasEntity = searchdomain.HasEntity(opts.Name); - if (hasEntity) - { - searchdomain.RemoveEntity(opts.Name); - } else - { - Console.Error.WriteLine($"No entity with the name {opts.Name} has been found."); - retval = 1; - } - }) - .WithNotParsed(action => - { - PrintErrorMissingParameters("entity --remove"); - retval = 1; - }); - } else if (opts.IsList) - { - parser.ParseArguments(args).WithParsed(opts => - { - Searchdomain searchdomain = GetSearchdomain("http://localhost:11434", opts.Searchdomain, opts.IP, opts.Username, opts.Password); - Console.WriteLine("Entities:"); - foreach (Entity entity in searchdomain.entityCache) - { - Dictionary datapointNames = []; - foreach (Datapoint datapoint in entity.datapoints) - { - datapointNames[datapoint.name] = datapoint.probMethod.Method.Name; - } - Console.WriteLine($"- {entity.name} | {JsonSerializer.Serialize(entity.attributes)} | {JsonSerializer.Serialize(datapointNames)}"); - } - }) - .WithNotParsed(action => - { - PrintErrorMissingParameters("entity --list"); - retval = 1; - }); - } else - { - PrintErrorUndeterminedAction("entity"); - retval = 1; - } - }); - - } else - { - Console.Error.WriteLine($"Unable to parse {args[0]}. Needs to be \"database\", \"searchdomain\", or \"entity\"."); - } -}); - -return retval; - -static List<(float, string)> Search(OptionsEntityQuery optionsEntityIndex) -{ - var searchdomain = GetSearchdomain(optionsEntityIndex.OllamaURL, optionsEntityIndex.Searchdomain, optionsEntityIndex.IP, optionsEntityIndex.Username, optionsEntityIndex.Password); - List<(float, string)> results = searchdomain.Search(optionsEntityIndex.Query); - return results; -} - - -static Searchdomain GetSearchdomain(string ollamaURL, string searchdomain, string ip, string username, string password, bool runEmpty = false) -{ - string connectionString = $"server={ip};database=embeddingsearch;uid={username};pwd={password};"; - // var ollamaConfig = new OllamaApiClient.Configuration - // { - // Uri = new Uri(ollamaURL) - // }; - var httpClient = new HttpClient - { - BaseAddress = new Uri(ollamaURL), - Timeout = TimeSpan.FromSeconds(36000) //.MaxValue //FromSeconds(timeout) - }; - var ollama = new OllamaApiClient(httpClient); - return new Searchdomain(searchdomain, connectionString, ollama, "sqlserver", runEmpty); -} - - -static void PrintErrorUndeterminedAction(string prefix) -{ - PrintErrorReferToHelp("Unable to determine an action", prefix); -} - -static void PrintErrorMissingParameters(string prefix) -{ - PrintErrorReferToHelp("Not all required parameters were specified", prefix); -} - -static void PrintErrorReferToHelp(string text, string prefix) // TODO make this not static and set retval to not zero -{ - Console.Error.WriteLine($"{text}. Please use `{prefix} --help` for more info"); -} diff --git a/src/cli/cli.csproj b/src/cli/cli.csproj deleted file mode 100644 index bf06cbf..0000000 --- a/src/cli/cli.csproj +++ /dev/null @@ -1,19 +0,0 @@ - - - Exe - net8.0 - enable - enable - - - - 2.9.1 - - - 5.1.13 - - - - - -