Added Elmah, Serilog to Server, improved logging

.gitignore

@@ -14,3 +14,4 @@ src/Indexer/bin
 src/Indexer/obj
 src/Indexer/Scripts/__pycache__
 src/Indexer/logs
+src/Server/logs

docs/Indexer.md

@@ -9,9 +9,8 @@ The indexer by default
 ## Docker installation
 (On Linux you might need root privileges, thus use `sudo` where necessary)
 1. Navigate to the `src` directory
-2. Build the docker container: `docker build -t indexer -f Indexer/Dockerfile .`
+2. Build the docker container: `docker build -t embeddingsearch-indexer -f Indexer/Dockerfile .`
-3. Run the docker container: `docker run --net=host -t indexer` (the `-t` is optional, but you get more meaningful output)
+3. Run the docker container: `docker run --net=host -t embeddingsearch-indexer` (the `-t` is optional, but you get more meaningful output. Or use `-d` to run it in the background)
-
 ## Installing the dependencies
 ## Ubuntu 24.04
 1. Install the .NET SDK: `sudo apt update && sudo apt install dotnet-sdk-8.0 -y`

src/Server/Controllers/EntityController.cs

@@ -28,6 +28,7 @@ public class EntityController : ControllerBase
             searchdomain_ = _domainManager.GetSearchdomain(searchdomain);
         } catch (Exception)
         {
+            _logger.LogError("Unable to retrieve the searchdomain {searchdomain} - it likely does not exist yet", [searchdomain]); // TODO DRY violation; perhaps move this logging to the SearchdomainManager?
             return Ok(new EntityQueryResults() {Results = []});
         }
         var results = searchdomain_.Search(query);
@@ -47,9 +48,9 @@ public class EntityController : ControllerBase
         {
             searchdomain_ = _domainManager.GetSearchdomain(searchdomain);
         }
-        catch (Exception ex)
+        catch (Exception)
         {
-            _logger.LogError("Unable to retrieve the searchdomain", [ex]);
+            _logger.LogError("Unable to retrieve the searchdomain {searchdomain} - it likely does not exist yet", [searchdomain]);
             return Ok(new EntityIndexResult() { Success = false });
         }
         List<Entity>? entities = searchdomain_.EntitiesFromJSON(JsonSerializer.Serialize(jsonEntity));
@@ -73,9 +74,9 @@ public class EntityController : ControllerBase
         try
         {
             searchdomain_ = _domainManager.GetSearchdomain(searchdomain);
-        } catch (Exception ex)
+        } catch (Exception)
         {
-            _logger.LogError("Unable to retrieve the searchdomain", [ex]);
+            _logger.LogError("Unable to retrieve the searchdomain {searchdomain} - it likely does not exist yet", [searchdomain]);
             return Ok(new EntityListResults() { Results = [], Success = false });
         }
         EntityListResults entityListResults = new() {Results = [], Success = true};
@@ -115,7 +116,7 @@ public class EntityController : ControllerBase
     }
 
     [HttpGet("Delete")]
-    public ActionResult<EntityDeleteResults> Delete(string searchdomain, string entityName) // TODO test this
+    public ActionResult<EntityDeleteResults> Delete(string searchdomain, string entityName)
     {
         Searchdomain searchdomain_;
         try
@@ -123,11 +124,13 @@ public class EntityController : ControllerBase
             searchdomain_ = _domainManager.GetSearchdomain(searchdomain);
         } catch (Exception)
         {
+            _logger.LogError("Unable to delete the entity {entityName} in {searchdomain} - the searchdomain likely does not exist", [entityName, searchdomain]);
             return Ok(new EntityDeleteResults() {Success = false});
         }
         Entity? entity_ = searchdomain_.GetEntity(entityName);
         if (entity_ is null)
         {
+            _logger.LogError("Unable to delete the entity {entityName} in {searchdomain} - it was not found under the specified name", [entityName, searchdomain]);
             return Ok(new EntityDeleteResults() {Success = false});
         }
         searchdomain_.RemoveEntity(entityName);

src/Server/Controllers/SearchdomainController.cs

@@ -21,8 +21,16 @@ public class SearchdomainController : ControllerBase
     [HttpGet("List")]
     public ActionResult<SearchdomainListResults> List()
     {
-        var results = _domainManager.ListSearchdomains()
+        List<string> results;
-            ?? throw new Exception("Unable to list searchdomains");
+        try
+        {
+            results = _domainManager.ListSearchdomains();
+        }
+        catch (Exception)
+        {
+            _logger.LogError("Unable to list searchdomains");
+            throw;
+        }
         SearchdomainListResults searchdomainListResults = new() {Searchdomains = results};
         return Ok(searchdomainListResults);
     }
@@ -36,6 +44,7 @@ public class SearchdomainController : ControllerBase
             return Ok(new SearchdomainCreateResults(){Id = id, Success = true});
         } catch (Exception)
         {
+            _logger.LogError("Unable to create Searchdomain {searchdomain}", [searchdomain]);
             return Ok(new SearchdomainCreateResults() { Id = null, Success = false });
         }
     }
@@ -49,9 +58,9 @@ public class SearchdomainController : ControllerBase
         {
             success = true;
             deletedEntries = _domainManager.DeleteSearchdomain(searchdomain);
-        } catch (Exception ex)
+        } catch (Exception)
         {
-            Console.WriteLine(ex);
+            _logger.LogError("Unable to delete searchdomain {searchdomain}", [searchdomain]);
             success = false;
             deletedEntries = 0;
         }
@@ -73,6 +82,7 @@ public class SearchdomainController : ControllerBase
             searchdomain_.helper.ExecuteSQLNonQuery("UPDATE searchdomain set name = @name, settings = @settings WHERE id = @id", parameters);
         } catch (Exception)
         {
+            _logger.LogError("Unable to update searchdomain {searchdomain}", [searchdomain]);
             return Ok(new SearchdomainUpdateResults() { Success = false });
         }
         return Ok(new SearchdomainUpdateResults(){Success = true});

src/Server/Program.cs

@@ -1,3 +1,6 @@
+using ElmahCore;
+using ElmahCore.Mvc;
+using Serilog;
 using Server;
 
 var builder = WebApplication.CreateBuilder(args);
@@ -8,16 +11,49 @@ builder.Services.AddControllers();
 // Learn more about configuring Swagger/OpenAPI at https://aka.ms/aspnetcore/swashbuckle
 builder.Services.AddEndpointsApiExplorer();
 builder.Services.AddSwaggerGen();
-
+Log.Logger = new LoggerConfiguration()
+    .ReadFrom.Configuration(builder.Configuration)
+    .CreateLogger();
+builder.Logging.AddSerilog();
 builder.Services.AddSingleton<SearchdomainManager>();
 
+builder.Services.AddElmah<XmlFileErrorLog>(Options =>
+{
+    Options.LogPath = builder.Configuration.GetValue<string>("Embeddingsearch:Elmah:LogFolder") ?? "~/logs";
+});
+
 var app = builder.Build();
 
+List<string>? allowedIps = builder.Configuration.GetSection("Embeddingsearch:Elmah:AllowedHosts")
+    .Get<List<string>>();
+
+app.Use(async (context, next) =>
+{
+    if (context.Request.Path.StartsWithSegments("/elmah"))
+    {
+        var remoteIp = context.Connection.RemoteIpAddress?.ToString();
+        bool blockRequest = allowedIps is null
+            || remoteIp is null
+            || !allowedIps.Contains(remoteIp);
+        if (blockRequest)
+        {
+            context.Response.StatusCode = 403;
+            await context.Response.WriteAsync("Forbidden");
+            return;
+        }
+    }
+
+    await next();
+});
+
+app.UseElmah();
+
 // Configure the HTTP request pipeline.
 if (app.Environment.IsDevelopment())
 {
     app.UseSwagger();
     app.UseSwaggerUI();
+    //app.UseElmahExceptionPage(); // Messes with JSON response for API calls. Leaving this here so I don't accidentally put this in again later on.
 }
 else
 {

src/Server/SearchdomainManager.cs

@@ -32,8 +32,16 @@ public class SearchdomainManager
         connection = new MySqlConnection(connectionString);
         connection.Open();
         helper = new SQLHelper(connection);
+        try
+        {
             DatabaseMigrations.Migrate(helper);
         }
+        catch (Exception ex)
+        {
+            _logger.LogCritical("Unable to migrate the database due to the exception: {ex}", [ex.Message]);
+            throw;
+        }
+    }
 
     public Searchdomain GetSearchdomain(string searchdomain)
     {
@@ -73,6 +81,7 @@ public class SearchdomainManager
     {
         if (searchdomains.TryGetValue(searchdomain, out Searchdomain? value))
         {
+            _logger.LogError("Searchdomain {searchdomain} could not be created, as it already exists", [searchdomain]);
             throw new Exception("Searchdomain already exists"); // TODO create proper SearchdomainAlreadyExists exception
         }
         Dictionary<string, dynamic> parameters = new()

src/Server/Server.csproj

@@ -7,6 +7,9 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <PackageReference Include="ElmahCore" Version="2.1.2" />
+    <PackageReference Include="Serilog.AspNetCore" Version="9.0.0" />
+    <PackageReference Include="Serilog.Sinks.File" Version="7.0.0" />
     <PackageReference Include="Swashbuckle.AspNetCore" Version="6.6.2" />
         <PackageReference Include="Microsoft.Data.SqlClient" Version="6.0.1" />
       <PackageReference Include="Microsoft.Data.Sqlite" Version="9.0.3" />

src/Server/appsettings.Development.json

@@ -17,6 +17,13 @@
     "ConnectionStrings": {
       "SQL": "server=localhost;database=embeddingsearch;uid=embeddingsearch;pwd=somepassword!;"
     },
+    "Elmah": {
+      "AllowedHosts": [
+        "127.0.0.1",
+        "::1",
+        "172.17.0.1"
+      ]
+    },
     "OllamaURL": "http://localhost:11434",
     "ApiKeys": ["Some UUID here", "Another UUID here"],
     "UseHttpsRedirection": true

src/Server/appsettings.json

@@ -5,5 +5,25 @@
       "Microsoft.AspNetCore": "Warning"
     }
   },
+  "Serilog": {
+    "Using":  [ "Serilog.Sinks.Console", "Serilog.Sinks.File" ],
+    "MinimumLevel": "Information",
+    "WriteTo": [
+      { "Name": "Console" },
+      { "Name": "File", "Args": { "path": "logs/log.txt", "rollingInterval": "Day", "retainedFileCountLimit": 7 } }
+    ],
+    "Properties": {
+        "Application": "Indexer"
+    }
+  },
+  "EmbeddingsearchIndexer": {
+    "Elmah": {
+      "AllowedHosts": [
+        "127.0.0.1",
+        "::1"
+      ],
+      "LogFolder": "./logs"
+    }
+  },
   "AllowedHosts": "*"
 }