Compare commits
11 Commits
3e433c3cbe
...
ad84efb611
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ad84efb611 | ||
| ecaa640ec0 | |||
|
|
37f1b285d8 | ||
| 71b273f5d7 | |||
|
|
1a823bb1e7 | ||
| aa4fc03c3d | |||
|
|
09832d1c0b | ||
| 68630fdbef | |||
|
|
c9907da846 | ||
| cddd305d26 | |||
| 6f4ffbcaa6 |
4
.gitignore
vendored
4
.gitignore
vendored
@@ -18,5 +18,5 @@ src/Server/logs
|
||||
src/Shared/bin
|
||||
src/Shared/obj
|
||||
src/Server/wwwroot/logs/*
|
||||
src/Server/CriticalCSS/node_modules
|
||||
src/Server/CriticalCSS/package*.json
|
||||
src/Server/Tools/CriticalCSS/node_modules
|
||||
src/Server/Tools/CriticalCSS/package*.json
|
||||
|
||||
@@ -247,17 +247,13 @@ public class SearchdomainController : ControllerBase
|
||||
[HttpGet("QueryCache/Size")]
|
||||
public ActionResult<SearchdomainSearchCacheSizeResults> GetSearchCacheSize([Required]string searchdomain)
|
||||
{
|
||||
if (!SearchdomainHelper.IsSearchdomainLoaded(_domainManager, searchdomain))
|
||||
{
|
||||
return Ok(new SearchdomainSearchCacheSizeResults() { QueryCacheSizeBytes = 0, Success = true });
|
||||
}
|
||||
(Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomain, _logger);
|
||||
if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new SearchdomainUpdateResults(){Success = false, Message = message});
|
||||
Dictionary<string, DateTimedSearchResult> searchCache = searchdomain_.searchCache;
|
||||
long sizeInBytes = 0;
|
||||
foreach (var entry in searchCache)
|
||||
{
|
||||
sizeInBytes += sizeof(int); // string length prefix
|
||||
sizeInBytes += entry.Key.Length * sizeof(char); // string characters
|
||||
sizeInBytes += entry.Value.EstimateSize();
|
||||
}
|
||||
return Ok(new SearchdomainSearchCacheSizeResults() { QueryCacheSizeBytes = sizeInBytes, Success = true });
|
||||
return Ok(new SearchdomainSearchCacheSizeResults() { QueryCacheSizeBytes = searchdomain_.GetSearchCacheSize(), Success = true });
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -5,8 +5,10 @@ using System.Text.Json;
|
||||
using AdaptiveExpressions;
|
||||
using ElmahCore;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.Extensions.Options;
|
||||
using Server.Exceptions;
|
||||
using Server.Helper;
|
||||
using Server.Models;
|
||||
using Shared.Models;
|
||||
|
||||
[ApiController]
|
||||
@@ -17,13 +19,15 @@ public class ServerController : ControllerBase
|
||||
private readonly IConfiguration _config;
|
||||
private AIProvider _aIProvider;
|
||||
private readonly SearchdomainManager _searchdomainManager;
|
||||
private readonly IOptions<EmbeddingSearchOptions> _options;
|
||||
|
||||
public ServerController(ILogger<ServerController> logger, IConfiguration config, AIProvider aIProvider, SearchdomainManager searchdomainManager)
|
||||
public ServerController(ILogger<ServerController> logger, IConfiguration config, AIProvider aIProvider, SearchdomainManager searchdomainManager, IOptions<EmbeddingSearchOptions> options)
|
||||
{
|
||||
_logger = logger;
|
||||
_config = config;
|
||||
_aIProvider = aIProvider;
|
||||
_searchdomainManager = searchdomainManager;
|
||||
_options = options;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -47,31 +51,51 @@ public class ServerController : ControllerBase
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the total memory size of the embedding cache
|
||||
/// Gets numeric info regarding the searchdomains
|
||||
/// </summary>
|
||||
[HttpGet("EmbeddingCache/Size")]
|
||||
public ActionResult<ServerGetEmbeddingCacheSizeResult> GetEmbeddingCacheSize()
|
||||
[HttpGet("Stats")]
|
||||
public async Task<ActionResult<ServerGetStatsResult>> Stats()
|
||||
{
|
||||
long size = 0;
|
||||
long elementCount = 0;
|
||||
long embeddingsCount = 0;
|
||||
LRUCache<string, Dictionary<string, float[]>> embeddingCache = _searchdomainManager.embeddingCache;
|
||||
var cacheListField = embeddingCache.GetType()
|
||||
.GetField("_cacheList", BindingFlags.Instance | BindingFlags.NonPublic) ?? throw new InvalidOperationException("_cacheList field not found"); // TODO Remove this unsafe reflection atrocity
|
||||
LinkedList<string> cacheListOriginal = (LinkedList<string>)cacheListField.GetValue(embeddingCache)!;
|
||||
LinkedList<string> cacheList = new(cacheListOriginal);
|
||||
|
||||
foreach (string key in cacheList)
|
||||
try
|
||||
{
|
||||
if (!embeddingCache.TryGet(key, out var entry))
|
||||
continue;
|
||||
long size = 0;
|
||||
long elementCount = 0;
|
||||
long embeddingsCount = 0;
|
||||
LRUCache<string, Dictionary<string, float[]>> embeddingCache = _searchdomainManager.embeddingCache;
|
||||
var cacheListField = embeddingCache.GetType()
|
||||
.GetField("_cacheList", BindingFlags.Instance | BindingFlags.NonPublic) ?? throw new InvalidOperationException("_cacheList field not found"); // TODO Remove this unsafe reflection atrocity
|
||||
LinkedList<string> cacheListOriginal = (LinkedList<string>)cacheListField.GetValue(embeddingCache)!;
|
||||
LinkedList<string> cacheList = new(cacheListOriginal);
|
||||
|
||||
// estimate size
|
||||
size += EstimateEntrySize(key, entry);
|
||||
elementCount++;
|
||||
embeddingsCount += entry.Keys.Count;
|
||||
foreach (string key in cacheList)
|
||||
{
|
||||
if (!embeddingCache.TryGet(key, out var entry))
|
||||
continue;
|
||||
|
||||
// estimate size
|
||||
size += EstimateEntrySize(key, entry);
|
||||
elementCount++;
|
||||
embeddingsCount += entry.Keys.Count;
|
||||
}
|
||||
var sqlHelper = DatabaseHelper.GetSQLHelper(_options.Value);
|
||||
Task<long> entityCountTask = DatabaseHelper.CountEntities(sqlHelper);
|
||||
long queryCacheUtilization = 0;
|
||||
foreach (string searchdomain in _searchdomainManager.ListSearchdomains())
|
||||
{
|
||||
if (SearchdomainHelper.IsSearchdomainLoaded(_searchdomainManager, searchdomain))
|
||||
{
|
||||
(Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_searchdomainManager, searchdomain, _logger);
|
||||
if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new ServerGetStatsResult(){Success = false, Message = message});
|
||||
queryCacheUtilization += searchdomain_.GetSearchCacheSize();
|
||||
}
|
||||
};
|
||||
long entityCount = await entityCountTask;
|
||||
return new ServerGetStatsResult() { Success = true, EntityCount = entityCount, QueryCacheUtilization = queryCacheUtilization, SizeInBytes = size, MaxElementCount = _searchdomainManager.EmbeddingCacheMaxCount, ElementCount = elementCount, EmbeddingsCount = embeddingsCount};
|
||||
} catch (Exception ex)
|
||||
{
|
||||
ElmahExtensions.RaiseError(ex);
|
||||
return StatusCode(500, new ServerGetStatsResult(){Success = false, Message = ex.Message});
|
||||
}
|
||||
return new ServerGetEmbeddingCacheSizeResult() { Success = true, SizeInBytes = size, MaxElementCount = _searchdomainManager.EmbeddingCacheMaxCount, ElementCount = elementCount, EmbeddingsCount = embeddingsCount};
|
||||
}
|
||||
|
||||
private static long EstimateEntrySize(string key, Dictionary<string, float[]> value)
|
||||
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -1,6 +1,9 @@
|
||||
using System.Configuration;
|
||||
using System.Data.Common;
|
||||
using System.Text;
|
||||
using MySql.Data.MySqlClient;
|
||||
using Server.Exceptions;
|
||||
using Server.Models;
|
||||
using Shared.Models;
|
||||
|
||||
namespace Server.Helper;
|
||||
@@ -9,6 +12,14 @@ public class DatabaseHelper(ILogger<DatabaseHelper> logger)
|
||||
{
|
||||
private readonly ILogger<DatabaseHelper> _logger = logger;
|
||||
|
||||
public static SQLHelper GetSQLHelper(EmbeddingSearchOptions embeddingSearchOptions)
|
||||
{
|
||||
string connectionString = embeddingSearchOptions.ConnectionStrings.SQL;
|
||||
MySqlConnection connection = new(connectionString);
|
||||
connection.Open();
|
||||
return new SQLHelper(connection, connectionString);
|
||||
}
|
||||
|
||||
public static void DatabaseInsertEmbeddingBulk(SQLHelper helper, int id_datapoint, List<(string model, byte[] embedding)> data)
|
||||
{
|
||||
Dictionary<string, object> parameters = [];
|
||||
@@ -211,4 +222,26 @@ public class DatabaseHelper(ILogger<DatabaseHelper> logger)
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
public static async Task<long> CountEntities(SQLHelper helper)
|
||||
{
|
||||
DbDataReader searchdomainSumReader = helper.ExecuteSQLCommand("SELECT COUNT(*) FROM entity;", []);
|
||||
bool success = searchdomainSumReader.Read();
|
||||
long result = success && !searchdomainSumReader.IsDBNull(0) ? searchdomainSumReader.GetInt64(0) : 0;
|
||||
searchdomainSumReader.Close();
|
||||
return result;
|
||||
}
|
||||
|
||||
public static long CountEntitiesForSearchdomain(SQLHelper helper, string searchdomain)
|
||||
{
|
||||
Dictionary<string, dynamic> parameters = new()
|
||||
{
|
||||
{ "searchdomain", searchdomain}
|
||||
};
|
||||
DbDataReader searchdomainSumReader = helper.ExecuteSQLCommand("SELECT COUNT(*) FROM entity e JOIN searchdomain s on e.id_searchdomain = s.id WHERE e.id_searchdomain = s.id AND s.name = @searchdomain;", parameters);
|
||||
bool success = searchdomainSumReader.Read();
|
||||
long result = success && !searchdomainSumReader.IsDBNull(0) ? searchdomainSumReader.GetInt64(0) : 0;
|
||||
searchdomainSumReader.Close();
|
||||
return result;
|
||||
}
|
||||
}
|
||||
@@ -299,4 +299,9 @@ public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelp
|
||||
return (null, 404, $"Unable to update searchdomain {searchdomain}");
|
||||
}
|
||||
}
|
||||
|
||||
public static bool IsSearchdomainLoaded(SearchdomainManager searchdomainManager, string name)
|
||||
{
|
||||
return searchdomainManager.IsSearchdomainLoaded(name);
|
||||
}
|
||||
}
|
||||
@@ -6,7 +6,7 @@ namespace Server.Models;
|
||||
|
||||
public class EmbeddingSearchOptions : ApiKeyOptions
|
||||
{
|
||||
public required ConnectionStringsSection ConnectionStrings { get; set; }
|
||||
public required ConnectionStringsOptions ConnectionStrings { get; set; }
|
||||
public ElmahOptions? Elmah { get; set; }
|
||||
public required long EmbeddingCacheMaxCount { get; set; }
|
||||
public required Dictionary<string, AiProvider> AiProviders { get; set; }
|
||||
@@ -34,3 +34,8 @@ public class SimpleUser
|
||||
public string Password { get; set; } = "";
|
||||
public string[] Roles { get; set; } = [];
|
||||
}
|
||||
|
||||
public class ConnectionStringsOptions
|
||||
{
|
||||
public required string SQL { get; set; }
|
||||
}
|
||||
@@ -55,7 +55,7 @@
|
||||
<value>Such-Cache</value>
|
||||
</data>
|
||||
<data name="Search cache utilization" xml:space="preserve">
|
||||
<value>Such-Cache Speicherauslastung</value>
|
||||
<value>Such-Cache-Speicherauslastung</value>
|
||||
</data>
|
||||
<data name="Clear" xml:space="preserve">
|
||||
<value>Leeren</value>
|
||||
@@ -121,7 +121,7 @@
|
||||
<value>Searchdomain Name</value>
|
||||
</data>
|
||||
<data name="Enable cache reconciliation" xml:space="preserve">
|
||||
<value>Cache Abgleich verwenden</value>
|
||||
<value>Cache-Abgleich verwenden</value>
|
||||
</data>
|
||||
<data name="Create entity" xml:space="preserve">
|
||||
<value>Entity erstellen</value>
|
||||
@@ -175,10 +175,10 @@
|
||||
<value>Searchdomain konnte nicht erstellt werden</value>
|
||||
</data>
|
||||
<data name="Searchdomain cache was cleared successfully" xml:space="preserve">
|
||||
<value>Searchdomain Cache wurde erfolgreich geleert</value>
|
||||
<value>Searchdomain-Cache wurde erfolgreich geleert</value>
|
||||
</data>
|
||||
<data name="Failed to clear searchdomain cache" xml:space="preserve">
|
||||
<value>Searchdomain Cache konnte nicht geleert werden</value>
|
||||
<value>Searchdomain-Cache konnte nicht geleert werden</value>
|
||||
</data>
|
||||
<data name="Entity was deleted successfully" xml:space="preserve">
|
||||
<value>Entity wurde erfolgreich gelöscht</value>
|
||||
@@ -229,7 +229,7 @@
|
||||
<value>Searchdomain Einstellungen konnten nicht abgerufen werden</value>
|
||||
</data>
|
||||
<data name="Unable to fetch searchdomain cache utilization" xml:space="preserve">
|
||||
<value>Searchdomain Cache-Auslastung konnte nicht abgerufen werden</value>
|
||||
<value>Searchdomain-Cache-Auslastung konnte nicht abgerufen werden</value>
|
||||
</data>
|
||||
<data name="Details" xml:space="preserve">
|
||||
<value>Details</value>
|
||||
@@ -243,4 +243,61 @@
|
||||
<data name="Close alert" xml:space="preserve">
|
||||
<value>Benachrichtigung schließen</value>
|
||||
</data>
|
||||
<data name="Recent queries" xml:space="preserve">
|
||||
<value>Letzte Queries</value>
|
||||
</data>
|
||||
<data name="Home" xml:space="preserve">
|
||||
<value>Dashboard</value>
|
||||
</data>
|
||||
<data name="Searchdomains" xml:space="preserve">
|
||||
<value>Searchdomains</value>
|
||||
</data>
|
||||
<data name="Swagger" xml:space="preserve">
|
||||
<value>Swagger</value>
|
||||
</data>
|
||||
<data name="Elmah" xml:space="preserve">
|
||||
<value>Elmah</value>
|
||||
</data>
|
||||
<data name="Hi!" xml:space="preserve">
|
||||
<value>Hallo!</value>
|
||||
</data>
|
||||
<data name="Hi, {0}!" xml:space="preserve">
|
||||
<value>Hallo {0}!</value>
|
||||
</data>
|
||||
<data name="Embedding Cache" xml:space="preserve">
|
||||
<value>Embedding-Cache</value>
|
||||
</data>
|
||||
<data name="Size" xml:space="preserve">
|
||||
<value>Größe</value>
|
||||
</data>
|
||||
<data name="Strings" xml:space="preserve">
|
||||
<value>Zeichenketten</value>
|
||||
</data>
|
||||
<data name="stringsCountInfo" xml:space="preserve">
|
||||
<value>Die Anzahl der Zeichenketten, für die Embeddings vorliegen. D.h. wenn zwei Modelle verwendet werden, ist die Zahl der Embeddings zweimal so hoch.</value>
|
||||
</data>
|
||||
<data name="Embeddings" xml:space="preserve">
|
||||
<value>Embeddings</value>
|
||||
</data>
|
||||
<data name="Health Checks" xml:space="preserve">
|
||||
<value>Health Checks</value>
|
||||
</data>
|
||||
<data name="Server" xml:space="preserve">
|
||||
<value>Server</value>
|
||||
</data>
|
||||
<data name="AI Providers" xml:space="preserve">
|
||||
<value>AI Providers</value>
|
||||
</data>
|
||||
<data name="Count" xml:space="preserve">
|
||||
<value>Anzahl</value>
|
||||
</data>
|
||||
<data name="Total Entities" xml:space="preserve">
|
||||
<value>Entities insgesamt</value>
|
||||
</data>
|
||||
<data name="Total query cache utilization" xml:space="preserve">
|
||||
<value>Query-Cache-Verwendung insgesamt</value>
|
||||
</data>
|
||||
<data name="Unable to fetch searchdomain database utilization" xml:space="preserve">
|
||||
<value>Searchdomain Datenbank-Auslastung konnte nicht abgerufen werden</value>
|
||||
</data>
|
||||
</root>
|
||||
@@ -243,4 +243,61 @@
|
||||
<data name="Close alert" xml:space="preserve">
|
||||
<value>Close alert</value>
|
||||
</data>
|
||||
<data name="Recent queries" xml:space="preserve">
|
||||
<value>Recent queries</value>
|
||||
</data>
|
||||
<data name="Home" xml:space="preserve">
|
||||
<value>Dashboard</value>
|
||||
</data>
|
||||
<data name="Searchdomains" xml:space="preserve">
|
||||
<value>Searchdomains</value>
|
||||
</data>
|
||||
<data name="Swagger" xml:space="preserve">
|
||||
<value>Swagger</value>
|
||||
</data>
|
||||
<data name="Elmah" xml:space="preserve">
|
||||
<value>Elmah</value>
|
||||
</data>
|
||||
<data name="Hi!" xml:space="preserve">
|
||||
<value>Hi!</value>
|
||||
</data>
|
||||
<data name="Hi, {0}!" xml:space="preserve">
|
||||
<value>Hi {0}!</value>
|
||||
</data>
|
||||
<data name="Embedding Cache" xml:space="preserve">
|
||||
<value>Embedding Cache</value>
|
||||
</data>
|
||||
<data name="Size" xml:space="preserve">
|
||||
<value>Size</value>
|
||||
</data>
|
||||
<data name="Strings" xml:space="preserve">
|
||||
<value>Strings</value>
|
||||
</data>
|
||||
<data name="stringsCountInfo" xml:space="preserve">
|
||||
<value>The number of strings for which there are embeddings. I.e. If you use two models, the amount of embeddings will be twice this number.</value>
|
||||
</data>
|
||||
<data name="Embeddings" xml:space="preserve">
|
||||
<value>Embeddings</value>
|
||||
</data>
|
||||
<data name="Health Checks" xml:space="preserve">
|
||||
<value>Health Checks</value>
|
||||
</data>
|
||||
<data name="Server" xml:space="preserve">
|
||||
<value>Server</value>
|
||||
</data>
|
||||
<data name="AI Providers" xml:space="preserve">
|
||||
<value>AI Providers</value>
|
||||
</data>
|
||||
<data name="Count" xml:space="preserve">
|
||||
<value>Count</value>
|
||||
</data>
|
||||
<data name="Total Entities" xml:space="preserve">
|
||||
<value>Total Entities</value>
|
||||
</data>
|
||||
<data name="Total query cache utilization" xml:space="preserve">
|
||||
<value>Total query cache utilization</value>
|
||||
</data>
|
||||
<data name="Unable to fetch searchdomain database utilization" xml:space="preserve">
|
||||
<value>Unable to fetch searchdomain database utilization</value>
|
||||
</data>
|
||||
</root>
|
||||
@@ -339,4 +339,16 @@ public class Searchdomain
|
||||
{
|
||||
searchCache = [];
|
||||
}
|
||||
|
||||
public long GetSearchCacheSize()
|
||||
{
|
||||
long sizeInBytes = 0;
|
||||
foreach (var entry in searchCache)
|
||||
{
|
||||
sizeInBytes += sizeof(int); // string length prefix
|
||||
sizeInBytes += entry.Key.Length * sizeof(char); // string characters
|
||||
sizeInBytes += entry.Value.EstimateSize();
|
||||
}
|
||||
return sizeInBytes;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,6 +6,8 @@ using Server.Exceptions;
|
||||
using AdaptiveExpressions;
|
||||
using Shared.Models;
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.Options;
|
||||
using Server.Models;
|
||||
|
||||
namespace Server;
|
||||
|
||||
@@ -13,24 +15,24 @@ public class SearchdomainManager
|
||||
{
|
||||
private Dictionary<string, Searchdomain> searchdomains = [];
|
||||
private readonly ILogger<SearchdomainManager> _logger;
|
||||
private readonly IConfiguration _config;
|
||||
private readonly EmbeddingSearchOptions _options;
|
||||
public readonly AIProvider aIProvider;
|
||||
private readonly DatabaseHelper _databaseHelper;
|
||||
private readonly string connectionString;
|
||||
private MySqlConnection connection;
|
||||
public SQLHelper helper;
|
||||
public LRUCache<string, Dictionary<string, float[]>> embeddingCache;
|
||||
public int EmbeddingCacheMaxCount;
|
||||
public long EmbeddingCacheMaxCount;
|
||||
|
||||
public SearchdomainManager(ILogger<SearchdomainManager> logger, IConfiguration config, AIProvider aIProvider, DatabaseHelper databaseHelper)
|
||||
public SearchdomainManager(ILogger<SearchdomainManager> logger, IOptions<EmbeddingSearchOptions> options, AIProvider aIProvider, DatabaseHelper databaseHelper)
|
||||
{
|
||||
_logger = logger;
|
||||
_config = config;
|
||||
_options = options.Value;
|
||||
this.aIProvider = aIProvider;
|
||||
_databaseHelper = databaseHelper;
|
||||
EmbeddingCacheMaxCount = config.GetValue<int?>("Embeddingsearch:EmbeddingCacheMaxCount") ?? 1000000;
|
||||
embeddingCache = new(EmbeddingCacheMaxCount);
|
||||
connectionString = _config.GetSection("Embeddingsearch").GetConnectionString("SQL") ?? "";
|
||||
EmbeddingCacheMaxCount = _options.EmbeddingCacheMaxCount;
|
||||
embeddingCache = new((int)EmbeddingCacheMaxCount);
|
||||
connectionString = _options.ConnectionStrings.SQL;
|
||||
connection = new MySqlConnection(connectionString);
|
||||
connection.Open();
|
||||
helper = new SQLHelper(connection, connectionString);
|
||||
@@ -122,4 +124,9 @@ public class SearchdomainManager
|
||||
searchdomains[name] = searchdomain;
|
||||
return searchdomain;
|
||||
}
|
||||
|
||||
public bool IsSearchdomainLoaded(string name)
|
||||
{
|
||||
return searchdomains.ContainsKey(name);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,7 +19,7 @@ const cookies = await page.cookies();
|
||||
await browser.close();
|
||||
|
||||
async function generateCriticalCSSForViews() {
|
||||
const viewsDir = '../Views';
|
||||
const viewsDir = '../../Views';
|
||||
|
||||
// Helper function to get all .cshtml files recursively
|
||||
function getAllCshtmlFiles(dir) {
|
||||
@@ -29,8 +29,6 @@ async function generateCriticalCSSForViews() {
|
||||
list.forEach(file => {
|
||||
const filePath = path.join(dir, file);
|
||||
const stat = fs.statSync(filePath);
|
||||
console.log("DEBUG@2");
|
||||
console.log(filePath);
|
||||
if (stat && stat.isDirectory()) {
|
||||
// Recursively get files from subdirectories
|
||||
results = results.concat(getAllCshtmlFiles(filePath));
|
||||
@@ -78,11 +76,11 @@ async function generateCriticalCSSForViews() {
|
||||
// Process each file
|
||||
for (const file of cshtmlFiles) {
|
||||
try {
|
||||
const urlPath = filePathToUrlPath(file).replace("../", "").replace("/Views", "");
|
||||
const urlPath = filePathToUrlPath(file).replace("../", "").replace("../", "").replace("/Views", "");
|
||||
|
||||
// Generate critical CSS
|
||||
await generate({
|
||||
src: `http://localhost:5146${urlPath}`,
|
||||
src: `http://localhost:5146${urlPath}?noCriticalCSS`,
|
||||
inline: false,
|
||||
width: 1920,
|
||||
height: 1080,
|
||||
@@ -92,7 +90,7 @@ async function generateCriticalCSSForViews() {
|
||||
},
|
||||
forceExclude: ['.btn'], // Otherwise buttons end up colorless and .btn overrides other classes like .btn-warning, etc. - so it has to be force-excluded here and re-added later
|
||||
forceInclude: [
|
||||
'[data-bs-theme=dark]',
|
||||
'[data-bs-theme="dark"]', '[data-bs-theme="dark"] body', '[data-bs-theme="dark"] .navbar', '[data-bs-theme="dark"] .card', '[data-bs-theme="dark"] .btn',
|
||||
'.navbar',
|
||||
'.col-md-4',
|
||||
'.visually-hidden', // visually hidden headings
|
||||
@@ -105,14 +103,14 @@ async function generateCriticalCSSForViews() {
|
||||
'.d-flex', '.justify-content-between', '.mt-2', // card - content
|
||||
'.progress', '.mt-3', // card - progress bar
|
||||
'.list-group', '.list-group-flush', '.list-group-item', '.list-group-flush>.list-group-item', '.list-group-flush>.list-group-item:last-child', '.badge', '.bg-warning', '.bg-success', '.h-100', // card - health check list
|
||||
'.btn', '.btn-sm', '.btn-primary', '.btn-warning', '.btn-danger', // Searchdomains buttons
|
||||
'.btn-primary', '.btn-warning', '.btn-danger', '.btn-info', // Searchdomains buttons
|
||||
'.col-md-8', '.sidebar',
|
||||
'.mb-0', '.mb-2', '.align-items-center',
|
||||
'h3', '.col-md-3', '.col-md-2', '.text-nowrap', '.overflow-auto'
|
||||
]
|
||||
},
|
||||
target: {
|
||||
css: path.join(criticalCssDir, urlPath.replace(/\//g, '.').replace(/^\./, '').replace("...", "") + '.css')
|
||||
css: path.join(criticalCssDir, "../../CriticalCSS/" + urlPath.replace(/\//g, '.').replace(/^\./, '').replace("...", "") + '.css')
|
||||
}
|
||||
});
|
||||
|
||||
@@ -8,3 +8,4 @@ npm install puppeteer
|
||||
```bash
|
||||
node CriticalCSSGenerator.js
|
||||
```
|
||||
3. Move the `.css` files from the current directory to the `CriticalCSS/` folder (overwrite existing files)
|
||||
78
src/Server/Tools/LocalizationChecker/LocalizationChecker.py
Normal file
78
src/Server/Tools/LocalizationChecker/LocalizationChecker.py
Normal file
@@ -0,0 +1,78 @@
|
||||
import re
|
||||
import sys
|
||||
|
||||
def extract_translations_from_View(view_path):
|
||||
"""Extract all translation strings from file A"""
|
||||
translations = {}
|
||||
|
||||
try:
|
||||
with open(view_path, 'r', encoding='utf-8') as file_a:
|
||||
for line_num, line in enumerate(file_a, 1):
|
||||
# Match T["..."] patterns
|
||||
matches = re.findall(r'T\["([^"]*)"\]', line)
|
||||
for match in matches:
|
||||
translations[match] = line_num
|
||||
except FileNotFoundError:
|
||||
print(f"Error: File {view_path} not found")
|
||||
sys.exit(1)
|
||||
except Exception as e:
|
||||
print(f"Error reading file {view_path}: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
return translations
|
||||
|
||||
def extract_localizations_from_resource_file(file_b_path):
|
||||
"""Extract all translation strings from file B"""
|
||||
translations = set()
|
||||
|
||||
try:
|
||||
with open(file_b_path, 'r', encoding='utf-8') as file_b:
|
||||
for line in file_b:
|
||||
# Match the pattern in file B
|
||||
match = re.search(r'<data name="([^"]*)"', line)
|
||||
if match:
|
||||
translations.add(match.group(1))
|
||||
except FileNotFoundError:
|
||||
print(f"Error: File {file_b_path} not found")
|
||||
sys.exit(1)
|
||||
except Exception as e:
|
||||
print(f"Error reading file {file_b_path}: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
return translations
|
||||
|
||||
def find_missing_translations(view, resource):
|
||||
"""Find translations in file A that don't exist in file B"""
|
||||
# Extract translations from both files
|
||||
file_a_translations = extract_translations_from_View(view)
|
||||
file_b_translations = extract_localizations_from_resource_file(resource)
|
||||
|
||||
# Find missing translations
|
||||
missing_translations = []
|
||||
|
||||
for translation_text, line_number in file_a_translations.items():
|
||||
if translation_text not in file_b_translations:
|
||||
missing_translations.append((translation_text, line_number))
|
||||
|
||||
return missing_translations
|
||||
|
||||
def main():
|
||||
views = ["Shared/_Layout.cshtml", "Home/Index.cshtml", "Home/Searchdomains.cshtml"]
|
||||
resources = ["SharedResources.en.resx", "SharedResources.de.resx"]
|
||||
|
||||
print("Checking for missing translations...")
|
||||
print("=" * 50)
|
||||
for view in views:
|
||||
for resource in resources:
|
||||
missing = find_missing_translations("../../Views/" + view, "../../Resources/" + resource)
|
||||
|
||||
if missing:
|
||||
print(f"Found {len(missing)} missing translations in {view}:")
|
||||
print("-" * 50)
|
||||
for translation_text, line_number in missing:
|
||||
print(f"Line {line_number}: T[\"{translation_text}\"]")
|
||||
else:
|
||||
print(f"All localizations in {view} have a matching resource in {resource}!")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -5,7 +5,6 @@
|
||||
@using Server
|
||||
|
||||
@inject LocalizationService T
|
||||
@inject AIProvider AIProvider
|
||||
@model HomeIndexViewModel
|
||||
@{
|
||||
ViewData["Title"] = "Home Page";
|
||||
@@ -41,7 +40,7 @@
|
||||
@T["Strings"]
|
||||
<i class="bi bi-info-circle-fill text-info"
|
||||
data-bs-toggle="tooltip"
|
||||
title="The number of strings for which there are embeddings. I.e. If you use two models, the amount of embeddings will be twice this number."></i>
|
||||
title="@T["stringsCountInfo"]"></i>
|
||||
</span>
|
||||
<strong id="embeddingcacheElementCount"></strong>
|
||||
</div>
|
||||
@@ -144,39 +143,14 @@
|
||||
searchdomains = result.Searchdomains;
|
||||
hideThrobber(searchdomainCount);
|
||||
searchdomainCount.textContent = searchdomains.length;
|
||||
|
||||
const perDomainPromises = searchdomains.map(async domain => {
|
||||
const [entityListResult, querycacheUtilizationResult] = await Promise.all([
|
||||
listEntities(domain),
|
||||
getQuerycacheUtilization(domain)
|
||||
]);
|
||||
|
||||
return {
|
||||
entityCount: entityListResult.Results.length,
|
||||
utilization: querycacheUtilizationResult.QueryCacheSizeBytes
|
||||
};
|
||||
});
|
||||
|
||||
const results = await Promise.all(perDomainPromises);
|
||||
|
||||
let entityCount = 0;
|
||||
let totalUtilization = 0;
|
||||
|
||||
for (const r of results) {
|
||||
entityCount += r.entityCount;
|
||||
totalUtilization += r.utilization;
|
||||
}
|
||||
|
||||
hideThrobber(searchdomainEntityCount);
|
||||
hideThrobber(totalQuerycacheUtilization);
|
||||
searchdomainEntityCount.textContent = entityCount;
|
||||
totalQuerycacheUtilization.textContent = NumberOfBytesAsHumanReadable(totalUtilization);
|
||||
});
|
||||
getEmbeddingcacheUtilization().then(result => {
|
||||
getServerStats().then(result => {
|
||||
let utilization = result.SizeInBytes;
|
||||
let maxElementCount = result.MaxElementCount;
|
||||
let elementCount = result.ElementCount;
|
||||
let embeddingCount = result.EmbeddingsCount;
|
||||
let entityCount = result.EntityCount;
|
||||
let queryCacheUtilization = result.QueryCacheUtilization;
|
||||
hideThrobber(embeddingcacheSize);
|
||||
embeddingcacheSize.textContent = NumberOfBytesAsHumanReadable(utilization);
|
||||
hideThrobber(embeddingcacheElementCount);
|
||||
@@ -184,6 +158,10 @@
|
||||
hideThrobber(embeddingcacheEmbeddingCount);
|
||||
embeddingcacheEmbeddingCount.textContent = embeddingCount;
|
||||
embeddingcacheElementCountProgressBar.style.width = `${elementCount / maxElementCount * 100}%`;
|
||||
hideThrobber(searchdomainEntityCount);
|
||||
searchdomainEntityCount.textContent = entityCount;
|
||||
hideThrobber(totalQuerycacheUtilization);
|
||||
totalQuerycacheUtilization.textContent = NumberOfBytesAsHumanReadable(queryCacheUtilization);
|
||||
});
|
||||
getHealthCheckStatusAndApply(healthchecksServer, "/healthz/Database");
|
||||
getHealthCheckStatusAndApply(healthchecksAiProvider, "/healthz/AIProvider");
|
||||
@@ -206,8 +184,8 @@
|
||||
.then(r => r.json());
|
||||
}
|
||||
|
||||
async function getEmbeddingcacheUtilization() {
|
||||
return await fetch(`/Server/EmbeddingCache/Size`)
|
||||
async function getServerStats() {
|
||||
return await fetch(`/Server/Stats`)
|
||||
.then(r => r.json());
|
||||
}
|
||||
|
||||
|
||||
@@ -92,7 +92,7 @@
|
||||
<div class="card section-card mb-4">
|
||||
<div class="card-body">
|
||||
<div class="d-flex justify-content-between align-items-center mb-2">
|
||||
<h3>Recent queries</h3>
|
||||
<h3>@T["Recent queries"]</h3>
|
||||
<input
|
||||
type="text"
|
||||
class="form-control form-control-sm w-25"
|
||||
@@ -103,8 +103,8 @@
|
||||
<table id="queriesTable" class="table table-striped" style="max-height: 60vh; overflow-y: auto; display: block;">
|
||||
<thead>
|
||||
<tr>
|
||||
<th class="visually-hidden">Name</th>
|
||||
<th class="visually-hidden">Action</th>
|
||||
<th class="visually-hidden">@T["Name"]</th>
|
||||
<th class="visually-hidden">@T["Action"]</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
@@ -129,8 +129,8 @@
|
||||
<table id="entitiesTable" class="table table-striped" style="max-height: 60vh; overflow-y: auto; display: block;">
|
||||
<thead>
|
||||
<tr>
|
||||
<th class="visually-hidden">Name</th>
|
||||
<th class="visually-hidden">Action</th>
|
||||
<th class="visually-hidden">@T["Name"]</th>
|
||||
<th class="visually-hidden">@T["Action"]</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
@@ -152,8 +152,8 @@
|
||||
<div class="modal-dialog modal-lg modal-dialog-scrollable">
|
||||
<div class="modal-content">
|
||||
<div class="modal-header bg-info">
|
||||
<h2 class="modal-title" id="entityDetailsTitle">@T["Entity Details"]</h2>
|
||||
<button type="button" class="btn-close" data-bs-dismiss="modal"></button>
|
||||
<h2 class="modal-title text-dark" id="entityDetailsTitle">@T["Entity Details"]</h2>
|
||||
<button type="button" class="btn-close" data-bs-dismiss="modal" style="filter: brightness(0);"></button>
|
||||
</div>
|
||||
|
||||
<div class="modal-body">
|
||||
@@ -200,8 +200,8 @@
|
||||
<div class="modal-content">
|
||||
|
||||
<div class="modal-header bg-info">
|
||||
<h2 class="modal-title" id="queryDetailsTitle">@T["Query Details"] - <span id="queryDetailsQueryName"></span></h2>
|
||||
<button type="button" class="btn-close" data-bs-dismiss="modal"></button>
|
||||
<h2 class="modal-title text-dark" id="queryDetailsTitle">@T["Query Details"] - <span id="queryDetailsQueryName"></span></h2>
|
||||
<button type="button" class="btn-close" data-bs-dismiss="modal" style="filter: brightness(0);"></button>
|
||||
</div>
|
||||
|
||||
<div class="modal-body">
|
||||
@@ -240,8 +240,8 @@
|
||||
<div class="modal-content">
|
||||
|
||||
<div class="modal-header bg-warning">
|
||||
<h2 class="modal-title" id="queryUpdateTitle">@T["Query Update"] - <span id="queryUpdateQueryName"></span></h2>
|
||||
<button type="button" class="btn-close" data-bs-dismiss="modal"></button>
|
||||
<h2 class="modal-title text-dark" id="queryUpdateTitle">@T["Query Update"] - <span id="queryUpdateQueryName"></span></h2>
|
||||
<button type="button" class="btn-close" data-bs-dismiss="modal" style="filter: brightness(0);"></button>
|
||||
</div>
|
||||
|
||||
<div class="modal-body">
|
||||
@@ -284,8 +284,8 @@
|
||||
<div class="modal-content">
|
||||
|
||||
<div class="modal-header bg-warning">
|
||||
<h2 class="modal-title" id="renameSearchdomainTitle">@T["Rename searchdomain"]</h2>
|
||||
<button type="button" class="btn-close" data-bs-dismiss="modal"></button>
|
||||
<h2 class="modal-title text-dark" id="renameSearchdomainTitle">@T["Rename searchdomain"]</h2>
|
||||
<button type="button" class="btn-close" data-bs-dismiss="modal" style="filter: brightness(0);"></button>
|
||||
</div>
|
||||
|
||||
<div class="modal-body">
|
||||
@@ -298,10 +298,10 @@
|
||||
|
||||
<div class="modal-footer">
|
||||
<button type="button" class="btn btn-warning" onclick="renameSearchdomain(getSelectedDomainKey(), document.getElementById('renameSearchdomainNewName').value)" data-bs-dismiss="modal">
|
||||
Rename
|
||||
@T["Rename"]
|
||||
</button>
|
||||
<button type="button" class="btn btn-secondary" data-bs-dismiss="modal">
|
||||
Close
|
||||
@T["Close"]
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
@@ -476,8 +476,8 @@
|
||||
<div class="modal-content">
|
||||
|
||||
<div class="modal-header bg-warning text">
|
||||
<h2 class="modal-title" id="updateEntityTitle">@T["Update entity"]</h2>
|
||||
<button type="button" class="btn-close" data-bs-dismiss="modal"></button>
|
||||
<h2 class="modal-title text-dark" id="updateEntityTitle">@T["Update entity"]</h2>
|
||||
<button type="button" class="btn-close" data-bs-dismiss="modal" style="filter: brightness(0);"></button>
|
||||
</div>
|
||||
|
||||
<div class="modal-body">
|
||||
|
||||
@@ -12,13 +12,18 @@
|
||||
<meta name="description" content="Embeddingsearch server" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>@ViewData["Title"] - embeddingsearch</title>
|
||||
@if (!Context.Request.Query.ContainsKey("renderRaw"))
|
||||
@if (!Context.Request.Query.ContainsKey("renderRaw") && !Context.Request.Query.ContainsKey("noCriticalCSS"))
|
||||
{
|
||||
<link rel="preload" href="~/lib/bootstrap/dist/css/bootstrap.min.css" as="style"/>
|
||||
<link rel="stylesheet" fetchpriority="high"
|
||||
href="~/lib/bootstrap/dist/css/bootstrap.min.css"
|
||||
media="print"
|
||||
onload="this.media='all'">
|
||||
} else if (Context.Request.Query.ContainsKey("noCriticalCSS"))
|
||||
{
|
||||
<link rel="preload" href="~/lib/bootstrap/dist/css/bootstrap.min.css" as="style"/>
|
||||
<link rel="stylesheet" fetchpriority="high"
|
||||
href="~/lib/bootstrap/dist/css/bootstrap.min.css">
|
||||
}
|
||||
<style>
|
||||
@Html.Raw(File.ReadAllText(System.IO.Path.Combine(Directory.GetCurrentDirectory(), "wwwroot", "css", "site.css")))
|
||||
@@ -29,7 +34,6 @@
|
||||
@if (Context.Request.Path.Value is not null)
|
||||
{
|
||||
string path = System.IO.Path.Combine("CriticalCSS", Context.Request.Path.Value.Trim('/').Replace("/", ".") + ".css");
|
||||
Console.WriteLine(path);
|
||||
if (File.Exists(path))
|
||||
{
|
||||
@Html.Raw(File.ReadAllText(path));
|
||||
@@ -43,9 +47,9 @@
|
||||
};
|
||||
</script>
|
||||
</head>
|
||||
<body>
|
||||
<body data-bs-theme="dark">
|
||||
<header>
|
||||
<nav class="navbar navbar-expand-sm navbar-toggleable-sm navbar-light bg-white border-bottom box-shadow mb-3">
|
||||
<nav class="navbar navbar-expand-sm navbar-toggleable-sm navbar-light border-bottom box-shadow mb-3">
|
||||
<div class="container-fluid">
|
||||
<a class="navbar-brand" asp-area="" asp-controller="Home" asp-action="Index">embeddingsearch</a>
|
||||
<button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target=".navbar-collapse" aria-controls="navbarSupportedContent"
|
||||
@@ -57,31 +61,31 @@
|
||||
@if (User.Identity?.IsAuthenticated == true)
|
||||
{
|
||||
<li class="nav-item">
|
||||
<a class="nav-link text-dark" asp-area="" asp-controller="Home" asp-action="Index">@T["Home"]</a>
|
||||
<a class="nav-link text" asp-area="" asp-controller="Home" asp-action="Index">@T["Home"]</a>
|
||||
</li>
|
||||
<li class="nav-item">
|
||||
<a class="nav-link text-dark" asp-area="" asp-controller="Home" asp-action="Searchdomains">@T["Searchdomains"]</a>
|
||||
<a class="nav-link text" asp-area="" asp-controller="Home" asp-action="Searchdomains">@T["Searchdomains"]</a>
|
||||
</li>
|
||||
@if (User.IsInRole("Admin") || User.IsInRole("Swagger"))
|
||||
{
|
||||
<li class="nav-item">
|
||||
<a class="nav-link text-dark" href="/swagger/index.html?ReturnUrl=@(currentUrl)">@T["Swagger"]</a>
|
||||
<a class="nav-link text" href="/swagger/index.html?ReturnUrl=@(currentUrl)">@T["Swagger"]</a>
|
||||
</li>
|
||||
}
|
||||
@if (User.IsInRole("Admin"))
|
||||
{
|
||||
<li class="nav-item">
|
||||
<a class="nav-link text-dark" href="/elmah?ReturnUrl=@(currentUrl)">@T["Elmah"]</a>
|
||||
<a class="nav-link text" href="/elmah?ReturnUrl=@(currentUrl)">@T["Elmah"]</a>
|
||||
</li>
|
||||
}
|
||||
<li class="nav-item">
|
||||
<a class="nav-link text-dark" asp-area="" asp-controller="Account" asp-action="Logout">@T["Logout"]</a>
|
||||
<a class="nav-link text" asp-area="" asp-controller="Account" asp-action="Logout">@T["Logout"]</a>
|
||||
</li>
|
||||
}
|
||||
else
|
||||
{
|
||||
<li class="nav-item">
|
||||
<a class="nav-link text-dark" asp-area="" asp-controller="Account" asp-action="Login">@T["Login"]</a>
|
||||
<a class="nav-link text" asp-area="" asp-controller="Account" asp-action="Login">@T["Login"]</a>
|
||||
</li>
|
||||
}
|
||||
</ul>
|
||||
@@ -106,3 +110,16 @@
|
||||
@await RenderSectionAsync("Scripts", required: false)
|
||||
</body>
|
||||
</html>
|
||||
<script>
|
||||
const mediaQuery = window.matchMedia('(prefers-color-scheme: dark)');
|
||||
|
||||
function applyTheme(e) {
|
||||
document.body.setAttribute(
|
||||
'data-bs-theme',
|
||||
e.matches ? 'dark' : 'light'
|
||||
);
|
||||
}
|
||||
|
||||
applyTheme(mediaQuery);
|
||||
mediaQuery.addEventListener('change', applyTheme);
|
||||
</script>
|
||||
|
||||
@@ -75,3 +75,8 @@ url("/fonts/bootstrap-icons.woff") format("woff");
|
||||
}
|
||||
|
||||
.bi-info-circle-fill::before { content: "\f430"; }
|
||||
|
||||
td.btn-group {
|
||||
display: revert;
|
||||
min-width: 15rem;
|
||||
}
|
||||
@@ -41,6 +41,7 @@
|
||||
}
|
||||
|
||||
/* expand on hover */
|
||||
.elmah-return-btn.show-label::before,
|
||||
.elmah-return-btn:hover::before {
|
||||
max-width: 220px;
|
||||
padding: 0.5rem;
|
||||
@@ -48,6 +49,7 @@
|
||||
}
|
||||
|
||||
/* hover colors */
|
||||
.elmah-return-btn.show-label,
|
||||
.elmah-return-btn:hover {
|
||||
background: #0b5ed7;
|
||||
color: white;
|
||||
|
||||
@@ -7,4 +7,11 @@ document.addEventListener('DOMContentLoaded', async () => {
|
||||
btn.className = "elmah-return-btn";
|
||||
|
||||
document.body.appendChild(btn);
|
||||
|
||||
const showLabelBriefly = () => {
|
||||
btn.classList.add("show-label");
|
||||
setTimeout(() => btn.classList.remove("show-label"), 2000);
|
||||
};
|
||||
|
||||
setTimeout(showLabelBriefly, 1000);
|
||||
});
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -8,14 +8,18 @@ public class ServerGetModelsResult : SuccesMessageBaseModel
|
||||
public string[]? Models { get; set; }
|
||||
}
|
||||
|
||||
public class ServerGetEmbeddingCacheSizeResult : SuccesMessageBaseModel
|
||||
public class ServerGetStatsResult : SuccesMessageBaseModel
|
||||
{
|
||||
[JsonPropertyName("SizeInBytes")]
|
||||
public required long? SizeInBytes { get; set; }
|
||||
public long? SizeInBytes { get; set; }
|
||||
[JsonPropertyName("MaxElementCount")]
|
||||
public required long? MaxElementCount { get; set; }
|
||||
public long? MaxElementCount { get; set; }
|
||||
[JsonPropertyName("ElementCount")]
|
||||
public required long? ElementCount { get; set; }
|
||||
public long? ElementCount { get; set; }
|
||||
[JsonPropertyName("EmbeddingsCount")]
|
||||
public required long? EmbeddingsCount { get; set; }
|
||||
public long? EmbeddingsCount { get; set; }
|
||||
[JsonPropertyName("EntityCount")]
|
||||
public long? EntityCount { get; set; }
|
||||
[JsonPropertyName("QueryCacheUtilization")]
|
||||
public long? QueryCacheUtilization { get; set; }
|
||||
}
|
||||
Reference in New Issue
Block a user