lucretia/embeddingsearch
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added database size estimation

Browse Source
This commit is contained in:
LD-Reborn
2025-12-19 12:22:51 +01:00
parent 2fcf5bea0f
commit 7eaf69379f
4 changed files with 100 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
src/Server/Controllers/SearchdomainController.cs
View File

@@ -2,6 +2,7 @@ using System.Text.Json;
using ElmahCore;
using Microsoft.AspNetCore.Mvc;
using Server.Exceptions;
using Server.Helper;
using Shared.Models;
namespace Server.Controllers;
@@ -224,4 +225,26 @@ public class SearchdomainController : ControllerBase
}
return Ok(new SearchdomainInvalidateCacheResults(){Success = true});
}
[HttpGet("GetDatabaseSize")]
public ActionResult<SearchdomainGetDatabaseSizeResult> GetDatabaseSize(string searchdomain)
{
Searchdomain searchdomain_;
try
{
searchdomain_ = _domainManager.GetSearchdomain(searchdomain);
}
catch (SearchdomainNotFoundException)
{
_logger.LogError("Unable to retrieve the searchdomain {searchdomain} - it likely does not exist yet", [searchdomain]);
return Ok(new SearchdomainGetDatabaseSizeResult() { SearchdomainDatabaseSizeBytes = null, Success = false, Message = "Searchdomain not found" });
}
catch (Exception ex)
{
_logger.LogError("Unable to retrieve the searchdomain {searchdomain} - {ex.Message} - {ex.StackTrace}", [searchdomain, ex.Message, ex.StackTrace]);
return Ok(new SearchdomainGetDatabaseSizeResult() { SearchdomainDatabaseSizeBytes = null, Success = false, Message = ex.Message });
}
long sizeInBytes = DatabaseHelper.GetSearchdomainDatabaseSize(searchdomain_.helper, searchdomain);
return Ok(new SearchdomainGetDatabaseSizeResult() { SearchdomainDatabaseSizeBytes = sizeInBytes, Success = true });
}
}

34
src/Server/Helper/DatabaseHelper.cs
View File

@@ -177,4 +177,38 @@ public class DatabaseHelper(ILogger<DatabaseHelper> logger)
return result;
}
}
public static long GetSearchdomainDatabaseSize(SQLHelper helper, string searchdomain)
{
Dictionary<string, dynamic> parameters = new()
{
{ "searchdomain", searchdomain}
};
DbDataReader searchdomainSumReader = helper.ExecuteSQLCommand("SELECT SUM(LENGTH(id) + LENGTH(name) + LENGTH(settings)) AS total_bytes FROM embeddingsearch.searchdomain WHERE name=@searchdomain", parameters);
bool success = searchdomainSumReader.Read();
long result = success && !searchdomainSumReader.IsDBNull(0) ? searchdomainSumReader.GetInt64(0) : 0;
searchdomainSumReader.Close();
DbDataReader entitySumReader = helper.ExecuteSQLCommand("SELECT SUM(LENGTH(e.id) + LENGTH(e.name) + LENGTH(e.probmethod) + LENGTH(e.id_searchdomain)) AS total_bytes FROM embeddingsearch.entity e JOIN embeddingsearch.searchdomain s ON e.id_searchdomain = s.id WHERE s.name=@searchdomain", parameters);
success = entitySumReader.Read();
result += success && !entitySumReader.IsDBNull(0) ? entitySumReader.GetInt64(0) : 0;
entitySumReader.Close();
DbDataReader datapointSumReader = helper.ExecuteSQLCommand("SELECT SUM(LENGTH(d.id) + LENGTH(d.name) + LENGTH(d.probmethod_embedding) + LENGTH(d.similaritymethod) + LENGTH(d.id_entity) + LENGTH(d.hash)) AS total_bytes FROM embeddingsearch.datapoint d JOIN embeddingsearch.entity e ON d.id_entity = e.id JOIN embeddingsearch.searchdomain s ON e.id_searchdomain = s.id WHERE s.name=@searchdomain", parameters);
success = datapointSumReader.Read();
result += success && !datapointSumReader.IsDBNull(0) ? datapointSumReader.GetInt64(0) : 0;
datapointSumReader.Close();
DbDataReader embeddingSumReader = helper.ExecuteSQLCommand("SELECT SUM(LENGTH(em.id) + LENGTH(em.id_datapoint) + LENGTH(em.model) + LENGTH(em.embedding)) AS total_bytes FROM embeddingsearch.embedding em JOIN embeddingsearch.datapoint d ON em.id_datapoint = d.id JOIN embeddingsearch.entity e ON d.id_entity = e.id JOIN embeddingsearch.searchdomain s ON e.id_searchdomain = s.id WHERE s.name=@searchdomain", parameters);
success = embeddingSumReader.Read();
result += success && !embeddingSumReader.IsDBNull(0) ? embeddingSumReader.GetInt64(0) : 0;
embeddingSumReader.Close();
DbDataReader attributeSumReader = helper.ExecuteSQLCommand("SELECT SUM(LENGTH(a.id) + LENGTH(a.id_entity) + LENGTH(a.attribute) + LENGTH(a.value)) AS total_bytes FROM embeddingsearch.attribute a JOIN embeddingsearch.entity e ON a.id_entity = e.id JOIN embeddingsearch.searchdomain s ON e.id_searchdomain = s.id WHERE s.name=@searchdomain", parameters);
success = attributeSumReader.Read();
result += success && !attributeSumReader.IsDBNull(0) ? attributeSumReader.GetInt64(0) : 0;
attributeSumReader.Close();
return result;
}
}

33
src/Server/Views/Home/Index.cshtml
View File

@@ -64,11 +64,20 @@
</div>
</div>
<h3 class="visually-hidden">@T["Cache"]</h3>
<h3 class="visually-hidden">@T["Search cache"]</h3>
<!-- Cache -->
<div class="d-flex align-items-center mb-4">
<div class="me-3">
<strong>Cache utilization:</strong> <span id="cacheUtilization">0.00MiB</span>
<strong>@T["Search cache utilization"]:</strong> <span id="cacheUtilization">0.00MiB</span>
</div>
<button id="cacheClear" class="btn btn-warning btn-sm">@T["Clear"]</button>
</div>
<h3 class="visually-hidden">@T["Database size"]</h3>
<!-- Database size -->
<div class="d-flex align-items-center mb-4">
<div class="me-3">
<strong>@T["Database size"]:</strong> <span id="databaseUtilization">0.00MiB</span>
</div>
<button id="cacheClear" class="btn btn-warning btn-sm">@T["Clear"]</button>
</div>
@@ -506,6 +515,11 @@
.then(r => r.json());
}
function getSearchdomainDatabaseUtilization(domainKey) {
return fetch(`/Searchdomain/GetDatabaseSize?searchdomain=${encodeURIComponent(domains[domainKey])}`)
.then(r => r.json());
}
function selectDomain(domainKey) {
document.querySelectorAll('.domain-item').forEach(item => {
item.classList.remove('active');
@@ -521,6 +535,7 @@
let configElementCacheReconsiliation = document.getElementById('searchdomainConfigCacheReconciliation');
let cacheUtilizationPromise = getSearchdomainCacheUtilization(getSelectedDomainKey());
let databaseUtilizationPromise = getSearchdomainDatabaseUtilization(getSelectedDomainKey());
/* ---------- ENTITIES ---------- */
let entitiesUrl = `/Entity/List?searchdomain=${encodeURIComponent(domainName)}&returnEmbeddings=false`;
@@ -575,14 +590,24 @@
cacheUtilizationPromise.then(cacheUtilization => {
if (cacheUtilization != null && cacheUtilization.SearchCacheSizeBytes != null)
{
console.log(cacheUtilization);
document.querySelector('#cacheUtilization').innerText =
`${(cacheUtilization.SearchCacheSizeBytes / (1024 * 1024)).toFixed(2)}MiB`;
} else {
// TODO add toast
console.error('Failed to fetch searchdomain cache utilization');
}
});
});
databaseUtilizationPromise.then(databaseUtilization => {
if (databaseUtilization != null && databaseUtilization.SearchdomainDatabaseSizeBytes != null)
{
document.querySelector('#databaseUtilization').innerText =
`${(databaseUtilization.SearchdomainDatabaseSizeBytes / (1024 * 1024)).toFixed(2)}MiB`;
} else {
// TODO add toast
console.error('Failed to fetch searchdomain database utilization');
}
});
}
function clearEntitiesTable() {

15
src/Shared/Models/SearchdomainResults.cs
View File

@@ -86,4 +86,17 @@ public class SearchdomainInvalidateCacheResults
[JsonPropertyName("Message")]
public string? Message { get; set; }
}
}
public class SearchdomainGetDatabaseSizeResult
{
[JsonPropertyName("Success")]
public required bool Success { get; set; }
[JsonPropertyName("Message")]
public string? Message { get; set; }
[JsonPropertyName("SearchdomainDatabaseSizeBytes")]
public required long? SearchdomainDatabaseSizeBytes { get; set; }
}