lucretia/embeddingsearch
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added weighted dictionary based weighted average

Browse Source
This commit is contained in:
LD-Reborn
2025-06-22 02:44:20 +02:00
parent 0e4dcdf7c0
commit 3813c5aaba
2 changed files with 30 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
src/Indexer/Scripts/example.py
View File

@@ -5,11 +5,15 @@ from dataclasses import asdict
import time import time
example_content = "./Scripts/example_content" example_content = "./Scripts/example_content"
example_searchdomain = "example" probmethod = "DictionaryWeightedAverage"
example_searchdomain = "example_" + probmethod
example_counter = 0 example_counter = 0
models = ["bge-m3", "mxbai-embed-large"] models = ["bge-m3", "mxbai-embed-large"]
probmethod_datapoint = "HighValueEmphasisWeightedAverage" probmethod_datapoint = probmethod
probmethod_entity = "HighValueEmphasisWeightedAverage" probmethod_entity = probmethod
# Example for a dictionary based weighted average:
# probmethod_datapoint = "DictionaryWeightedAverage:{\"bge-m3\": 4, \"mxbai-embed-large\": 1}"
# probmethod_entity = "DictionaryWeightedAverage:{\"title\": 2, \"filename\": 0.1, \"text\": 0.25}"
def init(toolset: Toolset): def init(toolset: Toolset):
global example_counter global example_counter

36
src/Server/Probmethods.cs
View File

@@ -5,12 +5,13 @@ namespace Server;
public static class Probmethods public static class Probmethods
{ {
public delegate float probMethodProtoDelegate(List<(string, float)> list, string parameters);
public delegate float probMethodDelegate(List<(string, float)> list); public delegate float probMethodDelegate(List<(string, float)> list);
public static readonly Dictionary<string, probMethodDelegate> probMethods; public static readonly Dictionary<string, probMethodProtoDelegate> probMethods;
static Probmethods() static Probmethods()
{ {
probMethods = new Dictionary<string, probMethodDelegate> probMethods = new Dictionary<string, probMethodProtoDelegate>
{ {
["Mean"] = Mean, ["Mean"] = Mean,
["HarmonicMean"] = HarmonicMean, ["HarmonicMean"] = HarmonicMean,
@@ -21,23 +22,32 @@ public static class Probmethods
["HighValueEmphasisWeightedAverage"] = HighValueEmphasisWeightedAverage, ["HighValueEmphasisWeightedAverage"] = HighValueEmphasisWeightedAverage,
["HVEWAvg"] = HighValueEmphasisWeightedAverage, ["HVEWAvg"] = HighValueEmphasisWeightedAverage,
["LowValueEmphasisWeightedAverage"] = LowValueEmphasisWeightedAverage, ["LowValueEmphasisWeightedAverage"] = LowValueEmphasisWeightedAverage,
["LVEWAvg"] = LowValueEmphasisWeightedAverage ["LVEWAvg"] = LowValueEmphasisWeightedAverage,
["DictionaryWeightedAverage"] = DictionaryWeightedAverage
}; };
} }
public static probMethodDelegate? GetMethod(string name) public static probMethodDelegate? GetMethod(string name)
{ {
try string methodName = name;
string? jsonArg = "";
// Detect if parameters are embedded
int colonIndex = name.IndexOf(':');
if (colonIndex != -1)
{ {
return probMethods[name]; methodName = name[..colonIndex];
jsonArg = name[(colonIndex + 1)..];
} }
catch
if (!probMethods.TryGetValue(methodName, out probMethodProtoDelegate? method))
{ {
return null; return null;
} }
return list => method(list, jsonArg);
} }
public static float Mean(List<(string, float)> list) public static float Mean(List<(string, float)> list, string __)
{ {
if (list.Count == 0) return 0; if (list.Count == 0) return 0;
float sum = 0; float sum = 0;
@@ -48,7 +58,7 @@ public static class Probmethods
return sum / list.Count; return sum / list.Count;
} }
public static float HarmonicMean(List<(string, float)> list) public static float HarmonicMean(List<(string, float)> list, string _)
{ {
int n_T = list.Count; int n_T = list.Count;
float[] nonzeros = [.. list.Select(t => t.Item2).Where(t => t != 0)]; float[] nonzeros = [.. list.Select(t => t.Item2).Where(t => t != 0)];
@@ -59,7 +69,7 @@ public static class Probmethods
return n_nz / nzSum * (n_nz / (float)n_T); return n_nz / nzSum * (n_nz / (float)n_T);
} }
public static float QuadraticMean(List<(string, float)> list) public static float QuadraticMean(List<(string, float)> list, string _)
{ {
float sum = 0; float sum = 0;
foreach (var (_, value) in list) foreach (var (_, value) in list)
@@ -69,7 +79,7 @@ public static class Probmethods
return (float)Math.Sqrt(sum / list.Count); return (float)Math.Sqrt(sum / list.Count);
} }
public static float GeometricMean(List<(string, float)> list) public static float GeometricMean(List<(string, float)> list, string __)
{ {
if (list.Count == 0) return 0; if (list.Count == 0) return 0;
float product = 1; float product = 1;
@@ -80,7 +90,7 @@ public static class Probmethods
return (float)Math.Pow(product, 1f / list.Count); return (float)Math.Pow(product, 1f / list.Count);
} }
public static float ExtremeValuesEmphasisWeightedAverage(List<(string, float)> list) public static float ExtremeValuesEmphasisWeightedAverage(List<(string, float)> list, string _)
{ {
float[] arr = [.. list.Select(x => x.Item2)]; float[] arr = [.. list.Select(x => x.Item2)];
if (arr.Contains(1)) return 1; if (arr.Contains(1)) return 1;
@@ -95,7 +105,7 @@ public static class Probmethods
return f / fm; return f / fm;
} }
public static float HighValueEmphasisWeightedAverage(List<(string, float)> list) public static float HighValueEmphasisWeightedAverage(List<(string, float)> list, string _)
{ {
float[] arr = [.. list.Select(x => x.Item2)]; float[] arr = [.. list.Select(x => x.Item2)];
if (arr.Contains(1)) return 1; if (arr.Contains(1)) return 1;
@@ -109,7 +119,7 @@ public static class Probmethods
return f / fm; return f / fm;
} }
public static float LowValueEmphasisWeightedAverage(List<(string, float)> list) public static float LowValueEmphasisWeightedAverage(List<(string, float)> list, string _)
{ {
float[] arr = [.. list.Select(x => x.Item2)]; float[] arr = [.. list.Select(x => x.Item2)];
if (arr.Contains(0)) return 0; if (arr.Contains(0)) return 0;