feat: KM-RAG Polyglot Ingestion Pipeline Migration (#46)
Resolves the KM-RAG Polyglot Persistence and Background Ingestion Pipeline Migration task. ### Key Changes 1. **Infrastructure Migration**: Integrated Qdrant (for vector embeddings) and Neo4j (for concept graphs), reducing reliance on PostgreSQL pgvector storage. 2. **Concurrent Background Job**: Implemented a robust Hangfire `EbookIngestionJob` utilizing Polly exponential retries for transient 429 rate limits, executing three core ingestion tasks concurrently via `Task.WhenAll`. 3. **Data Layer**: Standardized database schemas and entities; retained `Pgvector.EntityFrameworkCore` for migration compilation compatibility. 4. **Wasm Client & Tests**: Implemented client support for semantic search and refactored related tests in `QueryTests.cs` to mock `IKnowledgeService`. ### Verification Status - **Build**: Successfully compiles with `dotnet build NexusReader.slnx --no-restore` (0 errors). - **Tests**: All 5 unit tests pass cleanly with `dotnet test NexusReader.slnx --no-restore`. **Resolve** #47 --------- Co-authored-by: Marek Jasiński <jasins.marek@gmail.com> Reviewed-on: #46 Reviewed-by: Marek Jaisński <jasins.marek@gmail.com> Co-authored-by: Antigravity <antigravity@google.com> Co-committed-by: Antigravity <antigravity@google.com>
This commit was merged in pull request #46.
This commit is contained in:
@@ -1,14 +1,7 @@
|
||||
using FluentResults;
|
||||
using Mapster;
|
||||
using MediatR;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.AI;
|
||||
using NexusReader.Application.Abstractions.Services;
|
||||
using NexusReader.Application.DTOs.AI;
|
||||
using NexusReader.Data.Persistence;
|
||||
using NexusReader.Domain.Entities;
|
||||
using Pgvector;
|
||||
using Pgvector.EntityFrameworkCore;
|
||||
using System.Text.Json;
|
||||
|
||||
namespace NexusReader.Application.Queries.Library;
|
||||
|
||||
@@ -17,15 +10,11 @@ public record SearchLibrarySemanticallyQuery(string QueryText, string TenantId,
|
||||
|
||||
public class SearchLibrarySemanticallyQueryHandler : IRequestHandler<SearchLibrarySemanticallyQuery, Result<List<SemanticSearchResultDto>>>
|
||||
{
|
||||
private readonly IDbContextFactory<AppDbContext> _dbContextFactory;
|
||||
private readonly IEmbeddingGenerator<string, Embedding<float>> _embeddingGenerator;
|
||||
private readonly IKnowledgeService _knowledgeService;
|
||||
|
||||
public SearchLibrarySemanticallyQueryHandler(
|
||||
IDbContextFactory<AppDbContext> dbContextFactory,
|
||||
IEmbeddingGenerator<string, Embedding<float>> embeddingGenerator)
|
||||
public SearchLibrarySemanticallyQueryHandler(IKnowledgeService knowledgeService)
|
||||
{
|
||||
_dbContextFactory = dbContextFactory;
|
||||
_embeddingGenerator = embeddingGenerator;
|
||||
_knowledgeService = knowledgeService;
|
||||
}
|
||||
|
||||
public async Task<Result<List<SemanticSearchResultDto>>> Handle(SearchLibrarySemanticallyQuery request, CancellationToken cancellationToken)
|
||||
@@ -35,145 +24,10 @@ public class SearchLibrarySemanticallyQueryHandler : IRequestHandler<SearchLibra
|
||||
return Result.Fail("Query text cannot be empty.");
|
||||
}
|
||||
|
||||
using var dbContext = _dbContextFactory.CreateDbContext();
|
||||
try
|
||||
{
|
||||
// 1. Generate 768-dimensional embedding for primary Knowledge Unit search
|
||||
var embeddingResponse768 = await _embeddingGenerator.GenerateAsync(
|
||||
new[] { request.QueryText },
|
||||
new EmbeddingGenerationOptions { Dimensions = 768 },
|
||||
cancellationToken: cancellationToken);
|
||||
var queryVector768 = new Vector(embeddingResponse768.First().Vector.ToArray());
|
||||
|
||||
// 2. Perform Cosine Similarity Search on Knowledge Units
|
||||
List<KnowledgeUnit> candidates;
|
||||
bool isSqlite = dbContext.Database.ProviderName == "Microsoft.EntityFrameworkCore.Sqlite";
|
||||
|
||||
if (isSqlite)
|
||||
{
|
||||
var allUnits = await dbContext.KnowledgeUnits
|
||||
.AsNoTracking()
|
||||
.Where(x => (x.TenantId == request.TenantId || x.TenantId == "global") && x.Vector != null)
|
||||
.ToListAsync(cancellationToken);
|
||||
|
||||
candidates = allUnits
|
||||
.OrderBy(x => CalculateCosineDistance(x.Vector!, queryVector768))
|
||||
.Take(request.Limit)
|
||||
.ToList();
|
||||
}
|
||||
else
|
||||
{
|
||||
candidates = await dbContext.KnowledgeUnits
|
||||
.AsNoTracking()
|
||||
.Where(x => (x.TenantId == request.TenantId || x.TenantId == "global") && x.Vector != null)
|
||||
.OrderBy(x => x.Vector!.CosineDistance(queryVector768))
|
||||
.Take(request.Limit)
|
||||
.ToListAsync(cancellationToken);
|
||||
}
|
||||
|
||||
if (!candidates.Any())
|
||||
{
|
||||
// 3. Fallback to 1536-dimensional embedding for legacy cache search
|
||||
var embeddingResponse1536 = await _embeddingGenerator.GenerateAsync(
|
||||
new[] { request.QueryText },
|
||||
new EmbeddingGenerationOptions { Dimensions = 1536 },
|
||||
cancellationToken: cancellationToken);
|
||||
var queryVector1536 = new Vector(embeddingResponse1536.First().Vector.ToArray());
|
||||
|
||||
List<SemanticKnowledgeCache> legacyResults;
|
||||
if (isSqlite)
|
||||
{
|
||||
var allCache = await dbContext.SemanticKnowledgeCache
|
||||
.AsNoTracking()
|
||||
.Where(x => x.TenantId == request.TenantId && x.Vector != null)
|
||||
.ToListAsync(cancellationToken);
|
||||
|
||||
legacyResults = allCache
|
||||
.OrderBy(x => CalculateCosineDistance(x.Vector!, queryVector1536))
|
||||
.Take(request.Limit)
|
||||
.ToList();
|
||||
}
|
||||
else
|
||||
{
|
||||
legacyResults = await dbContext.SemanticKnowledgeCache
|
||||
.AsNoTracking()
|
||||
.Where(x => x.TenantId == request.TenantId && x.Vector != null)
|
||||
.OrderBy(x => x.Vector!.CosineDistance(queryVector1536))
|
||||
.Take(request.Limit)
|
||||
.ToListAsync(cancellationToken);
|
||||
}
|
||||
|
||||
return Result.Ok(legacyResults.Select(r => new SemanticSearchResultDto
|
||||
{
|
||||
ContentHash = r.ContentHash,
|
||||
Snippet = r.OriginalText,
|
||||
RelevanceScore = (float)(1 - (isSqlite ? CalculateCosineDistance(r.Vector!, queryVector1536) : r.Vector!.CosineDistance(queryVector1536)))
|
||||
}).ToList());
|
||||
}
|
||||
|
||||
// 3. Graph Expansion: Pull related units (e.g. Definitions, Next steps)
|
||||
var candidateIds = candidates.Select(c => c.Id).ToList();
|
||||
var links = await dbContext.KnowledgeUnitLinks
|
||||
.AsNoTracking()
|
||||
.Where(l => candidateIds.Contains(l.SourceUnitId) && (l.RelationType == "Defines" || l.RelationType == "Next"))
|
||||
.ToListAsync(cancellationToken);
|
||||
|
||||
var relatedIds = links.Select(l => l.TargetUnitId).Distinct().ToList();
|
||||
var relatedUnits = await dbContext.KnowledgeUnits
|
||||
.AsNoTracking()
|
||||
.Where(u => relatedIds.Contains(u.Id))
|
||||
.ToDictionaryAsync(u => u.Id, cancellationToken);
|
||||
|
||||
// 4. Mapping with Context Enrichment
|
||||
var dtos = candidates.Select(c =>
|
||||
{
|
||||
var dto = new SemanticSearchResultDto
|
||||
{
|
||||
ContentHash = c.Id.ToString(),
|
||||
Snippet = c.Content,
|
||||
UnitType = c.Type.ToString(),
|
||||
RelevanceScore = (float)(1 - (isSqlite ? CalculateCosineDistance(c.Vector!, queryVector768) : c.Vector!.CosineDistance(queryVector768))),
|
||||
Metadata = string.IsNullOrEmpty(c.MetadataJson)
|
||||
? null
|
||||
: JsonSerializer.Deserialize<Dictionary<string, object>>(c.MetadataJson)
|
||||
};
|
||||
|
||||
// Enrich snippet with definitions if present
|
||||
var unitLinks = links.Where(l => l.SourceUnitId == c.Id && l.RelationType == "Defines").ToList();
|
||||
if (unitLinks.Any())
|
||||
{
|
||||
var definitions = unitLinks
|
||||
.Where(l => relatedUnits.ContainsKey(l.TargetUnitId))
|
||||
.Select(l => relatedUnits[l.TargetUnitId].Content);
|
||||
dto.Snippet = $"[Context: {string.Join("; ", definitions)}]\n{dto.Snippet}";
|
||||
}
|
||||
|
||||
return dto;
|
||||
}).ToList();
|
||||
|
||||
return Result.Ok(dtos);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
return Result.Fail(new Error("Failed to perform semantic search").CausedBy(ex));
|
||||
}
|
||||
}
|
||||
|
||||
private static double CalculateCosineDistance(Vector v1, Vector v2)
|
||||
{
|
||||
var a = v1.ToArray();
|
||||
var b = v2.ToArray();
|
||||
if (a.Length != b.Length) return 1.0;
|
||||
double dotProduct = 0;
|
||||
double l1 = 0;
|
||||
double l2 = 0;
|
||||
for (int i = 0; i < a.Length; i++)
|
||||
{
|
||||
dotProduct += a[i] * b[i];
|
||||
l1 += a[i] * a[i];
|
||||
l2 += b[i] * b[i];
|
||||
}
|
||||
if (l1 == 0 || l2 == 0) return 1.0;
|
||||
return 1.0 - (dotProduct / (Math.Sqrt(l1) * Math.Sqrt(l2)));
|
||||
return await _knowledgeService.SearchLibrarySemanticallyAsync(
|
||||
request.QueryText,
|
||||
request.TenantId,
|
||||
request.Limit,
|
||||
cancellationToken);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user