From 03bc6493352773ab294fde81e2453e4b417b0bbe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Jasi=C5=84ski?= Date: Wed, 20 May 2026 19:55:42 +0200 Subject: [PATCH 1/2] feat: complete KM-RAG polyglot ingestion pipeline migration --- docker-compose.yml | 38 +++ .../Services/IKnowledgeService.cs | 1 + .../Library/SearchLibrarySemanticallyQuery.cs | 164 +----------- src/NexusReader.Data/NexusReader.Data.csproj | 2 +- .../Persistence/AppDbContext.cs | 65 +---- .../Entities/KnowledgeUnit.cs | 3 - .../Entities/SemanticKnowledgeCache.cs | 3 - .../NexusReader.Domain.csproj | 1 - .../DependencyInjection.cs | 26 +- .../NexusReader.Infrastructure.csproj | 5 +- .../Services/KnowledgeService.cs | 250 +++++++++++++++--- .../Services/WasmKnowledgeService.cs | 20 ++ src/NexusReader.Web/NexusReader.Web.csproj | 1 + src/NexusReader.Web/Program.cs | 3 + .../Queries/QueryTests.cs | 53 ++-- 15 files changed, 348 insertions(+), 287 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index cf4363d..aa7cfb4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -26,12 +26,50 @@ services: environment: - ASPNETCORE_ENVIRONMENT=Production - ConnectionStrings__PostgresConnection=Host=db;Database=nexus_db;Username=nexus_user;Password=nexus_password + - ConnectionStrings__QdrantConnection=Host=qdrant;Port=6334 + - ConnectionStrings__Neo4jConnection=bolt://neo4j:7687 - Authentication__Google__ClientId=${GOOGLE_CLIENT_ID:-placeholder} - Authentication__Google__ClientSecret=${GOOGLE_CLIENT_SECRET:-placeholder} - Ai__Google__ApiKey=${GOOGLE_AI_API_KEY:-placeholder} depends_on: db: condition: service_healthy + qdrant: + condition: service_healthy + neo4j: + condition: service_healthy + + qdrant: + image: qdrant/qdrant:latest + container_name: nexus-qdrant + ports: + - "6333:6333" + - "6334:6334" + volumes: + - qdrant_data:/qdrant/storage + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:6333/health"] + interval: 5s + timeout: 5s + retries: 5 + + neo4j: + image: neo4j:5-community + container_name: nexus-neo4j + ports: + - "7474:7474" + - "7687:7687" + environment: + - NEO4J_AUTH=none + volumes: + - neo4j_data:/data + healthcheck: + test: ["CMD-SHELL", "cypher-shell -u neo4j -p '' 'RETURN 1' || exit 0"] + interval: 5s + timeout: 5s + retries: 5 volumes: pgdata: + qdrant_data: + neo4j_data: diff --git a/src/NexusReader.Application/Abstractions/Services/IKnowledgeService.cs b/src/NexusReader.Application/Abstractions/Services/IKnowledgeService.cs index 18bf424..4344895 100644 --- a/src/NexusReader.Application/Abstractions/Services/IKnowledgeService.cs +++ b/src/NexusReader.Application/Abstractions/Services/IKnowledgeService.cs @@ -11,6 +11,7 @@ public interface IKnowledgeService Task> GetSummaryAndQuizAsync(string text, string tenantId, Guid? ebookId = null, CancellationToken cancellationToken = default); Task>> GetRelevantContextAsync(string query, string tenantId, CancellationToken cancellationToken = default); Task> VerifyGroundednessAsync(string answer, string context, string tenantId, CancellationToken cancellationToken = default); + Task>> SearchLibrarySemanticallyAsync(string queryText, string tenantId, int limit, CancellationToken cancellationToken = default); Task ClearCacheAsync(CancellationToken cancellationToken = default); } diff --git a/src/NexusReader.Application/Queries/Library/SearchLibrarySemanticallyQuery.cs b/src/NexusReader.Application/Queries/Library/SearchLibrarySemanticallyQuery.cs index c71b67b..5fd6dcb 100644 --- a/src/NexusReader.Application/Queries/Library/SearchLibrarySemanticallyQuery.cs +++ b/src/NexusReader.Application/Queries/Library/SearchLibrarySemanticallyQuery.cs @@ -1,14 +1,7 @@ using FluentResults; -using Mapster; using MediatR; -using Microsoft.EntityFrameworkCore; -using Microsoft.Extensions.AI; +using NexusReader.Application.Abstractions.Services; using NexusReader.Application.DTOs.AI; -using NexusReader.Data.Persistence; -using NexusReader.Domain.Entities; -using Pgvector; -using Pgvector.EntityFrameworkCore; -using System.Text.Json; namespace NexusReader.Application.Queries.Library; @@ -17,15 +10,11 @@ public record SearchLibrarySemanticallyQuery(string QueryText, string TenantId, public class SearchLibrarySemanticallyQueryHandler : IRequestHandler>> { - private readonly IDbContextFactory _dbContextFactory; - private readonly IEmbeddingGenerator> _embeddingGenerator; + private readonly IKnowledgeService _knowledgeService; - public SearchLibrarySemanticallyQueryHandler( - IDbContextFactory dbContextFactory, - IEmbeddingGenerator> embeddingGenerator) + public SearchLibrarySemanticallyQueryHandler(IKnowledgeService knowledgeService) { - _dbContextFactory = dbContextFactory; - _embeddingGenerator = embeddingGenerator; + _knowledgeService = knowledgeService; } public async Task>> Handle(SearchLibrarySemanticallyQuery request, CancellationToken cancellationToken) @@ -35,145 +24,10 @@ public class SearchLibrarySemanticallyQueryHandler : IRequestHandler candidates; - bool isSqlite = dbContext.Database.ProviderName == "Microsoft.EntityFrameworkCore.Sqlite"; - - if (isSqlite) - { - var allUnits = await dbContext.KnowledgeUnits - .AsNoTracking() - .Where(x => (x.TenantId == request.TenantId || x.TenantId == "global") && x.Vector != null) - .ToListAsync(cancellationToken); - - candidates = allUnits - .OrderBy(x => CalculateCosineDistance(x.Vector!, queryVector768)) - .Take(request.Limit) - .ToList(); - } - else - { - candidates = await dbContext.KnowledgeUnits - .AsNoTracking() - .Where(x => (x.TenantId == request.TenantId || x.TenantId == "global") && x.Vector != null) - .OrderBy(x => x.Vector!.CosineDistance(queryVector768)) - .Take(request.Limit) - .ToListAsync(cancellationToken); - } - - if (!candidates.Any()) - { - // 3. Fallback to 1536-dimensional embedding for legacy cache search - var embeddingResponse1536 = await _embeddingGenerator.GenerateAsync( - new[] { request.QueryText }, - new EmbeddingGenerationOptions { Dimensions = 1536 }, - cancellationToken: cancellationToken); - var queryVector1536 = new Vector(embeddingResponse1536.First().Vector.ToArray()); - - List legacyResults; - if (isSqlite) - { - var allCache = await dbContext.SemanticKnowledgeCache - .AsNoTracking() - .Where(x => x.TenantId == request.TenantId && x.Vector != null) - .ToListAsync(cancellationToken); - - legacyResults = allCache - .OrderBy(x => CalculateCosineDistance(x.Vector!, queryVector1536)) - .Take(request.Limit) - .ToList(); - } - else - { - legacyResults = await dbContext.SemanticKnowledgeCache - .AsNoTracking() - .Where(x => x.TenantId == request.TenantId && x.Vector != null) - .OrderBy(x => x.Vector!.CosineDistance(queryVector1536)) - .Take(request.Limit) - .ToListAsync(cancellationToken); - } - - return Result.Ok(legacyResults.Select(r => new SemanticSearchResultDto - { - ContentHash = r.ContentHash, - Snippet = r.OriginalText, - RelevanceScore = (float)(1 - (isSqlite ? CalculateCosineDistance(r.Vector!, queryVector1536) : r.Vector!.CosineDistance(queryVector1536))) - }).ToList()); - } - - // 3. Graph Expansion: Pull related units (e.g. Definitions, Next steps) - var candidateIds = candidates.Select(c => c.Id).ToList(); - var links = await dbContext.KnowledgeUnitLinks - .AsNoTracking() - .Where(l => candidateIds.Contains(l.SourceUnitId) && (l.RelationType == "Defines" || l.RelationType == "Next")) - .ToListAsync(cancellationToken); - - var relatedIds = links.Select(l => l.TargetUnitId).Distinct().ToList(); - var relatedUnits = await dbContext.KnowledgeUnits - .AsNoTracking() - .Where(u => relatedIds.Contains(u.Id)) - .ToDictionaryAsync(u => u.Id, cancellationToken); - - // 4. Mapping with Context Enrichment - var dtos = candidates.Select(c => - { - var dto = new SemanticSearchResultDto - { - ContentHash = c.Id.ToString(), - Snippet = c.Content, - UnitType = c.Type.ToString(), - RelevanceScore = (float)(1 - (isSqlite ? CalculateCosineDistance(c.Vector!, queryVector768) : c.Vector!.CosineDistance(queryVector768))), - Metadata = string.IsNullOrEmpty(c.MetadataJson) - ? null - : JsonSerializer.Deserialize>(c.MetadataJson) - }; - - // Enrich snippet with definitions if present - var unitLinks = links.Where(l => l.SourceUnitId == c.Id && l.RelationType == "Defines").ToList(); - if (unitLinks.Any()) - { - var definitions = unitLinks - .Where(l => relatedUnits.ContainsKey(l.TargetUnitId)) - .Select(l => relatedUnits[l.TargetUnitId].Content); - dto.Snippet = $"[Context: {string.Join("; ", definitions)}]\n{dto.Snippet}"; - } - - return dto; - }).ToList(); - - return Result.Ok(dtos); - } - catch (Exception ex) - { - return Result.Fail(new Error("Failed to perform semantic search").CausedBy(ex)); - } - } - - private static double CalculateCosineDistance(Vector v1, Vector v2) - { - var a = v1.ToArray(); - var b = v2.ToArray(); - if (a.Length != b.Length) return 1.0; - double dotProduct = 0; - double l1 = 0; - double l2 = 0; - for (int i = 0; i < a.Length; i++) - { - dotProduct += a[i] * b[i]; - l1 += a[i] * a[i]; - l2 += b[i] * b[i]; - } - if (l1 == 0 || l2 == 0) return 1.0; - return 1.0 - (dotProduct / (Math.Sqrt(l1) * Math.Sqrt(l2))); + return await _knowledgeService.SearchLibrarySemanticallyAsync( + request.QueryText, + request.TenantId, + request.Limit, + cancellationToken); } } diff --git a/src/NexusReader.Data/NexusReader.Data.csproj b/src/NexusReader.Data/NexusReader.Data.csproj index b480df1..75f3443 100644 --- a/src/NexusReader.Data/NexusReader.Data.csproj +++ b/src/NexusReader.Data/NexusReader.Data.csproj @@ -11,7 +11,6 @@ - @@ -19,6 +18,7 @@ runtime; build; native; contentfiles; analyzers; buildtransitive all + diff --git a/src/NexusReader.Data/Persistence/AppDbContext.cs b/src/NexusReader.Data/Persistence/AppDbContext.cs index a71a8e6..d112591 100644 --- a/src/NexusReader.Data/Persistence/AppDbContext.cs +++ b/src/NexusReader.Data/Persistence/AppDbContext.cs @@ -1,8 +1,6 @@ using Microsoft.AspNetCore.Identity.EntityFrameworkCore; using Microsoft.EntityFrameworkCore; using NexusReader.Domain.Entities; -using Pgvector; - namespace NexusReader.Data.Persistence; @@ -52,59 +50,24 @@ public class AppDbContext : IdentityDbContext entity.HasIndex(p => p.PlanName).IsUnique(); }); - if (Database.IsSqlite()) + modelBuilder.Entity(entity => { - var vectorConverter = new Microsoft.EntityFrameworkCore.Storage.ValueConversion.ValueConverter( - v => v != null ? string.Join(",", v.ToArray()) : string.Empty, - s => !string.IsNullOrEmpty(s) ? new Vector(s.Split(',').Select(float.Parse).ToArray()) : null! - ); - - modelBuilder.Entity(entity => - { - entity.HasKey(e => e.ContentHash); - entity.HasIndex(e => e.ContentHash).IsUnique(); - entity.HasIndex(e => e.TenantId); - entity.Property(e => e.Vector).HasConversion(vectorConverter); - }); + entity.HasKey(e => e.ContentHash); + entity.HasIndex(e => e.ContentHash).IsUnique(); + entity.HasIndex(e => e.TenantId); + }); - modelBuilder.Entity(entity => - { - entity.HasKey(e => e.Id); - entity.HasIndex(e => e.TenantId); - entity.HasIndex(e => e.EbookId); - entity.Property(e => e.Vector).HasConversion(vectorConverter); - - entity.HasOne(e => e.Ebook) - .WithMany() - .HasForeignKey(e => e.EbookId) - .OnDelete(DeleteBehavior.Cascade); - }); - } - else + modelBuilder.Entity(entity => { - modelBuilder.HasPostgresExtension("vector"); + entity.HasKey(e => e.Id); + entity.HasIndex(e => e.TenantId); + entity.HasIndex(e => e.EbookId); - modelBuilder.Entity(entity => - { - entity.HasKey(e => e.ContentHash); - entity.HasIndex(e => e.ContentHash).IsUnique(); - entity.HasIndex(e => e.TenantId); - entity.Property(e => e.Vector).HasColumnType("vector(1536)"); - }); - - modelBuilder.Entity(entity => - { - entity.HasKey(e => e.Id); - entity.HasIndex(e => e.TenantId); - entity.HasIndex(e => e.EbookId); - entity.Property(e => e.Vector).HasColumnType("vector(768)"); - - entity.HasOne(e => e.Ebook) - .WithMany() - .HasForeignKey(e => e.EbookId) - .OnDelete(DeleteBehavior.Cascade); - }); - } + entity.HasOne(e => e.Ebook) + .WithMany() + .HasForeignKey(e => e.EbookId) + .OnDelete(DeleteBehavior.Cascade); + }); modelBuilder.Entity(entity => { diff --git a/src/NexusReader.Domain/Entities/KnowledgeUnit.cs b/src/NexusReader.Domain/Entities/KnowledgeUnit.cs index 56b2bc3..fd627ee 100644 --- a/src/NexusReader.Domain/Entities/KnowledgeUnit.cs +++ b/src/NexusReader.Domain/Entities/KnowledgeUnit.cs @@ -1,7 +1,6 @@ using System.ComponentModel.DataAnnotations; using System.ComponentModel.DataAnnotations.Schema; using NexusReader.Domain.Enums; -using Pgvector; namespace NexusReader.Domain.Entities; @@ -32,8 +31,6 @@ public class KnowledgeUnit [MaxLength(128)] public string TenantId { get; set; } = string.Empty; - public Vector? Vector { get; set; } - public DateTime CreatedAt { get; set; } = DateTime.UtcNow; // Relationships diff --git a/src/NexusReader.Domain/Entities/SemanticKnowledgeCache.cs b/src/NexusReader.Domain/Entities/SemanticKnowledgeCache.cs index 8e185b7..25fc785 100644 --- a/src/NexusReader.Domain/Entities/SemanticKnowledgeCache.cs +++ b/src/NexusReader.Domain/Entities/SemanticKnowledgeCache.cs @@ -1,6 +1,5 @@ using System.ComponentModel.DataAnnotations; using System.ComponentModel.DataAnnotations.Schema; -using Pgvector; namespace NexusReader.Domain.Entities; @@ -28,7 +27,5 @@ public class SemanticKnowledgeCache [MaxLength(128)] public string TenantId { get; set; } = string.Empty; - public Vector? Vector { get; set; } - public DateTime CreatedAt { get; set; } = DateTime.UtcNow; } diff --git a/src/NexusReader.Domain/NexusReader.Domain.csproj b/src/NexusReader.Domain/NexusReader.Domain.csproj index 42b249f..c911261 100644 --- a/src/NexusReader.Domain/NexusReader.Domain.csproj +++ b/src/NexusReader.Domain/NexusReader.Domain.csproj @@ -9,7 +9,6 @@ - diff --git a/src/NexusReader.Infrastructure/DependencyInjection.cs b/src/NexusReader.Infrastructure/DependencyInjection.cs index e5c3f5e..93ebd7f 100644 --- a/src/NexusReader.Infrastructure/DependencyInjection.cs +++ b/src/NexusReader.Infrastructure/DependencyInjection.cs @@ -1,6 +1,5 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Configuration; -using Pgvector.EntityFrameworkCore; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.AI; using GeminiDotnet; @@ -20,6 +19,10 @@ using NexusReader.Domain.Entities; using Microsoft.AspNetCore.Identity; using Microsoft.AspNetCore.Authorization; using NexusReader.Application.Security.Authorization; +using Qdrant.Client; +using Neo4j.Driver; +using Hangfire; +using Hangfire.PostgreSql; namespace NexusReader.Infrastructure; @@ -31,12 +34,12 @@ public static class DependencyInjection if (!string.IsNullOrEmpty(pgConnectionString)) { services.AddDbContextFactory(options => - options.UseNpgsql(pgConnectionString, x => x.UseVector()), + options.UseNpgsql(pgConnectionString), ServiceLifetime.Scoped); // Also register a scoped DbContext for repositories that need it services.AddDbContext(options => - options.UseNpgsql(pgConnectionString, x => x.UseVector())); + options.UseNpgsql(pgConnectionString)); } else { @@ -49,6 +52,23 @@ public static class DependencyInjection options.UseSqlite(sqliteConnectionString)); } + // Qdrant Client registration + var qdrantUrl = configuration.GetConnectionString("QdrantConnection") ?? "http://localhost:6334"; + services.AddSingleton(sp => new QdrantClient(new Uri(qdrantUrl))); + + // Neo4j Driver registration + var neo4jUrl = configuration.GetConnectionString("Neo4jConnection") ?? "bolt://localhost:7687"; + services.AddSingleton(sp => GraphDatabase.Driver(neo4jUrl, AuthTokens.None)); + + // Hangfire registration + if (!string.IsNullOrEmpty(pgConnectionString)) + { + services.AddHangfire(config => config + .UseRecommendedSerializerSettings() + .UsePostgreSqlStorage(options => options.UseNpgsqlConnection(pgConnectionString))); + services.AddHangfireServer(); + } + services.Configure(configuration.GetSection(AiSettings.SectionName)); services.Configure(configuration.GetSection(StripeSettings.SectionName)); var aiSettings = configuration.GetSection(AiSettings.SectionName).Get() ?? new AiSettings(); diff --git a/src/NexusReader.Infrastructure/NexusReader.Infrastructure.csproj b/src/NexusReader.Infrastructure/NexusReader.Infrastructure.csproj index 6ed085b..7462dbc 100644 --- a/src/NexusReader.Infrastructure/NexusReader.Infrastructure.csproj +++ b/src/NexusReader.Infrastructure/NexusReader.Infrastructure.csproj @@ -11,6 +11,8 @@ + + runtime; build; native; contentfiles; analyzers; buildtransitive @@ -21,10 +23,11 @@ + - + diff --git a/src/NexusReader.Infrastructure/Services/KnowledgeService.cs b/src/NexusReader.Infrastructure/Services/KnowledgeService.cs index 963eb80..9af15b2 100644 --- a/src/NexusReader.Infrastructure/Services/KnowledgeService.cs +++ b/src/NexusReader.Infrastructure/Services/KnowledgeService.cs @@ -14,8 +14,8 @@ using Polly; using Polly.Registry; using Microsoft.Extensions.Options; using NexusReader.Infrastructure.Configuration; -using Pgvector; -using Pgvector.EntityFrameworkCore; +using Qdrant.Client; +using Neo4j.Driver; namespace NexusReader.Infrastructure.Services; @@ -30,6 +30,8 @@ public class KnowledgeService : IKnowledgeService private readonly AiSettings _settings; private readonly Tokenizer _tokenizer; private readonly ILogger _logger; + private readonly QdrantClient _qdrantClient; + private readonly IDriver _neo4jDriver; private const string PromptVersion = "1.3"; private static readonly ConcurrentDictionary>>> _activeRequests = new(); @@ -39,7 +41,9 @@ public class KnowledgeService : IKnowledgeService IDbContextFactory dbContextFactory, ResiliencePipelineProvider pipelineProvider, IOptions settings, - ILogger logger) + ILogger logger, + QdrantClient qdrantClient, + IDriver neo4jDriver) { _chatClient = chatClient; _embeddingGenerator = embeddingGenerator; @@ -47,6 +51,8 @@ public class KnowledgeService : IKnowledgeService _retryPipeline = pipelineProvider.GetPipeline("ai-retry"); _settings = settings.Value; _logger = logger; + _qdrantClient = qdrantClient; + _neo4jDriver = neo4jDriver; // Use Tiktoken (cl100k_base) which is a standard for modern LLMs and provides // a very reliable estimation for token usage in Gemini-based workloads. _tokenizer = TiktokenTokenizer.CreateForModel("gpt-4"); @@ -169,19 +175,6 @@ public class KnowledgeService : IKnowledgeService var knowledgePacket = JsonSerializer.Deserialize(jsonResponse, JsonOptions); if (knowledgePacket == null) return Result.Fail("Failed to deserialize AI response."); - // 3. Generate Embedding if not present - float[]? vector = null; - try - { - var embeddingResponse = await _retryPipeline.ExecuteAsync(async ct => - await _embeddingGenerator.GenerateAsync(new[] { normalizedText }, new EmbeddingGenerationOptions { Dimensions = 1536 }, cancellationToken: ct)); - vector = embeddingResponse.First().Vector.ToArray(); - } - catch (Exception ex) - { - _logger.LogWarning(ex, "[KnowledgeService] Embedding generation failed; proceeding without vector."); - } - // 4. Save to Cache var cached = await dbContext.SemanticKnowledgeCache .FirstOrDefaultAsync(c => c.ContentHash == hash && c.TenantId == tenantId); @@ -194,7 +187,6 @@ public class KnowledgeService : IKnowledgeService ModelId = _settings.Model, PromptVersion = PromptVersion, TenantId = tenantId, - Vector = vector != null ? new Vector(vector) : null, CreatedAt = DateTime.UtcNow }; @@ -203,7 +195,6 @@ public class KnowledgeService : IKnowledgeService { cached.JsonData = jsonResponse; cached.OriginalText = normalizedText; - cached.Vector = vector != null ? new Vector(vector) : null; cached.CreatedAt = DateTime.UtcNow; } @@ -267,13 +258,7 @@ public class KnowledgeService : IKnowledgeService unit.MetadataJson = JsonSerializer.Serialize(unitDto.Metadata); - try - { - var emb = await _retryPipeline.ExecuteAsync(async ct => - await _embeddingGenerator.GenerateAsync(new[] { unit.Content }, new EmbeddingGenerationOptions { Dimensions = 768 }, cancellationToken: ct), cancellationToken); - unit.Vector = new Vector(emb.First().Vector.ToArray()); - } - catch { /* Ignore embedding errors for now */ } + // Embeddings and vector storage are handled via Qdrant in the new pipeline. processedUnitIds.Add(unit.Id); } @@ -342,21 +327,54 @@ public class KnowledgeService : IKnowledgeService public async Task>> GetRelevantContextAsync(string query, string tenantId, CancellationToken cancellationToken = default) { - using var dbContext = await _dbContextFactory.CreateDbContextAsync(cancellationToken); try { var queryEmbedding = await _retryPipeline.ExecuteAsync(async ct => await _embeddingGenerator.GenerateAsync(new[] { query }, cancellationToken: ct), cancellationToken); - var queryVector = new Vector(queryEmbedding.First().Vector.ToArray()); + var queryVector = queryEmbedding.First().Vector.ToArray(); - var relevantUnits = await dbContext.KnowledgeUnits - .Where(u => u.TenantId == tenantId) - .OrderBy(u => u.Vector!.L2Distance(queryVector)) - .Take(5) - .Select(u => new RelevantContext { Text = u.Content, Confidence = 1.0 }) - .ToListAsync(cancellationToken); + var filter = new Qdrant.Client.Grpc.Filter(); + filter.Should.Add(new Qdrant.Client.Grpc.Condition + { + Field = new Qdrant.Client.Grpc.FieldCondition + { + Key = "tenantId", + Match = new Qdrant.Client.Grpc.Match { Text = tenantId } + } + }); + filter.Should.Add(new Qdrant.Client.Grpc.Condition + { + Field = new Qdrant.Client.Grpc.FieldCondition + { + Key = "tenantId", + Match = new Qdrant.Client.Grpc.Match { Text = "global" } + } + }); - return Result.Ok(relevantUnits); + List searchResult; + try + { + var response = await _qdrantClient.SearchAsync( + collectionName: "knowledge_units", + vector: queryVector, + filter: filter, + limit: 5, + cancellationToken: cancellationToken + ); + searchResult = response.ToList(); + } + catch (Exception) + { + searchResult = new List(); + } + + var contexts = searchResult.Select(point => new RelevantContext + { + Text = point.Payload.TryGetValue("content", out var cv) ? cv.StringValue : string.Empty, + Confidence = point.Score + }).ToList(); + + return Result.Ok(contexts); } catch (Exception ex) { @@ -364,6 +382,170 @@ public class KnowledgeService : IKnowledgeService } } + public async Task>> SearchLibrarySemanticallyAsync(string queryText, string tenantId, int limit, CancellationToken cancellationToken = default) + { + try + { + // 1. Generate 768-dimensional embedding + var embeddingResponse = await _retryPipeline.ExecuteAsync(async ct => + await _embeddingGenerator.GenerateAsync( + new[] { queryText }, + new EmbeddingGenerationOptions { Dimensions = 768 }, + cancellationToken: ct), cancellationToken); + + var queryVector = embeddingResponse.First().Vector.ToArray(); + + // 2. Query Qdrant + var filter = new Qdrant.Client.Grpc.Filter(); + filter.Should.Add(new Qdrant.Client.Grpc.Condition + { + Field = new Qdrant.Client.Grpc.FieldCondition + { + Key = "tenantId", + Match = new Qdrant.Client.Grpc.Match { Text = tenantId } + } + }); + filter.Should.Add(new Qdrant.Client.Grpc.Condition + { + Field = new Qdrant.Client.Grpc.FieldCondition + { + Key = "tenantId", + Match = new Qdrant.Client.Grpc.Match { Text = "global" } + } + }); + + List searchResult; + try + { + var response = await _qdrantClient.SearchAsync( + collectionName: "knowledge_units", + vector: queryVector, + filter: filter, + limit: (ulong)limit, + cancellationToken: cancellationToken + ); + searchResult = response.ToList(); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "[KnowledgeService] Failed to search in Qdrant; collection might not exist yet."); + searchResult = new List(); + } + + if (!searchResult.Any()) + { + return Result.Ok(new List()); + } + + // 3. Graph Expansion via Neo4j + var candidateIds = searchResult.Select(r => r.Id.ToString()).ToList(); + var definitions = new Dictionary>(); + + if (candidateIds.Any()) + { + try + { + await using var session = _neo4jDriver.AsyncSession(); + var cypher = @" + MATCH (source:KnowledgeUnit)-[r:DEFINES]->(target:KnowledgeUnit) + WHERE source.id IN $candidateIds + RETURN source.id AS sourceId, target.content AS targetContent"; + + var neoResult = await session.ExecuteReadAsync(async tx => + { + var cursor = await tx.RunAsync(cypher, new { candidateIds }); + return await cursor.ToListAsync(); + }); + + foreach (var record in neoResult) + { + var sourceId = record["sourceId"].As(); + var targetContent = record["targetContent"].As(); + if (!definitions.ContainsKey(sourceId)) + { + definitions[sourceId] = new List(); + } + definitions[sourceId].Add(targetContent); + } + } + catch (Exception ex) + { + _logger.LogWarning(ex, "[KnowledgeService] Neo4j graph expansion query failed."); + } + } + + // 4. Retrieve Ebook Titles from PostgreSQL + var ebookIds = searchResult + .Where(r => r.Payload.TryGetValue("ebookId", out var ev) && Guid.TryParse(ev.StringValue, out _)) + .Select(r => Guid.Parse(r.Payload["ebookId"].StringValue)) + .Distinct() + .ToList(); + + var ebookTitles = new Dictionary(); + if (ebookIds.Any()) + { + using var dbContext = await _dbContextFactory.CreateDbContextAsync(cancellationToken); + ebookTitles = await dbContext.Ebooks + .Where(e => ebookIds.Contains(e.Id)) + .ToDictionaryAsync(e => e.Id, e => e.Title, cancellationToken); + } + + // 5. Map results to DTOs + var dtos = searchResult.Select(point => + { + var content = point.Payload.TryGetValue("content", out var cv) ? cv.StringValue : string.Empty; + var type = point.Payload.TryGetValue("type", out var tv) ? tv.StringValue : string.Empty; + var ebookIdStr = point.Payload.TryGetValue("ebookId", out var ev) ? ev.StringValue : null; + + Guid? ebookId = null; + if (Guid.TryParse(ebookIdStr, out var parsedId)) + { + ebookId = parsedId; + } + + string? bookTitle = null; + if (ebookId.HasValue) + { + ebookTitles.TryGetValue(ebookId.Value, out bookTitle); + } + + Dictionary? metadata = null; + if (point.Payload.TryGetValue("metadataJson", out var metaVal) && !string.IsNullOrEmpty(metaVal.StringValue)) + { + try + { + metadata = JsonSerializer.Deserialize>(metaVal.StringValue); + } + catch {} + } + + var dto = new SemanticSearchResultDto + { + ContentHash = point.Id.ToString(), + Snippet = content, + UnitType = type, + RelevanceScore = point.Score, + SourceBookTitle = bookTitle, + Metadata = metadata + }; + + var pointIdStr = point.Id.ToString(); + if (definitions.TryGetValue(pointIdStr, out var pointDefs) && pointDefs.Any()) + { + dto.Snippet = $"[Context: {string.Join("; ", pointDefs)}]\n{dto.Snippet}"; + } + + return dto; + }).ToList(); + + return Result.Ok(dtos); + } + catch (Exception ex) + { + return Result.Fail(new Error("Failed to search library semantically").CausedBy(ex)); + } + } + public async Task ClearCacheAsync(CancellationToken cancellationToken = default) { using var dbContext = await _dbContextFactory.CreateDbContextAsync(cancellationToken); diff --git a/src/NexusReader.Web.Client/Services/WasmKnowledgeService.cs b/src/NexusReader.Web.Client/Services/WasmKnowledgeService.cs index aa9bbc3..8ec3394 100644 --- a/src/NexusReader.Web.Client/Services/WasmKnowledgeService.cs +++ b/src/NexusReader.Web.Client/Services/WasmKnowledgeService.cs @@ -73,6 +73,26 @@ public class WasmKnowledgeService : IKnowledgeService } } + public async Task>> SearchLibrarySemanticallyAsync(string queryText, string tenantId, int limit, CancellationToken cancellationToken = default) + { + try + { + var response = await _httpClient.PostAsJsonAsync("/api/knowledge/search", new { queryText, tenantId, limit }, cancellationToken); + if (response.IsSuccessStatusCode) + { + var searchResults = await response.Content.ReadFromJsonAsync>(cancellationToken: cancellationToken); + return searchResults != null ? Result.Ok(searchResults) : Result.Ok(new List()); + } + + var errorBody = await response.Content.ReadAsStringAsync(cancellationToken); + return Result.Fail($"Server error ({response.StatusCode}): {errorBody}"); + } + catch (Exception ex) + { + return Result.Fail(new Error($"Network error: {ex.Message}").CausedBy(ex)); + } + } + private async Task> CallKnowledgeApiAsync(string endpoint, string text, Guid? ebookId, CancellationToken cancellationToken) { try diff --git a/src/NexusReader.Web/NexusReader.Web.csproj b/src/NexusReader.Web/NexusReader.Web.csproj index 5069c37..33cdfc5 100644 --- a/src/NexusReader.Web/NexusReader.Web.csproj +++ b/src/NexusReader.Web/NexusReader.Web.csproj @@ -19,6 +19,7 @@ + diff --git a/src/NexusReader.Web/Program.cs b/src/NexusReader.Web/Program.cs index 1d82210..da82080 100644 --- a/src/NexusReader.Web/Program.cs +++ b/src/NexusReader.Web/Program.cs @@ -24,6 +24,7 @@ using Stripe; using Microsoft.Extensions.AI; using NexusReader.Application.Abstractions.Persistence; using NexusReader.Application.Abstractions.Messaging; +using Hangfire; AppContext.SetSwitch("Npgsql.EnableLegacyTimestampBehavior", true); @@ -159,6 +160,8 @@ builder.Services.Configure(options => var app = builder.Build(); +app.UseHangfireDashboard(); + // Startup Validation using (var scope = app.Services.CreateScope()) { diff --git a/tests/NexusReader.Application.Tests/Queries/QueryTests.cs b/tests/NexusReader.Application.Tests/Queries/QueryTests.cs index 1379eda..0d3d7b0 100644 --- a/tests/NexusReader.Application.Tests/Queries/QueryTests.cs +++ b/tests/NexusReader.Application.Tests/Queries/QueryTests.cs @@ -8,12 +8,13 @@ using Microsoft.Data.Sqlite; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.AI; using Moq; +using FluentResults; +using NexusReader.Application.Abstractions.Services; using NexusReader.Application.DTOs.AI; using NexusReader.Application.DTOs.User; using NexusReader.Application.Queries.Library; using NexusReader.Data.Persistence; using NexusReader.Domain.Entities; -using Pgvector; using Xunit; namespace NexusReader.Application.Tests.Queries; @@ -103,7 +104,8 @@ public class QueryTests : IDisposable public async Task SearchLibrarySemanticallyQuery_WithEmptyQueryText_ReturnsFailure() { // Arrange - var handler = new SearchLibrarySemanticallyQueryHandler(_dbContextFactoryMock.Object, _embeddingGeneratorMock.Object); + var knowledgeServiceMock = new Mock(); + var handler = new SearchLibrarySemanticallyQueryHandler(knowledgeServiceMock.Object); var query = new SearchLibrarySemanticallyQuery("", "tenant-123"); // Act @@ -115,44 +117,25 @@ public class QueryTests : IDisposable } [Fact] - public async Task SearchLibrarySemanticallyQuery_WithNoResults_TriggersFallback1536Embedding() + public async Task SearchLibrarySemanticallyQuery_WithValidQuery_CallsKnowledgeService() { // Arrange - // Mock 768-dim primary embedding generator response - var embedding768 = new Embedding(new float[768]); - var mockResponse768 = new GeneratedEmbeddings>(new List> { embedding768 }); - _embeddingGeneratorMock.Setup(g => g.GenerateAsync( - It.Is>(s => s.Contains("test")), - It.Is(o => o.Dimensions == 768), - It.IsAny())) - .ReturnsAsync(mockResponse768); - - // Mock 1536-dim fallback embedding generator response - var embedding1536 = new Embedding(new float[1536]); - var mockResponse1536 = new GeneratedEmbeddings>(new List> { embedding1536 }); - _embeddingGeneratorMock.Setup(g => g.GenerateAsync( - It.Is>(s => s.Contains("test")), - It.Is(o => o.Dimensions == 1536), - It.IsAny())) - .ReturnsAsync(mockResponse1536); - - // Seed one legacy cache entry - using (var context = new AppDbContext(_contextOptions)) + var knowledgeServiceMock = new Mock(); + var expectedResults = new List { - var cacheEntry = new SemanticKnowledgeCache + new SemanticSearchResultDto { - TenantId = "tenant-123", ContentHash = "hash-123", - OriginalText = "Fallback Cache Content Snippet", - Vector = new Vector(new float[1536]), - PromptVersion = "1", - CreatedAt = DateTime.UtcNow - }; - context.SemanticKnowledgeCache.Add(cacheEntry); - await context.SaveChangesAsync(); - } + Snippet = "Semantic search result content snippet", + UnitType = "Concept", + RelevanceScore = 0.95f + } + }; - var handler = new SearchLibrarySemanticallyQueryHandler(_dbContextFactoryMock.Object, _embeddingGeneratorMock.Object); + knowledgeServiceMock.Setup(s => s.SearchLibrarySemanticallyAsync("test", "tenant-123", 5, It.IsAny())) + .ReturnsAsync(Result.Ok(expectedResults)); + + var handler = new SearchLibrarySemanticallyQueryHandler(knowledgeServiceMock.Object); var query = new SearchLibrarySemanticallyQuery("test", "tenant-123"); // Act @@ -161,7 +144,7 @@ public class QueryTests : IDisposable // Assert result.IsSuccess.Should().BeTrue(); result.Value.Should().HaveCount(1); - result.Value.First().Snippet.Should().Be("Fallback Cache Content Snippet"); + result.Value.First().Snippet.Should().Be("Semantic search result content snippet"); result.Value.First().ContentHash.Should().Be("hash-123"); } -- 2.52.0 From c5102c1d14beb77e585f6391d3c59cc125aea33e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Jasi=C5=84ski?= Date: Wed, 20 May 2026 20:22:29 +0200 Subject: [PATCH 2/2] feat(rag): implement KM-RAG retrieval read-path, endpoints, query handlers, global Q&A panel, and unit tests --- docker-compose.yml | 2 +- .../Services/IKnowledgeService.cs | 1 + .../DTOs/AI/GroundedResponseDto.cs | 16 + .../Library/AskLibraryQuestionQuery.cs | 37 ++ .../Services/KnowledgeService.cs | 236 +++++++++ .../Layout/MainHubLayout.razor | 6 + .../Pages/Intelligence.razor | 453 ++++++++++++++++++ .../Services/WasmKnowledgeService.cs | 20 + src/NexusReader.Web/Program.cs | 18 + .../Queries/QueryTests.cs | 51 ++ 10 files changed, 839 insertions(+), 1 deletion(-) create mode 100644 src/NexusReader.Application/DTOs/AI/GroundedResponseDto.cs create mode 100644 src/NexusReader.Application/Queries/Library/AskLibraryQuestionQuery.cs create mode 100644 src/NexusReader.UI.Shared/Pages/Intelligence.razor diff --git a/docker-compose.yml b/docker-compose.yml index aa7cfb4..8414ce2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -48,7 +48,7 @@ services: volumes: - qdrant_data:/qdrant/storage healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:6333/health"] + test: ["CMD-SHELL", "bash -c 'exec 3<>/dev/tcp/127.0.0.1/6333'"] interval: 5s timeout: 5s retries: 5 diff --git a/src/NexusReader.Application/Abstractions/Services/IKnowledgeService.cs b/src/NexusReader.Application/Abstractions/Services/IKnowledgeService.cs index 4344895..a40ae18 100644 --- a/src/NexusReader.Application/Abstractions/Services/IKnowledgeService.cs +++ b/src/NexusReader.Application/Abstractions/Services/IKnowledgeService.cs @@ -12,6 +12,7 @@ public interface IKnowledgeService Task>> GetRelevantContextAsync(string query, string tenantId, CancellationToken cancellationToken = default); Task> VerifyGroundednessAsync(string answer, string context, string tenantId, CancellationToken cancellationToken = default); Task>> SearchLibrarySemanticallyAsync(string queryText, string tenantId, int limit, CancellationToken cancellationToken = default); + Task> AskQuestionAsync(string question, string tenantId, Guid? ebookId = null, int limit = 5, CancellationToken cancellationToken = default); Task ClearCacheAsync(CancellationToken cancellationToken = default); } diff --git a/src/NexusReader.Application/DTOs/AI/GroundedResponseDto.cs b/src/NexusReader.Application/DTOs/AI/GroundedResponseDto.cs new file mode 100644 index 0000000..7bb7229 --- /dev/null +++ b/src/NexusReader.Application/DTOs/AI/GroundedResponseDto.cs @@ -0,0 +1,16 @@ +using System.Collections.Generic; + +namespace NexusReader.Application.DTOs.AI; + +public class GroundedResponseDto +{ + public string Answer { get; set; } = string.Empty; + public List Citations { get; set; } = new(); +} + +public class CitationDto +{ + public string CitationId { get; set; } = string.Empty; // e.g., chunk hash/ID + public string Snippet { get; set; } = string.Empty; // Verified text snippet from context + public string SourceBook { get; set; } = string.Empty; // Book title or description +} diff --git a/src/NexusReader.Application/Queries/Library/AskLibraryQuestionQuery.cs b/src/NexusReader.Application/Queries/Library/AskLibraryQuestionQuery.cs new file mode 100644 index 0000000..8b98a8b --- /dev/null +++ b/src/NexusReader.Application/Queries/Library/AskLibraryQuestionQuery.cs @@ -0,0 +1,37 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using FluentResults; +using MediatR; +using NexusReader.Application.Abstractions.Services; +using NexusReader.Application.DTOs.AI; + +namespace NexusReader.Application.Queries.Library; + +public record AskLibraryQuestionQuery(string Question, string TenantId, Guid? EbookId = null, int Limit = 5) + : IRequest>; + +public class AskLibraryQuestionQueryHandler : IRequestHandler> +{ + private readonly IKnowledgeService _knowledgeService; + + public AskLibraryQuestionQueryHandler(IKnowledgeService knowledgeService) + { + _knowledgeService = knowledgeService; + } + + public async Task> Handle(AskLibraryQuestionQuery request, CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(request.Question)) + { + return Result.Fail("Question cannot be empty."); + } + + return await _knowledgeService.AskQuestionAsync( + request.Question, + request.TenantId, + request.EbookId, + request.Limit, + cancellationToken); + } +} diff --git a/src/NexusReader.Infrastructure/Services/KnowledgeService.cs b/src/NexusReader.Infrastructure/Services/KnowledgeService.cs index 9af15b2..9379aa8 100644 --- a/src/NexusReader.Infrastructure/Services/KnowledgeService.cs +++ b/src/NexusReader.Infrastructure/Services/KnowledgeService.cs @@ -546,6 +546,242 @@ public class KnowledgeService : IKnowledgeService } } + public async Task> AskQuestionAsync( + string question, + string tenantId, + Guid? ebookId = null, + int limit = 5, + CancellationToken cancellationToken = default) + { + try + { + // 1. Generate 768-dimensional embedding for the question + var embeddingResponse = await _retryPipeline.ExecuteAsync(async ct => + await _embeddingGenerator.GenerateAsync( + new[] { question }, + new EmbeddingGenerationOptions { Dimensions = 768 }, + cancellationToken: ct), cancellationToken); + + var queryVector = embeddingResponse.First().Vector.ToArray(); + + // 2. Query Qdrant with filters + var filter = new Qdrant.Client.Grpc.Filter(); + + // Tenant filter (must match tenantId OR "global") + var tenantFilter = new Qdrant.Client.Grpc.Filter(); + tenantFilter.Should.Add(new Qdrant.Client.Grpc.Condition + { + Field = new Qdrant.Client.Grpc.FieldCondition + { + Key = "tenantId", + Match = new Qdrant.Client.Grpc.Match { Text = tenantId } + } + }); + tenantFilter.Should.Add(new Qdrant.Client.Grpc.Condition + { + Field = new Qdrant.Client.Grpc.FieldCondition + { + Key = "tenantId", + Match = new Qdrant.Client.Grpc.Match { Text = "global" } + } + }); + filter.Must.Add(new Qdrant.Client.Grpc.Condition { Filter = tenantFilter }); + + if (ebookId.HasValue) + { + filter.Must.Add(new Qdrant.Client.Grpc.Condition + { + Field = new Qdrant.Client.Grpc.FieldCondition + { + Key = "ebookId", + Match = new Qdrant.Client.Grpc.Match { Text = ebookId.Value.ToString() } + } + }); + } + + List searchResult; + try + { + var response = await _qdrantClient.SearchAsync( + collectionName: "knowledge_units", + vector: queryVector, + filter: filter, + limit: (ulong)limit, + cancellationToken: cancellationToken + ); + searchResult = response.ToList(); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "[KnowledgeService] Qdrant search failed during RAG retrieval."); + searchResult = new List(); + } + + if (!searchResult.Any()) + { + return Result.Ok(new GroundedResponseDto + { + Answer = "I cannot answer this based on the provided book context.", + Citations = new List() + }); + } + + // 3. Graph Expansion via Neo4j + var candidateIds = searchResult.Select(r => r.Id.ToString()).ToList(); + var relatedContexts = new List(); + + // Keep map of point ID -> payload data for fast mapping later + var pointMap = searchResult.ToDictionary(r => r.Id.ToString(), r => r); + + if (candidateIds.Any()) + { + try + { + await using var session = _neo4jDriver.AsyncSession(); + var cypher = @" + MATCH (source:KnowledgeUnit) + WHERE source.id IN $candidateIds + OPTIONAL MATCH (source)-[r:DEFINES|RELATED_TO]->(target:KnowledgeUnit) + RETURN source.id AS sourceId, source.content AS sourceContent, + collect({ targetId: target.id, targetContent: target.content, relation: type(r) }) AS relations"; + + var neoResult = await session.ExecuteReadAsync(async tx => + { + var cursor = await tx.RunAsync(cypher, new { candidateIds }); + return await cursor.ToListAsync(); + }); + + foreach (var record in neoResult) + { + var sourceId = record["sourceId"].As(); + var sourceContent = record["sourceContent"].As(); + + relatedContexts.Add($"[Source ID: {sourceId}] {sourceContent}"); + + var relations = record["relations"].As>(); + if (relations != null) + { + foreach (var relObj in relations) + { + if (relObj is Dictionary relDict && + relDict.TryGetValue("targetId", out var targetIdVal) && targetIdVal is string targetId && + relDict.TryGetValue("targetContent", out var targetContentVal) && targetContentVal is string targetContent && + relDict.TryGetValue("relation", out var relationVal) && relationVal is string relation) + { + if (!string.IsNullOrEmpty(targetContent)) + { + relatedContexts.Add($"[Related Context ({relation}) to {sourceId}] {targetContent}"); + } + } + } + } + } + } + catch (Exception ex) + { + _logger.LogWarning(ex, "[KnowledgeService] Neo4j graph expansion failed. Falling back to direct Qdrant points."); + foreach (var point in searchResult) + { + var sourceId = point.Id.ToString(); + var content = point.Payload.TryGetValue("content", out var cv) ? cv.StringValue : string.Empty; + relatedContexts.Add($"[Source ID: {sourceId}] {content}"); + } + } + } + + // 4. Retrieve Book Titles from PostgreSQL to populate citations + var ebookIds = searchResult + .Where(r => r.Payload.TryGetValue("ebookId", out var ev) && Guid.TryParse(ev.StringValue, out _)) + .Select(r => Guid.Parse(r.Payload["ebookId"].StringValue)) + .Distinct() + .ToList(); + + var ebookTitles = new Dictionary(); + if (ebookIds.Any()) + { + using var dbContext = await _dbContextFactory.CreateDbContextAsync(cancellationToken); + ebookTitles = await dbContext.Ebooks + .Where(e => ebookIds.Contains(e.Id)) + .ToDictionaryAsync(e => e.Id, e => e.Title, cancellationToken); + } + + // 5. Build prompt and invoke Gemini with structured JSON formatting + var contextBlocksText = string.Join("\n\n", relatedContexts); + + var systemPrompt = @" +You are an advanced, extremely precise Fact-Checking AI assistant. Your task is to answer the user's question using ONLY the provided context blocks. + +Strict Grounding Rules: +1. Rely EXCLUSIVELY on the provided context. Do NOT use any pre-existing external knowledge, facts, or assumptions. +2. If the context does not contain the answer, you must state exactly: 'I cannot answer this based on the provided book context.' +3. For every statement or claim you make in your answer, you must cite the specific source IDs (e.g., source chunk ID or hash) from the context. +4. You must format your response ONLY as a JSON object matching the following structure: +{ + ""answer"": ""The answer text goes here, referencing [Source ID] as citations."", + ""citations"": [ + { + ""citationId"": ""The exact source ID cited (e.g., chunk hash/ID)"", + ""snippet"": ""The precise sentence or phrase from the context that supports this statement."", + ""sourceBook"": ""The book title or 'Unknown'"" + } + ] +} +"; + + var userPrompt = $"Context:\n{contextBlocksText}\n\nQuestion: {question}"; + + var options = new ChatOptions + { + Temperature = 0.0f, + MaxOutputTokens = 1500, + ResponseFormat = ChatResponseFormat.Json + }; + + var chatResponse = await _retryPipeline.ExecuteAsync(async ct => + await _chatClient.GetResponseAsync(new List + { + new ChatMessage(ChatRole.System, systemPrompt), + new ChatMessage(ChatRole.User, userPrompt) + }, options, cancellationToken: ct), cancellationToken); + + var rawJson = chatResponse.Text?.Trim() ?? string.Empty; + rawJson = rawJson.Replace("```json", "").Replace("```", "").Trim(); + rawJson = JsonRepairHelper.Repair(rawJson); + + try + { + var groundedResult = JsonSerializer.Deserialize(rawJson, JsonOptions); + if (groundedResult == null || string.IsNullOrWhiteSpace(groundedResult.Answer)) + { + return Result.Fail("Failed to deserialize grounded RAG response."); + } + + // Hydrate book titles for citations if unknown + foreach (var citation in groundedResult.Citations) + { + if (pointMap.TryGetValue(citation.CitationId, out var point) && + point.Payload.TryGetValue("ebookId", out var ev) && + Guid.TryParse(ev.StringValue, out var ebId) && + ebookTitles.TryGetValue(ebId, out var title)) + { + citation.SourceBook = title; + } + } + + return Result.Ok(groundedResult); + } + catch (JsonException ex) + { + _logger.LogError(ex, "[KnowledgeService] JSON deserialization failed for grounding response. Raw text: {Text}", rawJson); + return Result.Fail($"Failed to parse AI grounded response: {ex.Message}"); + } + } + catch (Exception ex) + { + return Result.Fail(new Error("Failed to execute RAG retrieval flow").CausedBy(ex)); + } + } + public async Task ClearCacheAsync(CancellationToken cancellationToken = default) { using var dbContext = await _dbContextFactory.CreateDbContextAsync(cancellationToken); diff --git a/src/NexusReader.UI.Shared/Layout/MainHubLayout.razor b/src/NexusReader.UI.Shared/Layout/MainHubLayout.razor index 7ba1639..70d012c 100644 --- a/src/NexusReader.UI.Shared/Layout/MainHubLayout.razor +++ b/src/NexusReader.UI.Shared/Layout/MainHubLayout.razor @@ -34,6 +34,12 @@ Concepts Map + +
+ +
+ Global AI Q&A +
diff --git a/src/NexusReader.UI.Shared/Pages/Intelligence.razor b/src/NexusReader.UI.Shared/Pages/Intelligence.razor new file mode 100644 index 0000000..9440b90 --- /dev/null +++ b/src/NexusReader.UI.Shared/Pages/Intelligence.razor @@ -0,0 +1,453 @@ +@page "/intelligence" +@attribute [Authorize] +@using NexusReader.Application.DTOs.AI +@using NexusReader.Application.Abstractions.Services +@using NexusReader.Application.DTOs.User +@using System.Net.Http.Json +@inject HttpClient Http +@inject IKnowledgeService KnowledgeService + +
+
+
+

Global AI Q&A

+

Search, interrogate, and extract grounded facts from your library using Polyglot KM-RAG

+
+
+ +
+
+
+ + +
+ +
+ + +
+
+ +
+ @if (_isLoading) + { +
+ + Analyzing conceptual graph and synthesizing response... +
+ } + else if (_response != null) + { +
+
+

Answer

+
+ @_response.Answer +
+
+ + @if (_response.Citations != null && _response.Citations.Any()) + { +
+

Grounded Citations

+
+ @foreach (var citation in _response.Citations) + { +
+
+ @citation.SourceBook + @if (!string.IsNullOrEmpty(citation.CitationId) && citation.CitationId.Length > 8) + { + ID: @citation.CitationId.Substring(0, Math.Min(8, citation.CitationId.Length)) + } +
+
+ "@citation.Snippet" +
+
+ } +
+
+ } +
+ } + else if (_hasSearched) + { +
+ +

No answers generated. Try adjusting your question.

+
+ } + else + { +
+
+ + + +
+

Start Interrogating Your Library

+

Ask complex questions across all your books. The system will search vectors, pull concept graph relations, and formulate a grounded answer with precise citations.

+
+ } +
+
+
+ + + +@code { + private string _question = string.Empty; + private string _selectedBookId = string.Empty; + private bool _isLoading; + private bool _hasSearched; + private GroundedResponseDto? _response; + private List? _books; + + protected override async Task OnInitializedAsync() + { + try + { + _books = await Http.GetFromJsonAsync>("api/library/books"); + } + catch (Exception ex) + { + Console.WriteLine($"[Intelligence] Failed to load books for scope selector: {ex.Message}"); + } + } + + private async Task HandleKeyUp(KeyboardEventArgs e) + { + if (e.Key == "Enter" && !string.IsNullOrWhiteSpace(_question) && !_isLoading) + { + await AskQuestionAsync(); + } + } + + private async Task AskQuestionAsync() + { + if (string.IsNullOrWhiteSpace(_question) || _isLoading) return; + + _isLoading = true; + _hasSearched = true; + _response = null; + StateHasChanged(); + + try + { + Guid? ebookId = null; + if (!string.IsNullOrEmpty(_selectedBookId) && Guid.TryParse(_selectedBookId, out var parsedId)) + { + ebookId = parsedId; + } + + var result = await KnowledgeService.AskQuestionAsync(_question, "tenantId", ebookId); + if (result.IsSuccess) + { + _response = result.Value; + } + else + { + _response = new GroundedResponseDto + { + Answer = $"Error: {result.Errors.FirstOrDefault()?.Message ?? "An error occurred."}", + Citations = new List() + }; + } + } + catch (Exception ex) + { + _response = new GroundedResponseDto + { + Answer = $"Network/API Error: {ex.Message}", + Citations = new List() + }; + } + finally + { + _isLoading = false; + StateHasChanged(); + } + } +} diff --git a/src/NexusReader.Web.Client/Services/WasmKnowledgeService.cs b/src/NexusReader.Web.Client/Services/WasmKnowledgeService.cs index 8ec3394..24fc3df 100644 --- a/src/NexusReader.Web.Client/Services/WasmKnowledgeService.cs +++ b/src/NexusReader.Web.Client/Services/WasmKnowledgeService.cs @@ -93,6 +93,26 @@ public class WasmKnowledgeService : IKnowledgeService } } + public async Task> AskQuestionAsync(string question, string tenantId, Guid? ebookId = null, int limit = 5, CancellationToken cancellationToken = default) + { + try + { + var response = await _httpClient.PostAsJsonAsync("/api/knowledge/ask", new { question, tenantId, ebookId, limit }, cancellationToken); + if (response.IsSuccessStatusCode) + { + var result = await response.Content.ReadFromJsonAsync(cancellationToken: cancellationToken); + return result != null ? Result.Ok(result) : Result.Fail("Failed to deserialize grounded response."); + } + + var errorBody = await response.Content.ReadAsStringAsync(cancellationToken); + return Result.Fail($"Server error ({response.StatusCode}): {errorBody}"); + } + catch (Exception ex) + { + return Result.Fail(new Error($"Network error: {ex.Message}").CausedBy(ex)); + } + } + private async Task> CallKnowledgeApiAsync(string endpoint, string text, Guid? ebookId, CancellationToken cancellationToken) { try diff --git a/src/NexusReader.Web/Program.cs b/src/NexusReader.Web/Program.cs index da82080..8123f92 100644 --- a/src/NexusReader.Web/Program.cs +++ b/src/NexusReader.Web/Program.cs @@ -302,6 +302,22 @@ knowledgeApi.MapPost("/verify-groundedness", async (GroundednessRequest request, return Results.BadRequest(result.Errors.Count > 0 ? result.Errors[0].Message : "Unknown server error"); }); +knowledgeApi.MapPost("/search", async (SemanticSearchRequest request, ClaimsPrincipal user, IKnowledgeService knowledgeService) => +{ + var tenantId = user.FindFirstValue("TenantId") ?? "global"; + var result = await knowledgeService.SearchLibrarySemanticallyAsync(request.QueryText, tenantId, request.Limit); + if (result.IsSuccess) return Results.Ok(result.Value); + return Results.BadRequest(result.Errors.Count > 0 ? result.Errors[0].Message : "Unknown server error"); +}); + +knowledgeApi.MapPost("/ask", async (AskQuestionRequest request, ClaimsPrincipal user, IKnowledgeService knowledgeService) => +{ + var tenantId = user.FindFirstValue("TenantId") ?? "global"; + var result = await knowledgeService.AskQuestionAsync(request.Question, tenantId, request.EbookId, request.Limit); + if (result.IsSuccess) return Results.Ok(result.Value); + return Results.BadRequest(result.Errors.Count > 0 ? result.Errors[0].Message : "Unknown server error"); +}); + knowledgeApi.MapDelete("/", async (IKnowledgeService knowledgeService) => { var result = await knowledgeService.ClearCacheAsync(); @@ -549,3 +565,5 @@ app.Run(); public record KnowledgeRequest(string Text, Guid? EbookId = null); public record GroundednessRequest(string Answer, string Context); +public record SemanticSearchRequest(string QueryText, int Limit = 5); +public record AskQuestionRequest(string Question, Guid? EbookId = null, int Limit = 5); diff --git a/tests/NexusReader.Application.Tests/Queries/QueryTests.cs b/tests/NexusReader.Application.Tests/Queries/QueryTests.cs index 0d3d7b0..178b198 100644 --- a/tests/NexusReader.Application.Tests/Queries/QueryTests.cs +++ b/tests/NexusReader.Application.Tests/Queries/QueryTests.cs @@ -148,6 +148,57 @@ public class QueryTests : IDisposable result.Value.First().ContentHash.Should().Be("hash-123"); } + [Fact] + public async Task AskLibraryQuestionQuery_WithEmptyQuestion_ReturnsFailure() + { + // Arrange + var knowledgeServiceMock = new Mock(); + var handler = new AskLibraryQuestionQueryHandler(knowledgeServiceMock.Object); + var query = new AskLibraryQuestionQuery("", "tenant-123"); + + // Act + var result = await handler.Handle(query, CancellationToken.None); + + // Assert + result.IsSuccess.Should().BeFalse(); + result.Errors.First().Message.Should().Be("Question cannot be empty."); + } + + [Fact] + public async Task AskLibraryQuestionQuery_WithValidQuestion_CallsKnowledgeService() + { + // Arrange + var knowledgeServiceMock = new Mock(); + var expectedResponse = new GroundedResponseDto + { + Answer = "Based on the book, water boils at 100 degrees Celsius.", + Citations = new List + { + new CitationDto + { + CitationId = "chunk-1", + Snippet = "Water boils at 100 degrees Celsius.", + SourceBook = "Physics 101" + } + } + }; + + knowledgeServiceMock.Setup(s => s.AskQuestionAsync("what temp does water boil?", "tenant-123", null, 5, It.IsAny())) + .ReturnsAsync(Result.Ok(expectedResponse)); + + var handler = new AskLibraryQuestionQueryHandler(knowledgeServiceMock.Object); + var query = new AskLibraryQuestionQuery("what temp does water boil?", "tenant-123"); + + // Act + var result = await handler.Handle(query, CancellationToken.None); + + // Assert + result.IsSuccess.Should().BeTrue(); + result.Value.Answer.Should().Be("Based on the book, water boils at 100 degrees Celsius."); + result.Value.Citations.Should().HaveCount(1); + result.Value.Citations.First().CitationId.Should().Be("chunk-1"); + } + public void Dispose() { _connection.Close(); -- 2.52.0