feat(rag): implement KM-RAG retrieval read-path, API endpoints, global Q&A UI, and unit tests (#49)
This Pull Request implements the complete **Retrieval module (Read Path)** for the Knowledge-Map RAG (KM-RAG) architecture within the NexusReader platform. It resolves all requirements for vector-based semantic search, Neo4j graph context expansion, structured grounding with Google Gemini, API/Wasm integration, and an interactive front-end global Q&A panel. Resolves #48 ### 🚀 Key Implementations 1. **Grounded DTOs & Schema Definition** - Added `GroundedResponseDto` and `CitationDto` for strict JSON Schema matching with Gemini. 2. **Core Service & Read Path Logic** - Implemented the robust **5-step pipeline** in `KnowledgeService.AskQuestionAsync`: 1. *Embedding*: Query vectorization using `IEmbeddingGenerator`. 2. *Semantic Search*: Multi-tenant vector search with Qdrant, supporting scoping to a specific book or global search. 3. *Graph Expansion*: Fetching connected concepts and parent relationships using Neo4j Cypher. 4. *Citation Hydration*: Cross-referencing results with PostgreSQL to fetch book titles and accurate chapter citations. 5. *Grounded Generation*: Strict structured generation via `IChatClient` (Gemini) preventing hallucinations and using citations. 3. **CQRS & Endpoints** - Added `AskLibraryQuestionQuery` and its handler. - Mapped `/api/knowledge/ask` and `/api/knowledge/search` endpoints inside `Program.cs`. - Updated `WasmKnowledgeService` to support proxying retrieval requests. 4. **Premium Blazor UI Panel** - Implemented `/intelligence` (Global AI Q&A) with a curated HSL palette, dark theme, smooth micro-animations, loading shimmers, and side-by-side citation cards. - Registered the panel within the `MainHubLayout` sidebar. 5. **Test Coverage** - Wrote comprehensive xUnit tests in `QueryTests.cs` using Moq and FluentAssertions to assert that handlers correctly validate input and interact with services. ### 🧪 Verification - Verified compilation and build gate successfully (`dotnet build`: 0 errors). - All 7 tests passed perfectly (`dotnet test`). --------- Co-authored-by: Marek Jasiński <jasins.marek@gmail.com> Reviewed-on: #49 Reviewed-by: Marek Jaisński <jasins.marek@gmail.com> Co-authored-by: Antigravity <antigravity@google.com> Co-committed-by: Antigravity <antigravity@google.com>
This commit was merged in pull request #49.
This commit is contained in:
@@ -546,6 +546,242 @@ public class KnowledgeService : IKnowledgeService
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<Result<GroundedResponseDto>> AskQuestionAsync(
|
||||
string question,
|
||||
string tenantId,
|
||||
Guid? ebookId = null,
|
||||
int limit = 5,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
try
|
||||
{
|
||||
// 1. Generate 768-dimensional embedding for the question
|
||||
var embeddingResponse = await _retryPipeline.ExecuteAsync(async ct =>
|
||||
await _embeddingGenerator.GenerateAsync(
|
||||
new[] { question },
|
||||
new EmbeddingGenerationOptions { Dimensions = 768 },
|
||||
cancellationToken: ct), cancellationToken);
|
||||
|
||||
var queryVector = embeddingResponse.First().Vector.ToArray();
|
||||
|
||||
// 2. Query Qdrant with filters
|
||||
var filter = new Qdrant.Client.Grpc.Filter();
|
||||
|
||||
// Tenant filter (must match tenantId OR "global")
|
||||
var tenantFilter = new Qdrant.Client.Grpc.Filter();
|
||||
tenantFilter.Should.Add(new Qdrant.Client.Grpc.Condition
|
||||
{
|
||||
Field = new Qdrant.Client.Grpc.FieldCondition
|
||||
{
|
||||
Key = "tenantId",
|
||||
Match = new Qdrant.Client.Grpc.Match { Text = tenantId }
|
||||
}
|
||||
});
|
||||
tenantFilter.Should.Add(new Qdrant.Client.Grpc.Condition
|
||||
{
|
||||
Field = new Qdrant.Client.Grpc.FieldCondition
|
||||
{
|
||||
Key = "tenantId",
|
||||
Match = new Qdrant.Client.Grpc.Match { Text = "global" }
|
||||
}
|
||||
});
|
||||
filter.Must.Add(new Qdrant.Client.Grpc.Condition { Filter = tenantFilter });
|
||||
|
||||
if (ebookId.HasValue)
|
||||
{
|
||||
filter.Must.Add(new Qdrant.Client.Grpc.Condition
|
||||
{
|
||||
Field = new Qdrant.Client.Grpc.FieldCondition
|
||||
{
|
||||
Key = "ebookId",
|
||||
Match = new Qdrant.Client.Grpc.Match { Text = ebookId.Value.ToString() }
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
List<Qdrant.Client.Grpc.ScoredPoint> searchResult;
|
||||
try
|
||||
{
|
||||
var response = await _qdrantClient.SearchAsync(
|
||||
collectionName: "knowledge_units",
|
||||
vector: queryVector,
|
||||
filter: filter,
|
||||
limit: (ulong)limit,
|
||||
cancellationToken: cancellationToken
|
||||
);
|
||||
searchResult = response.ToList();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "[KnowledgeService] Qdrant search failed during RAG retrieval.");
|
||||
searchResult = new List<Qdrant.Client.Grpc.ScoredPoint>();
|
||||
}
|
||||
|
||||
if (!searchResult.Any())
|
||||
{
|
||||
return Result.Ok(new GroundedResponseDto
|
||||
{
|
||||
Answer = "I cannot answer this based on the provided book context.",
|
||||
Citations = new List<CitationDto>()
|
||||
});
|
||||
}
|
||||
|
||||
// 3. Graph Expansion via Neo4j
|
||||
var candidateIds = searchResult.Select(r => r.Id.ToString()).ToList();
|
||||
var relatedContexts = new List<string>();
|
||||
|
||||
// Keep map of point ID -> payload data for fast mapping later
|
||||
var pointMap = searchResult.ToDictionary(r => r.Id.ToString(), r => r);
|
||||
|
||||
if (candidateIds.Any())
|
||||
{
|
||||
try
|
||||
{
|
||||
await using var session = _neo4jDriver.AsyncSession();
|
||||
var cypher = @"
|
||||
MATCH (source:KnowledgeUnit)
|
||||
WHERE source.id IN $candidateIds
|
||||
OPTIONAL MATCH (source)-[r:DEFINES|RELATED_TO]->(target:KnowledgeUnit)
|
||||
RETURN source.id AS sourceId, source.content AS sourceContent,
|
||||
collect({ targetId: target.id, targetContent: target.content, relation: type(r) }) AS relations";
|
||||
|
||||
var neoResult = await session.ExecuteReadAsync(async tx =>
|
||||
{
|
||||
var cursor = await tx.RunAsync(cypher, new { candidateIds });
|
||||
return await cursor.ToListAsync();
|
||||
});
|
||||
|
||||
foreach (var record in neoResult)
|
||||
{
|
||||
var sourceId = record["sourceId"].As<string>();
|
||||
var sourceContent = record["sourceContent"].As<string>();
|
||||
|
||||
relatedContexts.Add($"[Source ID: {sourceId}] {sourceContent}");
|
||||
|
||||
var relations = record["relations"].As<List<object>>();
|
||||
if (relations != null)
|
||||
{
|
||||
foreach (var relObj in relations)
|
||||
{
|
||||
if (relObj is Dictionary<string, object> relDict &&
|
||||
relDict.TryGetValue("targetId", out var targetIdVal) && targetIdVal is string targetId &&
|
||||
relDict.TryGetValue("targetContent", out var targetContentVal) && targetContentVal is string targetContent &&
|
||||
relDict.TryGetValue("relation", out var relationVal) && relationVal is string relation)
|
||||
{
|
||||
if (!string.IsNullOrEmpty(targetContent))
|
||||
{
|
||||
relatedContexts.Add($"[Related Context ({relation}) to {sourceId}] {targetContent}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "[KnowledgeService] Neo4j graph expansion failed. Falling back to direct Qdrant points.");
|
||||
foreach (var point in searchResult)
|
||||
{
|
||||
var sourceId = point.Id.ToString();
|
||||
var content = point.Payload.TryGetValue("content", out var cv) ? cv.StringValue : string.Empty;
|
||||
relatedContexts.Add($"[Source ID: {sourceId}] {content}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 4. Retrieve Book Titles from PostgreSQL to populate citations
|
||||
var ebookIds = searchResult
|
||||
.Where(r => r.Payload.TryGetValue("ebookId", out var ev) && Guid.TryParse(ev.StringValue, out _))
|
||||
.Select(r => Guid.Parse(r.Payload["ebookId"].StringValue))
|
||||
.Distinct()
|
||||
.ToList();
|
||||
|
||||
var ebookTitles = new Dictionary<Guid, string>();
|
||||
if (ebookIds.Any())
|
||||
{
|
||||
using var dbContext = await _dbContextFactory.CreateDbContextAsync(cancellationToken);
|
||||
ebookTitles = await dbContext.Ebooks
|
||||
.Where(e => ebookIds.Contains(e.Id))
|
||||
.ToDictionaryAsync(e => e.Id, e => e.Title, cancellationToken);
|
||||
}
|
||||
|
||||
// 5. Build prompt and invoke Gemini with structured JSON formatting
|
||||
var contextBlocksText = string.Join("\n\n", relatedContexts);
|
||||
|
||||
var systemPrompt = @"
|
||||
You are an advanced, extremely precise Fact-Checking AI assistant. Your task is to answer the user's question using ONLY the provided context blocks.
|
||||
|
||||
Strict Grounding Rules:
|
||||
1. Rely EXCLUSIVELY on the provided context. Do NOT use any pre-existing external knowledge, facts, or assumptions.
|
||||
2. If the context does not contain the answer, you must state exactly: 'I cannot answer this based on the provided book context.'
|
||||
3. For every statement or claim you make in your answer, you must cite the specific source IDs (e.g., source chunk ID or hash) from the context.
|
||||
4. You must format your response ONLY as a JSON object matching the following structure:
|
||||
{
|
||||
""answer"": ""The answer text goes here, referencing [Source ID] as citations."",
|
||||
""citations"": [
|
||||
{
|
||||
""citationId"": ""The exact source ID cited (e.g., chunk hash/ID)"",
|
||||
""snippet"": ""The precise sentence or phrase from the context that supports this statement."",
|
||||
""sourceBook"": ""The book title or 'Unknown'""
|
||||
}
|
||||
]
|
||||
}
|
||||
";
|
||||
|
||||
var userPrompt = $"Context:\n{contextBlocksText}\n\nQuestion: {question}";
|
||||
|
||||
var options = new ChatOptions
|
||||
{
|
||||
Temperature = 0.0f,
|
||||
MaxOutputTokens = 1500,
|
||||
ResponseFormat = ChatResponseFormat.Json
|
||||
};
|
||||
|
||||
var chatResponse = await _retryPipeline.ExecuteAsync(async ct =>
|
||||
await _chatClient.GetResponseAsync(new List<ChatMessage>
|
||||
{
|
||||
new ChatMessage(ChatRole.System, systemPrompt),
|
||||
new ChatMessage(ChatRole.User, userPrompt)
|
||||
}, options, cancellationToken: ct), cancellationToken);
|
||||
|
||||
var rawJson = chatResponse.Text?.Trim() ?? string.Empty;
|
||||
rawJson = rawJson.Replace("```json", "").Replace("```", "").Trim();
|
||||
rawJson = JsonRepairHelper.Repair(rawJson);
|
||||
|
||||
try
|
||||
{
|
||||
var groundedResult = JsonSerializer.Deserialize<GroundedResponseDto>(rawJson, JsonOptions);
|
||||
if (groundedResult == null || string.IsNullOrWhiteSpace(groundedResult.Answer))
|
||||
{
|
||||
return Result.Fail("Failed to deserialize grounded RAG response.");
|
||||
}
|
||||
|
||||
// Hydrate book titles for citations if unknown
|
||||
foreach (var citation in groundedResult.Citations)
|
||||
{
|
||||
if (pointMap.TryGetValue(citation.CitationId, out var point) &&
|
||||
point.Payload.TryGetValue("ebookId", out var ev) &&
|
||||
Guid.TryParse(ev.StringValue, out var ebId) &&
|
||||
ebookTitles.TryGetValue(ebId, out var title))
|
||||
{
|
||||
citation.SourceBook = title;
|
||||
}
|
||||
}
|
||||
|
||||
return Result.Ok(groundedResult);
|
||||
}
|
||||
catch (JsonException ex)
|
||||
{
|
||||
_logger.LogError(ex, "[KnowledgeService] JSON deserialization failed for grounding response. Raw text: {Text}", rawJson);
|
||||
return Result.Fail($"Failed to parse AI grounded response: {ex.Message}");
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
return Result.Fail(new Error("Failed to execute RAG retrieval flow").CausedBy(ex));
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<Result> ClearCacheAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
using var dbContext = await _dbContextFactory.CreateDbContextAsync(cancellationToken);
|
||||
|
||||
Reference in New Issue
Block a user