feat: KM-RAG Polyglot Ingestion Pipeline Migration (#46)
Resolves the KM-RAG Polyglot Persistence and Background Ingestion Pipeline Migration task. ### Key Changes 1. **Infrastructure Migration**: Integrated Qdrant (for vector embeddings) and Neo4j (for concept graphs), reducing reliance on PostgreSQL pgvector storage. 2. **Concurrent Background Job**: Implemented a robust Hangfire `EbookIngestionJob` utilizing Polly exponential retries for transient 429 rate limits, executing three core ingestion tasks concurrently via `Task.WhenAll`. 3. **Data Layer**: Standardized database schemas and entities; retained `Pgvector.EntityFrameworkCore` for migration compilation compatibility. 4. **Wasm Client & Tests**: Implemented client support for semantic search and refactored related tests in `QueryTests.cs` to mock `IKnowledgeService`. ### Verification Status - **Build**: Successfully compiles with `dotnet build NexusReader.slnx --no-restore` (0 errors). - **Tests**: All 5 unit tests pass cleanly with `dotnet test NexusReader.slnx --no-restore`. **Resolve** #47 --------- Co-authored-by: Marek Jasiński <jasins.marek@gmail.com> Reviewed-on: #46 Reviewed-by: Marek Jaisński <jasins.marek@gmail.com> Co-authored-by: Antigravity <antigravity@google.com> Co-committed-by: Antigravity <antigravity@google.com>
This commit was merged in pull request #46.
This commit is contained in:
@@ -1,6 +1,5 @@
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Pgvector.EntityFrameworkCore;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.AI;
|
||||
using GeminiDotnet;
|
||||
@@ -20,6 +19,10 @@ using NexusReader.Domain.Entities;
|
||||
using Microsoft.AspNetCore.Identity;
|
||||
using Microsoft.AspNetCore.Authorization;
|
||||
using NexusReader.Application.Security.Authorization;
|
||||
using Qdrant.Client;
|
||||
using Neo4j.Driver;
|
||||
using Hangfire;
|
||||
using Hangfire.PostgreSql;
|
||||
|
||||
namespace NexusReader.Infrastructure;
|
||||
|
||||
@@ -31,12 +34,12 @@ public static class DependencyInjection
|
||||
if (!string.IsNullOrEmpty(pgConnectionString))
|
||||
{
|
||||
services.AddDbContextFactory<AppDbContext>(options =>
|
||||
options.UseNpgsql(pgConnectionString, x => x.UseVector()),
|
||||
options.UseNpgsql(pgConnectionString),
|
||||
ServiceLifetime.Scoped);
|
||||
|
||||
// Also register a scoped DbContext for repositories that need it
|
||||
services.AddDbContext<AppDbContext>(options =>
|
||||
options.UseNpgsql(pgConnectionString, x => x.UseVector()));
|
||||
options.UseNpgsql(pgConnectionString));
|
||||
}
|
||||
else
|
||||
{
|
||||
@@ -49,6 +52,23 @@ public static class DependencyInjection
|
||||
options.UseSqlite(sqliteConnectionString));
|
||||
}
|
||||
|
||||
// Qdrant Client registration
|
||||
var qdrantUrl = configuration.GetConnectionString("QdrantConnection") ?? "http://localhost:6334";
|
||||
services.AddSingleton<QdrantClient>(sp => new QdrantClient(new Uri(qdrantUrl)));
|
||||
|
||||
// Neo4j Driver registration
|
||||
var neo4jUrl = configuration.GetConnectionString("Neo4jConnection") ?? "bolt://localhost:7687";
|
||||
services.AddSingleton<IDriver>(sp => GraphDatabase.Driver(neo4jUrl, AuthTokens.None));
|
||||
|
||||
// Hangfire registration
|
||||
if (!string.IsNullOrEmpty(pgConnectionString))
|
||||
{
|
||||
services.AddHangfire(config => config
|
||||
.UseRecommendedSerializerSettings()
|
||||
.UsePostgreSqlStorage(options => options.UseNpgsqlConnection(pgConnectionString)));
|
||||
services.AddHangfireServer();
|
||||
}
|
||||
|
||||
services.Configure<AiSettings>(configuration.GetSection(AiSettings.SectionName));
|
||||
services.Configure<StripeSettings>(configuration.GetSection(StripeSettings.SectionName));
|
||||
var aiSettings = configuration.GetSection(AiSettings.SectionName).Get<AiSettings>() ?? new AiSettings();
|
||||
|
||||
Reference in New Issue
Block a user