feat: implement AI-driven knowledge extraction service with semantic caching and persistent storage

src/NexusReader.Application/Abstractions/Services/IKnowledgeService.cs

@@ -0,0 +1,9 @@
+using FluentResults;
+using NexusReader.Application.DTOs.AI;
+
+namespace NexusReader.Application.Abstractions.Services;
+
+public interface IKnowledgeService
+{
+    Task<Result<KnowledgePacket>> GetKnowledgeAsync(string text, CancellationToken cancellationToken = default);
+}

src/NexusReader.Application/DTOs/AI/KnowledgePacket.cs

@@ -0,0 +1,19 @@
+using System.Text.Json.Serialization;
+
+namespace NexusReader.Application.DTOs.AI;
+
+public record KeyConcept(
+    [property: JsonPropertyName("title")] string Title,
+    [property: JsonPropertyName("description")] string Description
+);
+
+public record QuizQuestion(
+    [property: JsonPropertyName("question")] string Question,
+    [property: JsonPropertyName("options")] List<string> Options,
+    [property: JsonPropertyName("correct_index")] int CorrectIndex
+);
+
+public record KnowledgePacket(
+    [property: JsonPropertyName("concepts")] List<KeyConcept> Concepts,
+    [property: JsonPropertyName("quizzes")] List<QuizQuestion> Quizzes
+);

src/NexusReader.Domain/Entities/SemanticKnowledgeCache.cs

@@ -0,0 +1,23 @@
+using System.ComponentModel.DataAnnotations;
+
+namespace NexusReader.Domain.Entities;
+
+public class SemanticKnowledgeCache
+{
+    [Key]
+    [MaxLength(64)]
+    public string ContentHash { get; set; } = string.Empty;
+
+    [Required]
+    public string JsonData { get; set; } = string.Empty;
+
+    [Required]
+    [MaxLength(50)]
+    public string ModelId { get; set; } = "gemini-1.5-flash";
+
+    [Required]
+    [MaxLength(10)]
+    public string PromptVersion { get; set; } = "1.0";
+
+    public DateTime CreatedAt { get; set; } = DateTime.UtcNow;
+}

src/NexusReader.Infrastructure/DependencyInjection.cs

@@ -1,4 +1,10 @@
 using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Configuration;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.AI;
+using GeminiDotnet;
+using GeminiDotnet.Extensions.AI;
+using NexusReader.Infrastructure.Persistence;
 using NexusReader.Application.Abstractions.Services;
 using NexusReader.Infrastructure.Services;
 
@@ -6,8 +12,26 @@ namespace NexusReader.Infrastructure;
 
 public static class DependencyInjection
 {
-    public static IServiceCollection AddInfrastructure(this IServiceCollection services)
+    public static IServiceCollection AddInfrastructure(this IServiceCollection services, IConfiguration configuration)
     {
+        var connectionString = configuration.GetConnectionString("SqliteConnection") ?? "Data Source=nexus.db";
+        services.AddDbContext<AppDbContext>(options =>
+            options.UseSqlite(connectionString));
+
+        var apiKey = configuration["Ai:Google:ApiKey"];
+        if (string.IsNullOrWhiteSpace(apiKey))
+        {
+            throw new InvalidOperationException("AI Studio ApiKey is missing in configuration (Ai:Google:ApiKey).");
+        }
+        var modelId = configuration["Ai:Google:Model"] ?? "gemini-1.5-flash";
+
+        services.AddSingleton<IChatClient>(new GeminiChatClient(new GeminiClientOptions 
+        { 
+            ApiKey = apiKey, 
+            ModelId = modelId 
+        }));
+
+        services.AddScoped<IKnowledgeService, KnowledgeService>();
         services.AddTransient<IAiGenerateQuizService, FakeAiGenerateQuizService>();
         services.AddTransient<IEpubService, EpubService>();
         return services;

src/NexusReader.Infrastructure/Helpers/ContentHasher.cs

@@ -0,0 +1,37 @@
+using System.Security.Cryptography;
+using System.Text;
+using System.Text.RegularExpressions;
+
+namespace NexusReader.Infrastructure.Helpers;
+
+public static class ContentHasher
+{
+    public static string ComputeHash(string input)
+    {
+        if (string.IsNullOrWhiteSpace(input))
+        {
+            return string.Empty;
+        }
+
+        var normalizedInput = Normalize(input);
+        
+        var inputBytes = Encoding.UTF8.GetBytes(normalizedInput);
+        var hashBytes = SHA256.HashData(inputBytes);
+        
+        return Convert.ToHexString(hashBytes).ToLowerInvariant();
+    }
+
+    public static string Normalize(string input)
+    {
+        if (string.IsNullOrWhiteSpace(input))
+        {
+            return string.Empty;
+        }
+
+        // Trim and collapse all consecutive whitespace characters into a single space
+        var normalized = Regex.Replace(input.Trim(), @"\s+", " ");
+        
+        // Convert to lower-case as AI analysis is generally not case-sensitive for concepts/quizzes
+        return normalized.ToLowerInvariant();
+    }
+}

src/NexusReader.Infrastructure/NexusReader.Infrastructure.csproj

@@ -5,6 +5,15 @@
   </ItemGroup>
 
   <ItemGroup>
+    <PackageReference Include="GeminiDotnet.Extensions.AI" Version="0.23.0" />
+    <PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="10.0.7">
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+      <PrivateAssets>all</PrivateAssets>
+    </PackageReference>
+    <PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="10.0.7" />
+    <PackageReference Include="Microsoft.Extensions.AI" Version="10.5.0" />
+    <PackageReference Include="Polly" Version="8.6.6" />
+    <PackageReference Include="Polly.Extensions.Http" Version="3.0.0" />
     <PackageReference Include="VersOne.Epub" Version="3.3.6" />
   </ItemGroup>
 

src/NexusReader.Infrastructure/Persistence/AppDbContext.cs

@@ -0,0 +1,24 @@
+using Microsoft.EntityFrameworkCore;
+using NexusReader.Domain.Entities;
+
+namespace NexusReader.Infrastructure.Persistence;
+
+public class AppDbContext : DbContext
+{
+    public AppDbContext(DbContextOptions<AppDbContext> options) : base(options)
+    {
+    }
+
+    public DbSet<SemanticKnowledgeCache> SemanticKnowledgeCache => Set<SemanticKnowledgeCache>();
+
+    protected override void OnModelCreating(ModelBuilder modelBuilder)
+    {
+        base.OnModelCreating(modelBuilder);
+
+        modelBuilder.Entity<SemanticKnowledgeCache>(entity =>
+        {
+            entity.HasKey(e => e.ContentHash);
+            entity.HasIndex(e => e.ContentHash).IsUnique();
+        });
+    }
+}

src/NexusReader.Infrastructure/Services/KnowledgeService.cs

@@ -0,0 +1,145 @@
+using System.Text.Json;
+using FluentResults;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.AI;
+using NexusReader.Application.Abstractions.Services;
+using NexusReader.Application.DTOs.AI;
+using NexusReader.Domain.Entities;
+using NexusReader.Infrastructure.Helpers;
+using NexusReader.Infrastructure.Persistence;
+using Polly;
+using Polly.Retry;
+
+namespace NexusReader.Infrastructure.Services;
+
+public class KnowledgeService : IKnowledgeService
+{
+    private readonly IChatClient _chatClient;
+    private readonly AppDbContext _dbContext;
+    private const string PromptVersion = "1.0";
+    private const string ModelId = "gemini-1.5-flash";
+
+    private static readonly ResiliencePipeline _retryPipeline = new ResiliencePipelineBuilder()
+        .AddRetry(new RetryStrategyOptions
+        {
+            ShouldHandle = new PredicateBuilder().Handle<Exception>(ex => 
+                ex.Message.Contains("429") || ex.Message.Contains("Too Many Requests") || ex.Message.Contains("quota")),
+            BackoffType = DelayBackoffType.Exponential,
+            UseJitter = true,
+            MaxRetryAttempts = 3,
+            Delay = TimeSpan.FromSeconds(2)
+        })
+        .Build();
+
+    public KnowledgeService(IChatClient chatClient, AppDbContext dbContext)
+    {
+        _chatClient = chatClient;
+        _dbContext = dbContext;
+    }
+
+    public async Task<Result<KnowledgePacket>> GetKnowledgeAsync(string text, CancellationToken cancellationToken = default)
+    {
+        if (string.IsNullOrWhiteSpace(text))
+        {
+            return Result.Fail("Input text is empty.");
+        }
+
+        // Normalize text to ensure consistent hashing and reduce token noise
+        var normalizedText = ContentHasher.Normalize(text);
+
+        // Phase 4: Request Pre-processing (Token Saving)
+        const int MaxInputLength = 15000; // Roughly 3k-4k tokens
+        if (normalizedText.Length > MaxInputLength)
+        {
+            return Result.Fail($"Input text is too long ({normalizedText.Length} characters after normalization). Max allowed is {MaxInputLength}.");
+        }
+
+        // Simple token estimation (4 chars per token)
+        var estimatedTokens = normalizedText.Length / 4;
+        Console.WriteLine($"[KnowledgeService] Processing request with ~{estimatedTokens} tokens.");
+
+        var hash = ContentHasher.ComputeHash(normalizedText);
+
+        // 1. Check Cache
+        var cached = await _dbContext.SemanticKnowledgeCache
+            .FirstOrDefaultAsync(c => c.ContentHash == hash && c.PromptVersion == PromptVersion, cancellationToken);
+
+        if (cached != null)
+        {
+            try
+            {
+                var packet = JsonSerializer.Deserialize<KnowledgePacket>(cached.JsonData);
+                if (packet != null)
+                {
+                    return Result.Ok(packet);
+                }
+            }
+            catch (JsonException)
+            {
+                // If deserialization fails, we proceed to call the AI
+            }
+        }
+
+        // 2. Call AI Client
+        try
+        {
+            var options = new ChatOptions
+            {
+                ResponseFormat = ChatResponseFormat.Json,
+                Temperature = 0.1f,
+                MaxOutputTokens = 1000
+            };
+
+            var response = await _retryPipeline.ExecuteAsync(async ct => 
+                await _chatClient.GetResponseAsync(new List<ChatMessage>
+                {
+                    new ChatMessage(ChatRole.System, PromptRegistry.KnowledgeExtractionSystemPrompt),
+                    new ChatMessage(ChatRole.User, normalizedText)
+                }, options, cancellationToken: ct), cancellationToken);
+
+            var jsonResponse = response.Text;
+            if (string.IsNullOrWhiteSpace(jsonResponse))
+            {
+                return Result.Fail("AI returned an empty response.");
+            }
+
+            // Cleanup potential markdown if Gemini still adds it despite options
+            jsonResponse = jsonResponse.Replace("```json", "").Replace("```", "").Trim();
+
+            var knowledgePacket = JsonSerializer.Deserialize<KnowledgePacket>(jsonResponse);
+            if (knowledgePacket == null)
+            {
+                return Result.Fail("Failed to deserialize AI response.");
+            }
+
+            // 3. Save to Cache
+            var cacheEntry = new SemanticKnowledgeCache
+            {
+                ContentHash = hash,
+                JsonData = jsonResponse,
+                ModelId = ModelId,
+                PromptVersion = PromptVersion,
+                CreatedAt = DateTime.UtcNow
+            };
+
+            // Handle potential race condition if multiple requests for same text arrive
+            if (cached == null)
+            {
+                _dbContext.SemanticKnowledgeCache.Add(cacheEntry);
+            }
+            else
+            {
+                cached.JsonData = jsonResponse;
+                cached.CreatedAt = DateTime.UtcNow;
+            }
+
+            await _dbContext.SaveChangesAsync(cancellationToken);
+
+            return Result.Ok(knowledgePacket);
+        }
+        catch (Exception ex)
+        {
+            return Result.Fail(new Error("Failed to extract knowledge from AI").CausedBy(ex));
+        }
+    }
+}

src/NexusReader.Infrastructure/Services/PromptRegistry.cs

@@ -0,0 +1,9 @@
+namespace NexusReader.Infrastructure.Services;
+
+public static class PromptRegistry
+{
+    public const string KnowledgeExtractionSystemPrompt = 
+        "You are an expert educator. Analyze the provided text to extract key concepts and generate relevant quizzes. " +
+        "Return ONLY a minified JSON object that strictly adheres to the provided schema. No markdown formatting, no explanations. " +
+        "Schema: { \"concepts\": [ { \"title\": \"string\", \"description\": \"string\" } ], \"quizzes\": [ { \"question\": \"string\", \"options\": [ \"string\" ], \"correct_index\": 0 } ] }.";
+}

src/NexusReader.Web.New/Program.cs

@@ -26,7 +26,7 @@ builder.Services.AddScoped<IFocusModeService, FocusModeService>();
 builder.Services.AddScoped<IReaderNavigationService, ReaderNavigationService>();
 
 builder.Services.AddApplication();
-builder.Services.AddInfrastructure();
+builder.Services.AddInfrastructure(builder.Configuration);
 
 var app = builder.Build();
 

src/NexusReader.Web.New/appsettings.json

@@ -5,5 +5,14 @@
       "Microsoft.AspNetCore": "Warning"
     }
   },
-  "AllowedHosts": "*"
+  "AllowedHosts": "*",
+  "ConnectionStrings": {
+    "SqliteConnection": "Data Source=nexus.db"
+  },
+  "Ai": {
+    "Google": {
+      "ApiKey": "PLACEHOLDER",
+      "Model": "gemini-1.5-flash"
+    }
+  }
 }