From 1c6ee82d01aeb8baa6eabd71c6eba9aecac183f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Jasi=C5=84ski?= Date: Mon, 25 May 2026 08:51:21 +0200 Subject: [PATCH] feat: implement background ebook indexing with progress tracking and real-time UI updates --- .../Abstractions/Services/IEpubExtractor.cs | 17 ++ .../Library/IngestEbookCommandHandler.cs | 21 +- .../Commands/Library/ProcessEbookCommand.cs | 177 ++++++++++++++++ .../DTOs/User/UserProfileDto.cs | 1 + .../Queries/Graph/GraphViewModels.cs | 23 ++- .../Queries/Library/GetMyEbooksQuery.cs | 3 +- .../User/GetUserProfileQueryHandler.cs | 3 +- .../DependencyInjection.cs | 1 + .../Services/EpubExtractor.cs | 85 ++++++++ .../Services/KnowledgeService.cs | 2 +- .../Services/PromptRegistry.cs | 30 +-- .../Organisms/BookIngestionModal.razor | 88 +++++++- .../Organisms/BookIngestionModal.razor.css | 66 ++++++ .../Components/Organisms/ReaderCanvas.razor | 1 + .../Services/ISyncService.cs | 1 + .../Services/KnowledgeCoordinator.cs | 4 + .../Services/SyncService.cs | 6 + .../wwwroot/js/knowledgeGraph.js | 189 +++++++++++++++--- src/NexusReader.Web.Client/Program.cs | 7 + src/NexusReader.Web/Program.cs | 50 ++++- 20 files changed, 718 insertions(+), 57 deletions(-) create mode 100644 src/NexusReader.Application/Abstractions/Services/IEpubExtractor.cs create mode 100644 src/NexusReader.Application/Commands/Library/ProcessEbookCommand.cs create mode 100644 src/NexusReader.Infrastructure/Services/EpubExtractor.cs diff --git a/src/NexusReader.Application/Abstractions/Services/IEpubExtractor.cs b/src/NexusReader.Application/Abstractions/Services/IEpubExtractor.cs new file mode 100644 index 0000000..e5199d4 --- /dev/null +++ b/src/NexusReader.Application/Abstractions/Services/IEpubExtractor.cs @@ -0,0 +1,17 @@ +using FluentResults; + +namespace NexusReader.Application.Abstractions.Services; + +/// +/// Service abstraction to extract raw text content from EPUB chapters. +/// +public interface IEpubExtractor +{ + /// + /// Extracts the sanitized, plain-text content of each chapter in the EPUB file. + /// + /// The relative storage path of the EPUB file. + /// Cancellation token. + /// A list of plain-text chapters, or a failure result. + Task>> ExtractChaptersTextAsync(string relativePath, CancellationToken cancellationToken = default); +} diff --git a/src/NexusReader.Application/Commands/Library/IngestEbookCommandHandler.cs b/src/NexusReader.Application/Commands/Library/IngestEbookCommandHandler.cs index 0ae9e21..ca53adc 100644 --- a/src/NexusReader.Application/Commands/Library/IngestEbookCommandHandler.cs +++ b/src/NexusReader.Application/Commands/Library/IngestEbookCommandHandler.cs @@ -1,5 +1,6 @@ using FluentResults; using MediatR; +using Microsoft.Extensions.DependencyInjection; using NexusReader.Application.Abstractions.Messaging; using NexusReader.Application.Abstractions.Persistence; using NexusReader.Application.Abstractions.Services; @@ -11,13 +12,16 @@ public class IngestEbookCommandHandler : IRequestHandler> Handle(IngestEbookCommand request, CancellationToken cancellationToken) @@ -72,6 +76,21 @@ public class IngestEbookCommandHandler : IRequestHandler + { + try + { + using var scope = _scopeFactory.CreateScope(); + var mediator = scope.ServiceProvider.GetRequiredService(); + await mediator.Send(new ProcessEbookCommand(ebook.Id, request.UserId, request.TenantId)); + } + catch (Exception) + { + // Swallowed to prevent ThreadPool crashes + } + }); + return Result.Ok(ebook.Id); } catch (Exception ex) diff --git a/src/NexusReader.Application/Commands/Library/ProcessEbookCommand.cs b/src/NexusReader.Application/Commands/Library/ProcessEbookCommand.cs new file mode 100644 index 0000000..5a1e9c4 --- /dev/null +++ b/src/NexusReader.Application/Commands/Library/ProcessEbookCommand.cs @@ -0,0 +1,177 @@ +using System.Text.RegularExpressions; +using FluentResults; +using MediatR; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using NexusReader.Application.Abstractions.Messaging; +using NexusReader.Application.Abstractions.Services; +using NexusReader.Data.Persistence; + +namespace NexusReader.Application.Commands.Library; + +public record ProcessEbookCommand( + Guid EbookId, + string UserId, + string TenantId +) : ICommand; + +public class ProcessEbookCommandHandler : IRequestHandler> +{ + private readonly IDbContextFactory _dbContextFactory; + private readonly IKnowledgeService _knowledgeService; + private readonly IEpubExtractor _epubExtractor; + private readonly ISyncBroadcaster _broadcaster; + private readonly ILogger _logger; + + public ProcessEbookCommandHandler( + IDbContextFactory dbContextFactory, + IKnowledgeService knowledgeService, + IEpubExtractor epubExtractor, + ISyncBroadcaster broadcaster, + ILogger logger) + { + _dbContextFactory = dbContextFactory; + _knowledgeService = knowledgeService; + _epubExtractor = epubExtractor; + _broadcaster = broadcaster; + _logger = logger; + } + + public async Task> Handle(ProcessEbookCommand request, CancellationToken cancellationToken) + { + _logger.LogInformation("[ProcessEbook] Starting background processing for Ebook: {EbookId}", request.EbookId); + + try + { + await _broadcaster.BroadcastIngestionProgressAsync(request.UserId, "Wyszukiwanie e-booka w bazie danych...", 0.05, cancellationToken); + + using var dbContext = await _dbContextFactory.CreateDbContextAsync(cancellationToken); + var ebook = await dbContext.Ebooks.FindAsync(new object[] { request.EbookId }, cancellationToken); + if (ebook == null) + { + _logger.LogError("[ProcessEbook] Ebook not found in database: {EbookId}", request.EbookId); + return Result.Fail($"Ebook nie znaleziony w bazie danych: {request.EbookId}"); + } + + _logger.LogInformation("[ProcessEbook] Extracting chapters text for Ebook: {Title} ({FilePath})", ebook.Title, ebook.FilePath); + await _broadcaster.BroadcastIngestionProgressAsync(request.UserId, "Otwieranie i parsowanie pliku EPUB...", 0.1, cancellationToken); + + var extractionResult = await _epubExtractor.ExtractChaptersTextAsync(ebook.FilePath, cancellationToken); + if (extractionResult.IsFailed) + { + var errorMsg = extractionResult.Errors.FirstOrDefault()?.Message ?? "Failed to extract text chapters."; + _logger.LogError("[ProcessEbook] Extraction failed: {Error}", errorMsg); + return Result.Fail(extractionResult.Errors); + } + + var chapters = extractionResult.Value; + if (chapters == null || !chapters.Any()) + { + _logger.LogWarning("[ProcessEbook] EPUB has no readable content files: {EbookId}", request.EbookId); + return Result.Fail("EPUB nie zawiera czytelnych rozdziałów."); + } + + int totalChapters = chapters.Count; + _logger.LogInformation("[ProcessEbook] Processing {Count} chapters for Ebook: {Title}", totalChapters, ebook.Title); + + await _broadcaster.BroadcastIngestionProgressAsync(request.UserId, $"Analizowanie struktury ({totalChapters} rozdziałów)...", 0.15, cancellationToken); + + int processedChapters = 0; + + for (int i = 0; i < totalChapters; i++) + { + var cleanText = chapters[i]; + + if (cleanText.Length < 100) + { + _logger.LogInformation("[ProcessEbook] Skipping chapter {Index} (text too short: {Length} chars)", i, cleanText.Length); + processedChapters++; + continue; + } + + // Chunk the text to maintain granular Knowledge Units + var chunks = ChunkText(cleanText, 3000); + _logger.LogInformation("[ProcessEbook] Chapter {Index} split into {ChunkCount} chunk(s)", i, chunks.Count); + + foreach (var chunk in chunks) + { + try + { + // Invoke GetKnowledgeMapAsync to extract, embed, and upsert knowledge units + var result = await _knowledgeService.GetKnowledgeMapAsync(chunk, request.TenantId, request.EbookId, cancellationToken); + if (result.IsFailed) + { + _logger.LogWarning("[ProcessEbook] Failed to generate knowledge map for a chunk of chapter {Index}: {Error}", i, result.Errors.FirstOrDefault()?.Message); + } + } + catch (Exception ex) + { + _logger.LogError(ex, "[ProcessEbook] Exception during AI vectorization of chapter {Index} chunk", i); + } + } + + processedChapters++; + double progress = 0.15 + (0.75 * processedChapters / totalChapters); + await _broadcaster.BroadcastIngestionProgressAsync( + request.UserId, + $"Przetwarzanie rozdziału {processedChapters} z {totalChapters} przez AI...", + progress, + cancellationToken); + } + + // Mark the ebook as ready + ebook.IsReadyForReading = true; + await dbContext.SaveChangesAsync(cancellationToken); + + _logger.LogInformation("[ProcessEbook] Ingestion and vector indexing completed for: {Title}", ebook.Title); + + await _broadcaster.BroadcastIngestionProgressAsync( + request.UserId, + "Indeksowanie wektorowe e-booka przez Nexus AI zakończone pomyślnie!", + 1.0, + cancellationToken); + + return Result.Ok(true); + } + catch (Exception ex) + { + _logger.LogError(ex, "[ProcessEbook] Critical error during background EPUB vectorization of ebook {EbookId}", request.EbookId); + await _broadcaster.BroadcastIngestionProgressAsync( + request.UserId, + $"Błąd indeksowania: {ex.Message}", + 1.0, + cancellationToken); + return Result.Fail(new Error("Wystąpił błąd podczas indeksowania e-booka przez AI").CausedBy(ex)); + } + } + + private static List ChunkText(string text, int maxWords = 3000) + { + var words = text.Split(' ', StringSplitOptions.RemoveEmptyEntries); + var chunks = new List(); + if (words.Length <= maxWords) + { + chunks.Add(text); + return chunks; + } + var currentChunk = new List(); + int count = 0; + foreach (var word in words) + { + currentChunk.Add(word); + count++; + if (count >= maxWords) + { + chunks.Add(string.Join(" ", currentChunk)); + currentChunk.Clear(); + count = 0; + } + } + if (currentChunk.Any()) + { + chunks.Add(string.Join(" ", currentChunk)); + } + return chunks; + } +} diff --git a/src/NexusReader.Application/DTOs/User/UserProfileDto.cs b/src/NexusReader.Application/DTOs/User/UserProfileDto.cs index 27a0850..212a591 100644 --- a/src/NexusReader.Application/DTOs/User/UserProfileDto.cs +++ b/src/NexusReader.Application/DTOs/User/UserProfileDto.cs @@ -38,4 +38,5 @@ public record LastReadBookDto public string? LastChapter { get; init; } public int LastChapterIndex { get; init; } public string? Description { get; init; } + public bool IsReadyForReading { get; init; } } diff --git a/src/NexusReader.Application/Queries/Graph/GraphViewModels.cs b/src/NexusReader.Application/Queries/Graph/GraphViewModels.cs index 19d81e4..27b9ed6 100644 --- a/src/NexusReader.Application/Queries/Graph/GraphViewModels.cs +++ b/src/NexusReader.Application/Queries/Graph/GraphViewModels.cs @@ -1,9 +1,24 @@ +using System.Text.Json.Serialization; + namespace NexusReader.Application.Queries.Graph; -public record GraphNodeDto(string Id, string Label, string Group, string? Type = null); -public record GraphLinkDto(string Source, string Target, string RelationType, int Value = 1); +public record GraphNodeDto( + [property: JsonPropertyName("id")] string Id, + [property: JsonPropertyName("label")] string Label, + [property: JsonPropertyName("group")] string Group, + [property: JsonPropertyName("description")] string? Description = null, + [property: JsonPropertyName("type")] string? Type = null +); + +public record GraphLinkDto( + [property: JsonPropertyName("source")] string Source, + [property: JsonPropertyName("target")] string Target, + [property: JsonPropertyName("type")] string RelationType, + [property: JsonPropertyName("value")] int Value = 1 +); + public record GraphDataDto { - public List Nodes { get; init; } = new(); - public List Links { get; init; } = new(); + [JsonPropertyName("nodes")] public List Nodes { get; init; } = new(); + [JsonPropertyName("links")] public List Links { get; init; } = new(); } diff --git a/src/NexusReader.Application/Queries/Library/GetMyEbooksQuery.cs b/src/NexusReader.Application/Queries/Library/GetMyEbooksQuery.cs index d3eef7e..712d8a9 100644 --- a/src/NexusReader.Application/Queries/Library/GetMyEbooksQuery.cs +++ b/src/NexusReader.Application/Queries/Library/GetMyEbooksQuery.cs @@ -37,7 +37,8 @@ public class GetMyEbooksQueryHandler : IRequestHandler ur.UserId == u.Id) diff --git a/src/NexusReader.Infrastructure/DependencyInjection.cs b/src/NexusReader.Infrastructure/DependencyInjection.cs index 93ebd7f..6bc61ab 100644 --- a/src/NexusReader.Infrastructure/DependencyInjection.cs +++ b/src/NexusReader.Infrastructure/DependencyInjection.cs @@ -112,6 +112,7 @@ public static class DependencyInjection services.AddScoped(); services.AddTransient(); services.AddTransient(); + services.AddTransient(); // Fix #4: Scoped instead of Singleton — BookStorageService uses path resolution // that is environment-specific and incompatible with Singleton lifetime in MAUI. diff --git a/src/NexusReader.Infrastructure/Services/EpubExtractor.cs b/src/NexusReader.Infrastructure/Services/EpubExtractor.cs new file mode 100644 index 0000000..81f0d42 --- /dev/null +++ b/src/NexusReader.Infrastructure/Services/EpubExtractor.cs @@ -0,0 +1,85 @@ +using System.Text.RegularExpressions; +using FluentResults; +using Microsoft.Extensions.Logging; +using NexusReader.Application.Abstractions.Services; +using VersOne.Epub; + +namespace NexusReader.Infrastructure.Services; + +public class EpubExtractor : IEpubExtractor +{ + private readonly ILogger _logger; + + public EpubExtractor(ILogger logger) + { + _logger = logger; + } + + public async Task>> ExtractChaptersTextAsync(string relativePath, CancellationToken cancellationToken = default) + { + try + { + var fullPath = ResolvePath(relativePath); + if (string.IsNullOrEmpty(fullPath) || !File.Exists(fullPath)) + { + _logger.LogError("[EpubExtractor] EPUB file not found at path: {FilePath}", relativePath); + return Result.Fail>($"Plik EPUB nie został znaleziony na dysku: {relativePath}"); + } + + using var bookRef = await EpubReader.OpenBookAsync(fullPath); + var readingOrder = bookRef.GetReadingOrder(); + + if (readingOrder == null || !readingOrder.Any()) + { + return Result.Fail>("EPUB nie zawiera czytelnych rozdziałów."); + } + + var chapters = new List(); + foreach (var chapterRef in readingOrder) + { + if (cancellationToken.IsCancellationRequested) + { + break; + } + + var rawContent = await chapterRef.ReadContentAsTextAsync(); + var cleanText = StripHtml(rawContent); + chapters.Add(cleanText); + } + + return Result.Ok(chapters); + } + catch (Exception ex) + { + _logger.LogError(ex, "[EpubExtractor] Error extracting chapters from EPUB: {FilePath}", relativePath); + return Result.Fail>(new Error("Failed to parse and extract text from EPUB").CausedBy(ex)); + } + } + + private static string? ResolvePath(string relativePath) + { + var normalized = relativePath.Replace('/', Path.DirectorySeparatorChar); + var currentDir = new DirectoryInfo(AppDomain.CurrentDomain.BaseDirectory); + while (currentDir != null) + { + var candidate = Path.Combine(currentDir.FullName, "wwwroot", normalized); + if (File.Exists(candidate)) return candidate; + + var devCandidate = Path.Combine(currentDir.FullName, "src", "NexusReader.Web", "wwwroot", normalized); + if (File.Exists(devCandidate)) return devCandidate; + + currentDir = currentDir.Parent; + } + return null; + } + + private static string StripHtml(string html) + { + if (string.IsNullOrEmpty(html)) return string.Empty; + var clean = Regex.Replace(html, @"<(style|script)\b[^>]*>.*?", "", RegexOptions.IgnoreCase | RegexOptions.Singleline); + clean = Regex.Replace(clean, @"<[^>]*>", " "); + clean = System.Net.WebUtility.HtmlDecode(clean); + clean = Regex.Replace(clean, @"\s+", " ").Trim(); + return clean; + } +} diff --git a/src/NexusReader.Infrastructure/Services/KnowledgeService.cs b/src/NexusReader.Infrastructure/Services/KnowledgeService.cs index 5868f7b..4f84f5c 100644 --- a/src/NexusReader.Infrastructure/Services/KnowledgeService.cs +++ b/src/NexusReader.Infrastructure/Services/KnowledgeService.cs @@ -33,7 +33,7 @@ public class KnowledgeService : IKnowledgeService private readonly ILogger _logger; private readonly QdrantClient _qdrantClient; private readonly IDriver _neo4jDriver; - private const string PromptVersion = "1.3"; + private const string PromptVersion = "1.6"; private static readonly ConcurrentDictionary>>> _activeRequests = new(); public KnowledgeService( diff --git a/src/NexusReader.Infrastructure/Services/PromptRegistry.cs b/src/NexusReader.Infrastructure/Services/PromptRegistry.cs index f456e61..bb64433 100644 --- a/src/NexusReader.Infrastructure/Services/PromptRegistry.cs +++ b/src/NexusReader.Infrastructure/Services/PromptRegistry.cs @@ -4,9 +4,10 @@ public static class PromptRegistry { public const string KnowledgeExtractionSystemPrompt = "You are an expert educator. Analyze the provided text to extract key concepts, generate relevant quizzes, and construct a knowledge graph. " + - "CRITICAL: Restrict 'concept.label' to a maximum of 3 words (e.g., 'Dependency Injection' instead of full sentences). " + + "**LANGUAGE CRITICAL**: Detect the language of the provided text. You MUST generate all human-readable fields ('title', 'description', 'question', 'options', 'label') in the EXACT SAME LANGUAGE as the source text. Do NOT translate them to English unless the source text is in English. " + + "CRITICAL: Restrict 'concept.label' to a maximum of 3 words (e.g., 'Dependency Injection' or its exact foreign equivalent, never full sentences). " + "CRITICAL: Extract a MAXIMUM of 15 key concepts/plot points from the text. " + - "CRITICAL: Code blocks (e.g., markdown code snippets) must be excluded from the relationship graph, or summarized as a single node (e.g., 'Code Example'). Do NOT create nodes for variables, functions, namespaces, or individual lines of code. " + + "CRITICAL: Code blocks (e.g., markdown code snippets) must be excluded from the relationship graph, or summarized as a single node with the label 'Code Example' translated to the detected language. Do NOT create nodes for variables, functions, namespaces, or individual lines of code. " + "CRITICAL: Return ONLY a minified JSON object. Do NOT include markdown formatting like ```json or ```. Do NOT include explanations. " + "Schema: { " + "\"concepts\": [ { \"title\": \"string\", \"description\": \"string\" } ], " + @@ -15,26 +16,31 @@ public static class PromptRegistry "}."; public const string GraphExtractionPrompt = - "You are an expert at information architecture. Extract key concepts and paragraph mappings from the text to build a unified knowledge graph. " + + "You are an expert at information architecture. Extract a highly strategic, clean, and educational knowledge graph from the provided technical text to act as a clear structural roadmap, avoiding clutter or hyper-connected noise hubs. " + + "**LANGUAGE CRITICAL**: Detect the language of the provided text. The 'label' and 'description' fields MUST be generated in the EXACT SAME LANGUAGE as the source text. Do NOT translate them to English. " + "The input text consists of several paragraphs, each starting with its unique block ID in the format '[ID: seg-X]'. " + - "Extract two types of nodes: " + - "1. Concept Nodes (group: 'concept'): Extract the main technical concepts discussed (e.g., ID: 'dependency-injection', label: 'Dependency Injection'). Max 10 concepts. Labels must be at most 3 words. " + - "2. Block Nodes (group: 'current'): For each paragraph in the input, create a node representing that paragraph where 'id' is the exact block ID (e.g., 'seg-1'), and 'label' is a brief summary of that paragraph's content (max 3 words). " + - "CRITICAL: If a paragraph is a code block, represent it as a single block node with label 'Code Example' (group: 'current'). Do NOT extract low-level code elements (like variables, classes, methods, or namespaces) as separate concept nodes. " + - "CRITICAL: Connect related concept nodes together, and connect each concept node to the block nodes ('seg-X') where it is discussed. " + - "Limit connections to a MAXIMUM of 15 most relevant links. " + - "Return ONLY minified JSON. Schema: { \"graph\": { \"nodes\": [ { \"id\": \"string\", \"label\": \"string\", \"group\": \"concept|current\" } ], \"links\": [ { \"source\": \"string\", \"target\": \"string\", \"value\": 1 } ] } }"; - + "Extract three distinct types of nodes based on strict hierarchical validation: " + + "1. Concept Nodes (group: 'concept'): Extract major global architectural pillars discussed. Max 6 per segment. Labels must be 1-3 words max. " + + "2. Bridge Nodes (group: 'bridge'): If the text directly compares a legacy paradigm (e.g., Desktop/WPF) to a modern framework alternative (.NET 10/Blazor), extract them as paired concepts to visually bridge the structural evolution. " + + "3. Block Nodes (group: 'current'): Create a node ONLY for significant structural landmarks in the text (e.g., major headings). Do NOT connect every concept to every individual paragraph wrapper. Connect concepts only to the main section block where they are anchored. " + + "CRITICAL NOISE SUPPRESSION: Absolutely forbid creating separate nodes for individual configuration files, files names, simple classes, servers, or methods (e.g., 'appsettings.json', 'Kestrel', 'Thread.Sleep', 'OnInitializedAsync'). These low-level details MUST be collapsed and described only within the 'description' field of their parent concept node. " + + "CRITICAL: Code blocks must be completely ignored as separate nodes; represent them only as contextual attributes within descriptions. " + + "Limit topology connections to a MAXIMUM of 10 highly relevant links total per segment. " + + "System keys configuration: 'group' must be strictly 'concept', 'bridge', 'current', 'rule', 'definition', 'table', or 'section'. " + + "Return ONLY minified JSON. Schema: { \"graph\": { \"nodes\": [ { \"id\": \"string\", \"label\": \"string\", \"group\": \"concept|bridge|current\", \"description\": \"string\" } ], \"links\": [ { \"source\": \"string\", \"target\": \"string\", \"type\": \"maps_to|contains|relates_to\" } ] } }"; public const string SummaryAndQuizPrompt = "You are an expert educator. Provide a concise summary of the text and generate a challenging quiz (3-5 questions). " + + "**LANGUAGE CRITICAL**: Detect the language of the provided text. The generated 'summary', 'question', and 'options' MUST be in the EXACT SAME LANGUAGE as the source text. " + "Return ONLY minified JSON. Schema: { \"summary\": \"string\", \"quizzes\": [ { \"question\": \"string\", \"options\": [ \"string\" ], \"correct_index\": 0 } ] }"; public const string KM_ExtractionPrompt = "You are an expert at Knowledge Engineering. Segment the provided text into discrete Knowledge Units. " + + "**LANGUAGE CRITICAL**: Detect the language of the provided text. The 'content' field MUST be in the EXACT SAME LANGUAGE as the source text. " + "Identify 'units' (sections, tables, definitions, rules) and 'links' (how they relate). " + "CRITICAL: Units must be granular. " + - "CRITICAL: Code blocks must be summarized under the parent unit or represented as a single 'Code Example' unit. Do NOT segment code blocks into granular low-level code details (e.g., classes, variables, parameters). " + + "CRITICAL: Code blocks must be summarized under the parent unit or represented as a single 'Code Example' unit (translate the name to the detected language). Do NOT segment code blocks into granular low-level code details (e.g., classes, variables, parameters). " + + "CRITICAL SYSTEM VALUES: The fields 'type' (strictly: 'Section', 'Table', 'Definition', or 'Rule') and 'relation' (strictly: 'Next', 'Defines', 'Contains', or 'References') are system keys and MUST remain in English as specified. " + "Schema: { " + "\"units\": [ { \"id\": \"string\", \"type\": \"Section|Table|Definition|Rule\", \"content\": \"string\", \"metadata\": { \"page\": 0 } } ], " + "\"links\": [ { \"source\": \"string\", \"target\": \"string\", \"relation\": \"Next|Defines|Contains|References\" } ] " + diff --git a/src/NexusReader.UI.Shared/Components/Organisms/BookIngestionModal.razor b/src/NexusReader.UI.Shared/Components/Organisms/BookIngestionModal.razor index 950da9a..4af21f7 100644 --- a/src/NexusReader.UI.Shared/Components/Organisms/BookIngestionModal.razor +++ b/src/NexusReader.UI.Shared/Components/Organisms/BookIngestionModal.razor @@ -2,12 +2,14 @@ @using NexusReader.Application.Abstractions.Services @using NexusReader.Application.Queries.Reader @using NexusReader.Application.Commands.Library +@using NexusReader.UI.Shared.Services @using System.Net.Http.Json @inject IEpubMetadataExtractor MetadataExtractor @inject ILogger Logger @inject HttpClient Http @inject IReaderNavigationService ReaderNavigation @inject IJSRuntime JSRuntime +@inject ISyncService SyncService @implements IAsyncDisposable @if (IsOpen) @@ -16,20 +18,23 @@

Add New Book

- + @if (!IsIngesting && !IsIndexing) + { + + }
-
+

Scanning metadata...

-
+
@if (Metadata != null) {
@@ -74,7 +79,7 @@
@@ -87,6 +92,18 @@
+
+
+ +

Nexus AI Indexing

+

@IngestionStatusMessage

+
+
+
+ @((IngestionProgressPercent * 100).ToString("F0"))% +
+
+ @if (!string.IsNullOrEmpty(ErrorMessage)) { @@ -118,6 +135,10 @@ private bool IsParsing { get; set; } private bool IsVerifying { get; set; } private bool IsIngesting { get; set; } + private bool IsIndexing { get; set; } + private string IngestionStatusMessage { get; set; } = "Initializing..."; + private double IngestionProgressPercent { get; set; } + private Guid IngestedBookId { get; set; } = Guid.Empty; private LocalEpubMetadata? Metadata { get; set; } private string? ErrorMessage { get; set; } private byte[]? _epubBytes; @@ -125,8 +146,42 @@ // Allow up to 50 MB private const long MaxFileSize = 50 * 1024 * 1024; + protected override async Task OnInitializedAsync() + { + await SyncService.InitializeAsync(); + SyncService.OnIngestionProgressReceived += HandleIngestionProgress; + } + + private async Task HandleIngestionProgress(string message, double progress) + { + if (!IsIndexing) return; + + IngestionStatusMessage = message; + IngestionProgressPercent = progress; + + await InvokeAsync(StateHasChanged); + + if (progress >= 1.0) + { + // Give the user a moment to see the completion message + await Task.Delay(2500); + + // Now close the modal and navigate to the book + if (IngestedBookId != Guid.Empty) + { + var bookId = IngestedBookId; + await InvokeAsync(async () => { + await CloseModal(); + ReaderNavigation.NavigateToBook(bookId); + }); + } + } + } + private async Task CloseModal() { + if (IsIngesting || IsIndexing) return; + IsOpen = false; Reset(); await IsOpenChanged.InvokeAsync(false); @@ -137,6 +192,10 @@ IsParsing = false; IsVerifying = false; IsIngesting = false; + IsIndexing = false; + IngestionStatusMessage = "Initializing..."; + IngestionProgressPercent = 0.0; + IngestedBookId = Guid.Empty; Metadata = null; ErrorMessage = null; _isDragging = false; @@ -220,33 +279,40 @@ var result = await response.Content.ReadFromJsonAsync(); if (result != null) { - await CloseModal(); - ReaderNavigation.NavigateToBook(result.Id); + IngestedBookId = result.Id; + IsVerifying = false; + IsIngesting = false; + IsIndexing = true; + IngestionStatusMessage = "Book saved! Starting background indexing..."; + IngestionProgressPercent = 0.0; + StateHasChanged(); } } else { ErrorMessage = await response.Content.ReadAsStringAsync(); + IsIngesting = false; } } catch (Exception ex) { Logger.LogError(ex, "Error during ingestion"); ErrorMessage = "Failed to save book to library. Please try again."; + IsIngesting = false; } finally { - IsIngesting = false; StateHasChanged(); } } private record IngestResult(Guid Id); - public ValueTask DisposeAsync() + public async ValueTask DisposeAsync() { + SyncService.OnIngestionProgressReceived -= HandleIngestionProgress; // Clear the large byte array so it is eligible for GC even if the component is cached. _epubBytes = null; - return ValueTask.CompletedTask; + await ValueTask.CompletedTask; } } diff --git a/src/NexusReader.UI.Shared/Components/Organisms/BookIngestionModal.razor.css b/src/NexusReader.UI.Shared/Components/Organisms/BookIngestionModal.razor.css index 64f412f..519639b 100644 --- a/src/NexusReader.UI.Shared/Components/Organisms/BookIngestionModal.razor.css +++ b/src/NexusReader.UI.Shared/Components/Organisms/BookIngestionModal.razor.css @@ -377,6 +377,72 @@ animation: spin 0.8s linear infinite; } +/* Indexing State */ +.indexing-state { + flex: 1; + display: flex; + justify-content: center; + align-items: center; + border-radius: 12px; + background: rgba(255, 255, 255, 0.02); + border: 1px solid rgba(255, 255, 255, 0.05); + box-shadow: inset 0 0 12px rgba(255, 255, 255, 0.02); + position: relative; + overflow: hidden; + padding: 2rem; + animation: fadeIn 0.4s ease-out; +} + +.indexing-content { + display: flex; + flex-direction: column; + align-items: center; + text-align: center; + width: 100%; + gap: 1.25rem; +} + +.indexing-content h3 { + margin: 0; + font-size: 1.25rem; + color: var(--nexus-neon, #00ffaa); + text-shadow: 0 0 10px rgba(0, 255, 153, 0.2); + letter-spacing: 0.5px; +} + +.status-msg { + margin: 0; + font-size: 0.9rem; + color: var(--nexus-text-muted, #888); + min-height: 2.5rem; + line-height: 1.4; +} + +.progress-bar-container { + width: 100%; + height: 8px; + background: rgba(255, 255, 255, 0.05); + border-radius: 4px; + overflow: hidden; + position: relative; + border: 1px solid rgba(255, 255, 255, 0.05); +} + +.progress-bar-fill { + height: 100%; + background: linear-gradient(90deg, var(--nexus-neon, #00ffaa) 0%, #00b3ff 100%); + box-shadow: 0 0 10px rgba(0, 255, 153, 0.4); + border-radius: 4px; + transition: width 0.4s cubic-bezier(0.4, 0, 0.2, 1); +} + +.percent { + font-family: var(--nexus-font-mono, monospace); + font-size: 1.1rem; + font-weight: 700; + color: var(--nexus-text); +} + @keyframes fadeIn { from { opacity: 0; } to { opacity: 1; } diff --git a/src/NexusReader.UI.Shared/Components/Organisms/ReaderCanvas.razor b/src/NexusReader.UI.Shared/Components/Organisms/ReaderCanvas.razor index 4e622c2..c34e6fd 100644 --- a/src/NexusReader.UI.Shared/Components/Organisms/ReaderCanvas.razor +++ b/src/NexusReader.UI.Shared/Components/Organisms/ReaderCanvas.razor @@ -211,6 +211,7 @@ private async Task LoadChapterAsync(int index) { + await Coordinator.ClearAsync(); _isLoadingChapter = true; StatusMessage = "Wczytywanie treści..."; StateHasChanged(); diff --git a/src/NexusReader.UI.Shared/Services/ISyncService.cs b/src/NexusReader.UI.Shared/Services/ISyncService.cs index adcec95..8fce9aa 100644 --- a/src/NexusReader.UI.Shared/Services/ISyncService.cs +++ b/src/NexusReader.UI.Shared/Services/ISyncService.cs @@ -7,5 +7,6 @@ public interface ISyncService Task InitializeAsync(); Task UpdateProgressAsync(string pageId, Guid ebookId, double progress, string? chapterTitle, int chapterIndex); event Func OnProgressReceived; + event Func? OnIngestionProgressReceived; Task DisposeAsync(); } diff --git a/src/NexusReader.UI.Shared/Services/KnowledgeCoordinator.cs b/src/NexusReader.UI.Shared/Services/KnowledgeCoordinator.cs index 7121ba5..619cd4e 100644 --- a/src/NexusReader.UI.Shared/Services/KnowledgeCoordinator.cs +++ b/src/NexusReader.UI.Shared/Services/KnowledgeCoordinator.cs @@ -94,11 +94,15 @@ public sealed partial class KnowledgeCoordinator : IDisposable if (OnGraphUpdated != null) await OnGraphUpdated.Invoke(packet.Graph); await _platformService.VibrateSuccessAsync(); + return; } } + + await _graphService.SetLoading(false); } catch (Exception ex) { + await _graphService.SetLoading(false); LogGraphError(ex, tenantId); } } diff --git a/src/NexusReader.UI.Shared/Services/SyncService.cs b/src/NexusReader.UI.Shared/Services/SyncService.cs index 16c986f..214d7ef 100644 --- a/src/NexusReader.UI.Shared/Services/SyncService.cs +++ b/src/NexusReader.UI.Shared/Services/SyncService.cs @@ -16,6 +16,7 @@ public class SyncService : ISyncService, IAsyncDisposable private CancellationTokenSource? _debounceCts; public event Func? OnProgressReceived; + public event Func? OnIngestionProgressReceived; public SyncService( HttpClient httpClient, @@ -53,6 +54,11 @@ public class SyncService : ISyncService, IAsyncDisposable if (OnProgressReceived != null) await OnProgressReceived(pageId, timestamp); }); + _hubConnection.On("IngestionProgress", async (message, progress) => + { + if (OnIngestionProgressReceived != null) await OnIngestionProgressReceived(message, progress); + }); + try { await _hubConnection.StartAsync(); diff --git a/src/NexusReader.UI.Shared/wwwroot/js/knowledgeGraph.js b/src/NexusReader.UI.Shared/wwwroot/js/knowledgeGraph.js index f83f487..7356915 100644 --- a/src/NexusReader.UI.Shared/wwwroot/js/knowledgeGraph.js +++ b/src/NexusReader.UI.Shared/wwwroot/js/knowledgeGraph.js @@ -3,6 +3,110 @@ import * as d3 from 'https://esm.sh/d3@7'; const getDisplayLabel = d => d.label.length > 20 ? d.label.substring(0, 17) + "..." : d.label; const getPillWidth = d => getDisplayLabel(d).length * 8 + 30; +const getNodeType = d => { + if (d) { + if (d.type) { + const t = d.type.toLowerCase(); + if (t === 'definition') return 'definition'; + if (t === 'table') return 'table'; + if (t === 'rule') return 'rule'; + if (t === 'section') return 'section'; + } + if (d.group) { + const g = d.group.toLowerCase(); + if (g === 'definition') return 'definition'; + if (g === 'table') return 'table'; + if (g === 'rule') return 'rule'; + if (g === 'section') return 'section'; + } + } + return null; +}; + +const getNodeGroup = d => { + if (d && d.group) { + const g = d.group.toLowerCase(); + if (g === 'bridge') return 'bridge'; + if (g === 'current') return 'current'; + if (g === 'concept') return 'concept'; + } + return 'concept'; // fallback +}; + +const getCategoryStyle = d => { + const type = getNodeType(d); + const group = getNodeGroup(d); + + // 1. Rule (red/coral) + if (type === 'rule') { + return { + color: '#ff4646', + fill: 'rgba(255, 70, 70, 0.1)', + opacity: 0.8, + glowKey: 'rule', + textColor: '#ff8b8b' + }; + } + // 2. Definition (gold/amber) + if (type === 'definition') { + return { + color: '#ffb03a', + fill: 'rgba(255, 176, 58, 0.1)', + opacity: 0.8, + glowKey: 'definition', + textColor: '#ffd18c' + }; + } + // 3. Table (purple/magenta) + if (type === 'table') { + return { + color: '#d946ef', + fill: 'rgba(217, 70, 239, 0.1)', + opacity: 0.8, + glowKey: 'table', + textColor: '#f5d0fe' + }; + } + // 4. Section (blue/indigo) + if (type === 'section') { + return { + color: '#3b82f6', + fill: 'rgba(59, 130, 246, 0.1)', + opacity: 0.8, + glowKey: 'section', + textColor: '#93c5fd' + }; + } + // 5. Bridge (cyan/comparison) + if (group === 'bridge') { + return { + color: '#06b6d4', + fill: 'rgba(6, 182, 212, 0.1)', + opacity: 0.7, + glowKey: 'bridge', + textColor: '#67e8f9' + }; + } + // 6. Current (active/focus landmark - neon green) + if (group === 'current') { + return { + color: 'var(--nexus-neon)', + fill: 'rgba(0, 255, 153, 0.15)', + opacity: 0.9, + glowKey: 'current', + textColor: '#ffffff' + }; + } + // 7. Concept / Default (subtle cool steel blue/teal) + return { + color: '#00d2c4', + fill: 'rgba(0, 210, 196, 0.05)', + opacity: 0.4, + glowKey: 'concept', + textColor: '#e0e0e0' + }; +}; + let simulation; let zoomBehavior; let svgElement; @@ -24,8 +128,10 @@ export function mount(containerId, data, dotNetHelper) { .attr("height", "100%") .style("background", "radial-gradient(circle, #1a1a1a 0%, #121212 100%)"); - // Radial gradient for Nebula effect + // Radial gradients for Nebula effects const defs = svgElement.append("defs"); + + // Fallback radial gradient for legacy nebulaGlow const radialGradient = defs.append("radialGradient") .attr("id", "nebulaGlow") .attr("cx", "50%") @@ -34,6 +140,26 @@ export function mount(containerId, data, dotNetHelper) { radialGradient.append("stop").attr("offset", "0%").attr("stop-color", "var(--nexus-neon)").attr("stop-opacity", 1); radialGradient.append("stop").attr("offset", "100%").attr("stop-color", "var(--nexus-neon)").attr("stop-opacity", 0); + const colors = { + 'rule': '#ff4646', + 'definition': '#ffb03a', + 'table': '#d946ef', + 'section': '#3b82f6', + 'bridge': '#06b6d4', + 'current': 'var(--nexus-neon)', + 'concept': '#00d2c4' + }; + + Object.entries(colors).forEach(([key, color]) => { + const radGrad = defs.append("radialGradient") + .attr("id", `nebulaGlow-${key}`) + .attr("cx", "50%") + .attr("cy", "50%") + .attr("r", "50%"); + radGrad.append("stop").attr("offset", "0%").attr("stop-color", color).attr("stop-opacity", 1); + radGrad.append("stop").attr("offset", "100%").attr("stop-color", color).attr("stop-opacity", 0); + }); + // Root Group for Zoom rootGroup = svgElement.append("g").attr("class", "zoom-containment"); @@ -135,21 +261,33 @@ export function updateData(data) { } }); + // Sanitize links to filter out any references to non-existent nodes + const nodeIds = new Set(data.nodes.map(n => n.id)); + const validLinks = (data.links || []).filter(l => { + const srcId = typeof l.source === 'object' ? l.source.id : l.source; + const tgtId = typeof l.target === 'object' ? l.target.id : l.target; + return nodeIds.has(srcId) && nodeIds.has(tgtId); + }); + // Update Links link = rootGroup.select(".links-layer") .selectAll("path") - .data(data.links, d => d.source + "-" + d.target + "-" + d.relationType) + .data(validLinks, d => { + const srcId = typeof d.source === 'object' ? d.source.id : d.source; + const tgtId = typeof d.target === 'object' ? d.target.id : d.target; + return srcId + "-" + tgtId + "-" + d.type; + }) .join( enter => enter.append("path") .attr("stroke", d => { - if (d.relationType === 'Defines') return 'var(--nexus-accent)'; - if (d.relationType === 'Next') return 'rgba(255,255,255,0.2)'; - if (d.relationType === 'Contains') return 'var(--nexus-neon)'; + if (d.type === 'Defines' || d.type === 'maps_to') return 'var(--nexus-accent, #00ffaa)'; + if (d.type === 'Next' || d.type === 'relates_to') return 'rgba(255,255,255,0.2)'; + if (d.type === 'Contains' || d.type === 'contains') return 'var(--nexus-neon)'; return 'rgba(255,255,255,0.1)'; }) .attr("fill", "none") - .attr("stroke-width", d => d.relationType === 'Defines' ? 2 : 1) - .attr("stroke-dasharray", d => d.relationType === 'References' ? "5,5" : "0") + .attr("stroke-width", d => (d.type === 'Defines' || d.type === 'maps_to') ? 2 : 1) + .attr("stroke-dasharray", d => d.type === 'References' ? "5,5" : "0") .style("opacity", 0) .call(enter => enter.transition().duration(500).style("opacity", 1)), update => update, @@ -174,13 +312,8 @@ export function updateData(data) { g.append("circle") .attr("r", 30) - .attr("fill", d => { - if (d.type === 'Definition') return 'var(--nexus-accent)'; - if (d.type === 'Table') return 'var(--nexus-neon)'; - if (d.type === 'Rule') return '#ff4444'; - return "url(#nebulaGlow)"; - }) - .attr("opacity", d => d.group === 'current' ? 0.6 : 0.2); + .attr("fill", d => `url(#nebulaGlow-${getCategoryStyle(d).glowKey})`) + .attr("opacity", d => getCategoryStyle(d).opacity); g.append("rect") .attr("class", "node-pill") @@ -189,23 +322,20 @@ export function updateData(data) { .attr("width", d => getPillWidth(d)) .attr("height", 30) .attr("rx", 15) - .attr("fill", "rgba(20, 20, 20, 0.9)") - .attr("stroke", d => { - if (d.type === 'Definition') return 'var(--nexus-accent)'; - if (d.type === 'Rule') return '#ff4444'; - return "rgba(255, 255, 255, 0.1)"; - }) - .attr("stroke-width", 1); + .attr("fill", "rgba(20, 20, 20, 0.95)") + .attr("stroke", d => getCategoryStyle(d).color) + .attr("stroke-width", d => getNodeGroup(d) === 'current' ? 2 : 1.2); g.append("text") .text(d => getDisplayLabel(d)) .attr("text-anchor", "middle") .attr("y", 5) - .attr("fill", d => d.type === 'Definition' ? 'var(--nexus-accent)' : '#ccc') - .attr("font-size", "0.8rem"); + .attr("fill", d => getCategoryStyle(d).textColor) + .attr("font-size", "0.8rem") + .attr("font-weight", d => getNodeGroup(d) === 'current' ? '600' : 'normal'); g.append("title") - .text(d => d.label); + .text(d => d.description ? `${d.label}\n\n${d.description}` : d.label); g.transition().duration(500).style("opacity", 1); @@ -216,7 +346,7 @@ export function updateData(data) { ); simulation.nodes(data.nodes); - simulation.force("link").links(data.links); + simulation.force("link").links(validLinks); simulation.alpha(0.5).restart(); // Trigger zoom to fit after a short delay to allow simulation to settle @@ -398,6 +528,15 @@ export function clear() { } simulation.nodes([]); } + + // Reset selections + link = null; + node = null; + + // Reset D3 zoom transform to clean identity state + if (svgElement && zoomBehavior) { + svgElement.call(zoomBehavior.transform, d3.zoomIdentity); + } } catch (e) { console.warn("Failed to clear force simulation safely:", e); } diff --git a/src/NexusReader.Web.Client/Program.cs b/src/NexusReader.Web.Client/Program.cs index dcc3569..bf431b7 100644 --- a/src/NexusReader.Web.Client/Program.cs +++ b/src/NexusReader.Web.Client/Program.cs @@ -51,6 +51,7 @@ builder.Services.AddSingleton>>(new builder.Services.AddSingleton(new ThrowingBookStorageService()); builder.Services.AddSingleton(new ThrowingEbookRepository()); builder.Services.AddSingleton(new ThrowingSyncBroadcaster()); +builder.Services.AddSingleton(new ThrowingEpubExtractor()); builder.Services.AddApplication(); builder.Services.AddScoped(); @@ -99,3 +100,9 @@ public class ThrowingSyncBroadcaster : ISyncBroadcaster public Task BroadcastIngestionProgressAsync(string userId, string message, double progress, CancellationToken cancellationToken = default) => throw new NotSupportedException("Real-time broadcasting can only be performed by the server."); } + +public class ThrowingEpubExtractor : IEpubExtractor +{ + public Task>> ExtractChaptersTextAsync(string relativePath, CancellationToken cancellationToken = default) + => throw new NotSupportedException("EPUB text extraction is not supported in the WASM client."); +} diff --git a/src/NexusReader.Web/Program.cs b/src/NexusReader.Web/Program.cs index 8123f92..56230f7 100644 --- a/src/NexusReader.Web/Program.cs +++ b/src/NexusReader.Web/Program.cs @@ -194,6 +194,7 @@ using (var scope = app.Services.CreateScope()) await dbContext.Database.MigrateAsync(); await DbInitializer.SeedAsync(services); + await TriggerBackgroundProcessingForUnindexedBooksAsync(services); if (logger.IsEnabled(LogLevel.Information)) { @@ -337,13 +338,16 @@ app.MapPost("/api/library/ingest", async ([FromBody] IngestEbookRequest request, ? Convert.FromBase64String(request.CoverImageBase64) : null; + var tenantId = user.FindFirst("TenantId")?.Value ?? "global"; + var command = new IngestEbookCommand( request.Title, request.AuthorName, coverData, epubData, request.Description, - userId + userId, + tenantId ); var result = await mediator.Send(command); @@ -563,6 +567,50 @@ app.MapRazorComponents() app.Run(); +async Task TriggerBackgroundProcessingForUnindexedBooksAsync(IServiceProvider services) +{ + var logger = services.GetRequiredService>(); + try + { + var dbContextFactory = services.GetRequiredService>(); + using var dbContext = await dbContextFactory.CreateDbContextAsync(); + + var unindexedEbooks = await dbContext.Ebooks + .Where(e => !e.IsReadyForReading) + .ToListAsync(); + + if (unindexedEbooks.Any()) + { + logger.LogInformation("[Startup] Found {Count} un-indexed ebooks. Triggering background processing...", unindexedEbooks.Count); + + foreach (var ebook in unindexedEbooks) + { + logger.LogInformation("[Startup] Queuing background processing for ebook: '{Title}' ({Id})", ebook.Title, ebook.Id); + + _ = Task.Run(async () => + { + try + { + using var scope = services.CreateScope(); + var scopedMediator = scope.ServiceProvider.GetRequiredService(); + await scopedMediator.Send(new ProcessEbookCommand(ebook.Id, ebook.UserId, ebook.TenantId)); + } + catch (Exception ex) + { + using var scope = services.CreateScope(); + var scopedLogger = scope.ServiceProvider.GetRequiredService>(); + scopedLogger.LogError(ex, "Failed to run background processing for ebook {EbookId} on startup", ebook.Id); + } + }); + } + } + } + catch (Exception ex) + { + logger.LogError(ex, "Error checking or triggering background processing for unindexed books on startup."); + } +} + public record KnowledgeRequest(string Text, Guid? EbookId = null); public record GroundednessRequest(string Answer, string Context); public record SemanticSearchRequest(string QueryText, int Limit = 5);