diff --git a/scraper/internal/server/server.go b/scraper/internal/server/server.go index 1e37daa..03d32ec 100644 --- a/scraper/internal/server/server.go +++ b/scraper/internal/server/server.go @@ -126,6 +126,8 @@ func (s *Server) ListenAndServe(ctx context.Context) error { // Scrape status mux.HandleFunc("GET /api/scrape/status", s.handleScrapeStatus) mux.HandleFunc("GET /api/scrape/tasks", s.handleScrapeTasks) + // Re-index chapters for a book from MinIO into PocketBase chapters_idx + mux.HandleFunc("POST /api/reindex/{slug}", s.handleReindex) // Progress API mux.HandleFunc("GET /api/progress", s.handleGetProgress) mux.HandleFunc("POST /api/progress/{slug}", s.handleSetProgress) @@ -1067,3 +1069,43 @@ func (s *Server) handleScrapeTasks(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(tasks) } + +// handleReindex handles POST /api/reindex/{slug}. +// It rebuilds the chapters_idx PocketBase collection for the given book by +// walking its MinIO objects. Use this when chapters were scraped but the index +// is out of sync (e.g. after a failed UpsertChapterIdx during scraping). +func (s *Server) handleReindex(w http.ResponseWriter, r *http.Request) { + slug := r.PathValue("slug") + if slug == "" { + http.Error(w, `{"error":"missing slug"}`, http.StatusBadRequest) + return + } + + type reindexer interface { + ReindexChapters(ctx context.Context, slug string) (int, error) + } + ri, ok := s.store.(reindexer) + if !ok { + http.Error(w, `{"error":"store does not support reindex"}`, http.StatusNotImplemented) + return + } + + count, err := ri.ReindexChapters(r.Context(), slug) + if err != nil { + s.log.Error("reindex failed", "slug", slug, "indexed", count, "err", err) + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusInternalServerError) + _ = json.NewEncoder(w).Encode(map[string]interface{}{ + "error": err.Error(), + "indexed": count, + }) + return + } + + s.log.Info("reindex complete", "slug", slug, "indexed", count) + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]interface{}{ + "slug": slug, + "indexed": count, + }) +} diff --git a/scraper/internal/storage/hybrid.go b/scraper/internal/storage/hybrid.go index 69710c4..f33cd7d 100644 --- a/scraper/internal/storage/hybrid.go +++ b/scraper/internal/storage/hybrid.go @@ -140,6 +140,73 @@ func (h *HybridStore) CountChapters(ctx context.Context, slug string) int { return h.pb.CountChapterIdx(ctx, slug) } +// ReindexChapters walks all MinIO objects for slug, reads the title from the +// first line of each chapter markdown, and upserts them into chapters_idx. +// This repairs the PocketBase index when it falls out of sync with MinIO. +// Returns the number of chapters indexed and any non-fatal errors encountered. +func (h *HybridStore) ReindexChapters(ctx context.Context, slug string) (int, error) { + keys, err := h.minio.ListChapterKeys(ctx, slug) + if err != nil { + return 0, fmt.Errorf("reindex: list chapter keys: %w", err) + } + + count := 0 + var errs []string + for _, key := range keys { + // Parse chapter number from key: {slug}/vol-N/lo-hi/chapter-N.md + n := chapterNumberFromKey(key) + if n <= 0 { + h.log.Warn("ReindexChapters: could not parse chapter number from key", "key", key) + continue + } + + raw, readErr := h.minio.GetChapter(ctx, slug, 0, n) + if readErr != nil { + errs = append(errs, fmt.Sprintf("ch%d: %v", n, readErr)) + continue + } + + // Extract title from first line ("# Title text") or fall back to empty. + rawTitle := "" + if line, _, found := strings.Cut(raw, "\n"); found || raw != "" { + rawTitle = strings.TrimPrefix(strings.TrimSpace(line), "# ") + } + title, dateLabel := splitChapterTitle(rawTitle) + + if upsertErr := h.pb.UpsertChapterIdx(ctx, slug, n, title, dateLabel); upsertErr != nil { + errs = append(errs, fmt.Sprintf("ch%d upsert: %v", n, upsertErr)) + continue + } + count++ + } + + if len(errs) > 0 { + return count, fmt.Errorf("reindex: %d error(s): %s", len(errs), strings.Join(errs, "; ")) + } + return count, nil +} + +// chapterNumberFromKey parses the chapter number from a MinIO object key of the +// form "{slug}/vol-N/lo-hi/chapter-N.md". +func chapterNumberFromKey(key string) int { + // Grab the filename portion after the last '/'. + _, file, _ := strings.Cut(key, "/") + parts := strings.Split(key, "/") + if len(parts) == 0 { + return 0 + } + filename := parts[len(parts)-1] + // filename is "chapter-N.md" + filename = strings.TrimSuffix(filename, ".md") + filename = strings.TrimPrefix(filename, "chapter-") + n, err := strconv.Atoi(filename) + if err != nil || n <= 0 { + return 0 + } + _ = file + return n +} + // ─── Ranking ───────────────────────────────────────────────────────────────── func (h *HybridStore) WriteRankingItem(ctx context.Context, item RankingItem) error { diff --git a/scraper/internal/storage/store.go b/scraper/internal/storage/store.go index f3d2e09..d2360df 100644 --- a/scraper/internal/storage/store.go +++ b/scraper/internal/storage/store.go @@ -91,6 +91,9 @@ type Store interface { ListChapters(ctx context.Context, slug string) ([]ChapterInfo, error) // CountChapters returns the number of stored chapters for slug. CountChapters(ctx context.Context, slug string) int + // ReindexChapters rebuilds chapters_idx from MinIO objects for slug. + // Returns the number of chapters indexed. + ReindexChapters(ctx context.Context, slug string) (int, error) // ── Ranking ──────────────────────────────────────────────────────────── diff --git a/ui/Dockerfile b/ui/Dockerfile index a3adf46..f291b92 100644 --- a/ui/Dockerfile +++ b/ui/Dockerfile @@ -14,6 +14,10 @@ WORKDIR /app # adapter-node produces a standalone build/ COPY --from=builder /app/build ./build COPY --from=builder /app/package.json ./ +COPY --from=builder /app/package-lock.json ./ + +# Install production dependencies (e.g. marked) that are imported at runtime +RUN npm ci --omit=dev ENV NODE_ENV=production ENV PORT=3000