fix(ui): install prod deps in Docker runtime; add reindex endpoint for chapters_idx
- ui/Dockerfile: copy package-lock.json and run npm ci --omit=dev in the
runtime stage so marked (and other runtime deps) are available to
adapter-node at startup — fixes ERR_MODULE_NOT_FOUND for 'marked'
- storage: add ReindexChapters to Store interface and HybridStore — walks
MinIO objects for a slug, reads chapter titles, upserts chapters_idx
- server: add POST /api/reindex/{slug} to rebuild chapters_idx from MinIO
This commit is contained in:
@@ -126,6 +126,8 @@ func (s *Server) ListenAndServe(ctx context.Context) error {
|
||||
// Scrape status
|
||||
mux.HandleFunc("GET /api/scrape/status", s.handleScrapeStatus)
|
||||
mux.HandleFunc("GET /api/scrape/tasks", s.handleScrapeTasks)
|
||||
// Re-index chapters for a book from MinIO into PocketBase chapters_idx
|
||||
mux.HandleFunc("POST /api/reindex/{slug}", s.handleReindex)
|
||||
// Progress API
|
||||
mux.HandleFunc("GET /api/progress", s.handleGetProgress)
|
||||
mux.HandleFunc("POST /api/progress/{slug}", s.handleSetProgress)
|
||||
@@ -1067,3 +1069,43 @@ func (s *Server) handleScrapeTasks(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(tasks)
|
||||
}
|
||||
|
||||
// handleReindex handles POST /api/reindex/{slug}.
|
||||
// It rebuilds the chapters_idx PocketBase collection for the given book by
|
||||
// walking its MinIO objects. Use this when chapters were scraped but the index
|
||||
// is out of sync (e.g. after a failed UpsertChapterIdx during scraping).
|
||||
func (s *Server) handleReindex(w http.ResponseWriter, r *http.Request) {
|
||||
slug := r.PathValue("slug")
|
||||
if slug == "" {
|
||||
http.Error(w, `{"error":"missing slug"}`, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
type reindexer interface {
|
||||
ReindexChapters(ctx context.Context, slug string) (int, error)
|
||||
}
|
||||
ri, ok := s.store.(reindexer)
|
||||
if !ok {
|
||||
http.Error(w, `{"error":"store does not support reindex"}`, http.StatusNotImplemented)
|
||||
return
|
||||
}
|
||||
|
||||
count, err := ri.ReindexChapters(r.Context(), slug)
|
||||
if err != nil {
|
||||
s.log.Error("reindex failed", "slug", slug, "indexed", count, "err", err)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
_ = json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
"indexed": count,
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
s.log.Info("reindex complete", "slug", slug, "indexed", count)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"slug": slug,
|
||||
"indexed": count,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -140,6 +140,73 @@ func (h *HybridStore) CountChapters(ctx context.Context, slug string) int {
|
||||
return h.pb.CountChapterIdx(ctx, slug)
|
||||
}
|
||||
|
||||
// ReindexChapters walks all MinIO objects for slug, reads the title from the
|
||||
// first line of each chapter markdown, and upserts them into chapters_idx.
|
||||
// This repairs the PocketBase index when it falls out of sync with MinIO.
|
||||
// Returns the number of chapters indexed and any non-fatal errors encountered.
|
||||
func (h *HybridStore) ReindexChapters(ctx context.Context, slug string) (int, error) {
|
||||
keys, err := h.minio.ListChapterKeys(ctx, slug)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("reindex: list chapter keys: %w", err)
|
||||
}
|
||||
|
||||
count := 0
|
||||
var errs []string
|
||||
for _, key := range keys {
|
||||
// Parse chapter number from key: {slug}/vol-N/lo-hi/chapter-N.md
|
||||
n := chapterNumberFromKey(key)
|
||||
if n <= 0 {
|
||||
h.log.Warn("ReindexChapters: could not parse chapter number from key", "key", key)
|
||||
continue
|
||||
}
|
||||
|
||||
raw, readErr := h.minio.GetChapter(ctx, slug, 0, n)
|
||||
if readErr != nil {
|
||||
errs = append(errs, fmt.Sprintf("ch%d: %v", n, readErr))
|
||||
continue
|
||||
}
|
||||
|
||||
// Extract title from first line ("# Title text") or fall back to empty.
|
||||
rawTitle := ""
|
||||
if line, _, found := strings.Cut(raw, "\n"); found || raw != "" {
|
||||
rawTitle = strings.TrimPrefix(strings.TrimSpace(line), "# ")
|
||||
}
|
||||
title, dateLabel := splitChapterTitle(rawTitle)
|
||||
|
||||
if upsertErr := h.pb.UpsertChapterIdx(ctx, slug, n, title, dateLabel); upsertErr != nil {
|
||||
errs = append(errs, fmt.Sprintf("ch%d upsert: %v", n, upsertErr))
|
||||
continue
|
||||
}
|
||||
count++
|
||||
}
|
||||
|
||||
if len(errs) > 0 {
|
||||
return count, fmt.Errorf("reindex: %d error(s): %s", len(errs), strings.Join(errs, "; "))
|
||||
}
|
||||
return count, nil
|
||||
}
|
||||
|
||||
// chapterNumberFromKey parses the chapter number from a MinIO object key of the
|
||||
// form "{slug}/vol-N/lo-hi/chapter-N.md".
|
||||
func chapterNumberFromKey(key string) int {
|
||||
// Grab the filename portion after the last '/'.
|
||||
_, file, _ := strings.Cut(key, "/")
|
||||
parts := strings.Split(key, "/")
|
||||
if len(parts) == 0 {
|
||||
return 0
|
||||
}
|
||||
filename := parts[len(parts)-1]
|
||||
// filename is "chapter-N.md"
|
||||
filename = strings.TrimSuffix(filename, ".md")
|
||||
filename = strings.TrimPrefix(filename, "chapter-")
|
||||
n, err := strconv.Atoi(filename)
|
||||
if err != nil || n <= 0 {
|
||||
return 0
|
||||
}
|
||||
_ = file
|
||||
return n
|
||||
}
|
||||
|
||||
// ─── Ranking ─────────────────────────────────────────────────────────────────
|
||||
|
||||
func (h *HybridStore) WriteRankingItem(ctx context.Context, item RankingItem) error {
|
||||
|
||||
@@ -91,6 +91,9 @@ type Store interface {
|
||||
ListChapters(ctx context.Context, slug string) ([]ChapterInfo, error)
|
||||
// CountChapters returns the number of stored chapters for slug.
|
||||
CountChapters(ctx context.Context, slug string) int
|
||||
// ReindexChapters rebuilds chapters_idx from MinIO objects for slug.
|
||||
// Returns the number of chapters indexed.
|
||||
ReindexChapters(ctx context.Context, slug string) (int, error)
|
||||
|
||||
// ── Ranking ────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@@ -14,6 +14,10 @@ WORKDIR /app
|
||||
# adapter-node produces a standalone build/
|
||||
COPY --from=builder /app/build ./build
|
||||
COPY --from=builder /app/package.json ./
|
||||
COPY --from=builder /app/package-lock.json ./
|
||||
|
||||
# Install production dependencies (e.g. marked) that are imported at runtime
|
||||
RUN npm ci --omit=dev
|
||||
|
||||
ENV NODE_ENV=production
|
||||
ENV PORT=3000
|
||||
|
||||
Reference in New Issue
Block a user