package server import ( "context" "encoding/json" "fmt" "net/http" "time" "github.com/libnovel/scraper/internal/orchestrator" "github.com/libnovel/scraper/internal/storage" ) func (s *Server) handleScrapeCatalogue(w http.ResponseWriter, r *http.Request) { cfg := s.oCfg cfg.SingleBookURL = "" // full catalogue s.runAsync(w, cfg) } func (s *Server) handleScrapeBook(w http.ResponseWriter, r *http.Request) { var body struct { URL string `json:"url"` } if err := json.NewDecoder(r.Body).Decode(&body); err != nil || body.URL == "" { http.Error(w, `{"error":"request body must be JSON with \"url\" field"}`, http.StatusBadRequest) return } cfg := s.oCfg cfg.SingleBookURL = body.URL s.runAsync(w, cfg) } // handleScrapeBookRange handles POST /api/scrape/book/range. // Body: {"url": "...", "from": N, "to": M} // Scrapes only chapters in the range [from, to] (inclusive). // from=0 means "start from chapter 1"; to=0 means "no upper limit". func (s *Server) handleScrapeBookRange(w http.ResponseWriter, r *http.Request) { var body struct { URL string `json:"url"` From int `json:"from"` To int `json:"to"` } if err := json.NewDecoder(r.Body).Decode(&body); err != nil || body.URL == "" { http.Error(w, `{"error":"request body must be JSON with \"url\" field"}`, http.StatusBadRequest) return } cfg := s.oCfg cfg.SingleBookURL = body.URL cfg.FromChapter = body.From cfg.ToChapter = body.To s.runAsync(w, cfg) } // runAsync launches an orchestrator in the background and returns 202 Accepted. // Only one scrape job runs at a time; concurrent requests receive 409 Conflict. func (s *Server) runAsync(w http.ResponseWriter, cfg orchestrator.Config) { s.mu.Lock() if s.running { s.mu.Unlock() http.Error(w, `{"error":"a scrape job is already running"}`, http.StatusConflict) return } s.running = true s.mu.Unlock() w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusAccepted) _ = json.NewEncoder(w).Encode(map[string]string{"status": "accepted"}) go func() { defer func() { s.mu.Lock() s.running = false s.mu.Unlock() }() ctx, cancel := context.WithTimeout(context.Background(), 24*time.Hour) defer cancel() // Determine task kind and target. kind := "catalogue" targetURL := "" if cfg.SingleBookURL != "" { kind = "book" targetURL = cfg.SingleBookURL } // Create the task record in PocketBase. taskID, err := s.store.CreateScrapeTask(ctx, kind, targetURL) if err != nil { s.log.Warn("could not create scraping_tasks record", "err", err) // Non-fatal: continue without task tracking. } // flush pushes the latest counters to PocketBase (best-effort). flush := func(p orchestrator.Progress, status, errMsg string, finished bool) { if taskID == "" { return } u := storage.ScrapeTaskUpdate{ Status: status, BooksFound: p.BooksFound, ChaptersScraped: p.ChaptersScraped, ChaptersSkipped: p.ChaptersSkipped, Errors: p.Errors, ErrorMessage: errMsg, } if finished { u.Finished = time.Now().UTC() } if updateErr := s.store.UpdateScrapeTask(ctx, taskID, u); updateErr != nil { s.log.Warn("could not update scraping_tasks record", "task_id", taskID, "err", updateErr) } } cfg.OnProgress = func(p orchestrator.Progress) { flush(p, "running", "", false) } o := orchestrator.New(cfg, s.novel, s.log, s.store) runErr := o.Run(ctx) // After a successful full-catalogue run, refresh the ranking list. if runErr == nil && cfg.SingleBookURL == "" { s.log.Info("runAsync: starting ScrapeRanking after catalogue run") rankCtx, rankCancel := context.WithTimeout(context.Background(), 30*time.Minute) defer rankCancel() rankEntries, rankErrs := s.novel.ScrapeRanking(rankCtx, 0) rank := 1 for meta := range rankEntries { item := storage.RankingItem{ Rank: rank, Slug: meta.Slug, Title: meta.Title, Author: meta.Author, Cover: meta.Cover, Status: meta.Status, Genres: meta.Genres, SourceURL: meta.SourceURL, } if werr := s.store.WriteRankingItem(rankCtx, item); werr != nil { s.log.Warn("runAsync: WriteRankingItem failed", "slug", meta.Slug, "err", werr) } rank++ } if rerr := <-rankErrs; rerr != nil { s.log.Warn("runAsync: ScrapeRanking finished with error", "err", rerr) } else { s.log.Info("runAsync: ScrapeRanking complete", "count", rank-1) } } // Determine final status. finalStatus := "done" errMsg := "" if runErr != nil { s.log.Error("scrape job failed", "err", fmt.Sprintf("%v", runErr)) if ctx.Err() != nil { finalStatus = "cancelled" } else { finalStatus = "failed" } errMsg = runErr.Error() } // Best-effort: read last known progress counters via a zero-value // OnProgress — we don't have a snapshot here, so re-use whatever the // last OnProgress call delivered (the orchestrator calls notify() at // the very end, so this is always accurate after Run returns). // We issue one final flush with the terminal status and finished time. if taskID != "" { // Re-fetch current counters by listing the task (cheapest path). tasks, listErr := s.store.ListScrapeTasks(ctx) var last storage.ScrapeTaskUpdate if listErr == nil { for _, t := range tasks { if t.ID == taskID { last = storage.ScrapeTaskUpdate{ BooksFound: t.BooksFound, ChaptersScraped: t.ChaptersScraped, ChaptersSkipped: t.ChaptersSkipped, Errors: t.Errors, } break } } } last.Status = finalStatus last.ErrorMessage = errMsg last.Finished = time.Now().UTC() if updateErr := s.store.UpdateScrapeTask(ctx, taskID, last); updateErr != nil { s.log.Warn("could not finalize scraping_tasks record", "task_id", taskID, "err", updateErr) } } }() } // ─── Scrape status API ──────────────────────────────────────────────────────── // handleScrapeStatus handles GET /api/scrape/status. // Returns JSON: {"running": bool} func (s *Server) handleScrapeStatus(w http.ResponseWriter, _ *http.Request) { s.mu.Lock() running := s.running s.mu.Unlock() w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(map[string]bool{"running": running}) } // handleScrapeTasks handles GET /api/scrape/tasks. // Returns JSON array of all scraping_tasks records, newest first. func (s *Server) handleScrapeTasks(w http.ResponseWriter, r *http.Request) { tasks, err := s.store.ListScrapeTasks(r.Context()) if err != nil { s.log.Error("handleScrapeTasks: list failed", "err", err) http.Error(w, `{"error":"failed to list tasks"}`, http.StatusInternalServerError) return } if tasks == nil { tasks = []storage.ScrapeTask{} } w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(tasks) } // handleReindex handles POST /api/reindex/{slug}. // It rebuilds the chapters_idx PocketBase collection for the given book by // walking its MinIO objects. Use this when chapters were scraped but the index // is out of sync (e.g. after a failed UpsertChapterIdx during scraping). func (s *Server) handleReindex(w http.ResponseWriter, r *http.Request) { slug := r.PathValue("slug") if slug == "" { http.Error(w, `{"error":"missing slug"}`, http.StatusBadRequest) return } type reindexer interface { ReindexChapters(ctx context.Context, slug string) (int, error) } ri, ok := s.store.(reindexer) if !ok { http.Error(w, `{"error":"store does not support reindex"}`, http.StatusNotImplemented) return } count, err := ri.ReindexChapters(r.Context(), slug) if err != nil { s.log.Error("reindex failed", "slug", slug, "indexed", count, "err", err) w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusInternalServerError) _ = json.NewEncoder(w).Encode(map[string]interface{}{ "error": err.Error(), "indexed": count, }) return } s.log.Info("reindex complete", "slug", slug, "indexed", count) w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(map[string]interface{}{ "slug": slug, "indexed": count, }) }