a54d8d43aa
Adds FromChapter/ToChapter fields to orchestrator.Config and skips out-of-range chapters in processBook. Exposes POST /scrape/book/range Go endpoint and a matching UI proxy at /api/scrape/range. The book detail page now shows admins a range input (from/to chapter) and a per-chapter 'scrape from here up' button.
271 lines
8.2 KiB
Go
271 lines
8.2 KiB
Go
package server
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"net/http"
|
|
"time"
|
|
|
|
"github.com/libnovel/scraper/internal/orchestrator"
|
|
"github.com/libnovel/scraper/internal/storage"
|
|
)
|
|
|
|
func (s *Server) handleScrapeCatalogue(w http.ResponseWriter, r *http.Request) {
|
|
cfg := s.oCfg
|
|
cfg.SingleBookURL = "" // full catalogue
|
|
|
|
s.runAsync(w, cfg)
|
|
}
|
|
|
|
func (s *Server) handleScrapeBook(w http.ResponseWriter, r *http.Request) {
|
|
var body struct {
|
|
URL string `json:"url"`
|
|
}
|
|
if err := json.NewDecoder(r.Body).Decode(&body); err != nil || body.URL == "" {
|
|
http.Error(w, `{"error":"request body must be JSON with \"url\" field"}`, http.StatusBadRequest)
|
|
return
|
|
}
|
|
|
|
cfg := s.oCfg
|
|
cfg.SingleBookURL = body.URL
|
|
|
|
s.runAsync(w, cfg)
|
|
}
|
|
|
|
// handleScrapeBookRange handles POST /api/scrape/book/range.
|
|
// Body: {"url": "...", "from": N, "to": M}
|
|
// Scrapes only chapters in the range [from, to] (inclusive).
|
|
// from=0 means "start from chapter 1"; to=0 means "no upper limit".
|
|
func (s *Server) handleScrapeBookRange(w http.ResponseWriter, r *http.Request) {
|
|
var body struct {
|
|
URL string `json:"url"`
|
|
From int `json:"from"`
|
|
To int `json:"to"`
|
|
}
|
|
if err := json.NewDecoder(r.Body).Decode(&body); err != nil || body.URL == "" {
|
|
http.Error(w, `{"error":"request body must be JSON with \"url\" field"}`, http.StatusBadRequest)
|
|
return
|
|
}
|
|
|
|
cfg := s.oCfg
|
|
cfg.SingleBookURL = body.URL
|
|
cfg.FromChapter = body.From
|
|
cfg.ToChapter = body.To
|
|
|
|
s.runAsync(w, cfg)
|
|
}
|
|
|
|
// runAsync launches an orchestrator in the background and returns 202 Accepted.
|
|
// Only one scrape job runs at a time; concurrent requests receive 409 Conflict.
|
|
func (s *Server) runAsync(w http.ResponseWriter, cfg orchestrator.Config) {
|
|
s.mu.Lock()
|
|
if s.running {
|
|
s.mu.Unlock()
|
|
http.Error(w, `{"error":"a scrape job is already running"}`, http.StatusConflict)
|
|
return
|
|
}
|
|
s.running = true
|
|
s.mu.Unlock()
|
|
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.WriteHeader(http.StatusAccepted)
|
|
_ = json.NewEncoder(w).Encode(map[string]string{"status": "accepted"})
|
|
|
|
go func() {
|
|
defer func() {
|
|
s.mu.Lock()
|
|
s.running = false
|
|
s.mu.Unlock()
|
|
}()
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 24*time.Hour)
|
|
defer cancel()
|
|
|
|
// Determine task kind and target.
|
|
kind := "catalogue"
|
|
targetURL := ""
|
|
if cfg.SingleBookURL != "" {
|
|
kind = "book"
|
|
targetURL = cfg.SingleBookURL
|
|
}
|
|
|
|
// Create the task record in PocketBase.
|
|
taskID, err := s.store.CreateScrapeTask(ctx, kind, targetURL)
|
|
if err != nil {
|
|
s.log.Warn("could not create scraping_tasks record", "err", err)
|
|
// Non-fatal: continue without task tracking.
|
|
}
|
|
|
|
// flush pushes the latest counters to PocketBase (best-effort).
|
|
flush := func(p orchestrator.Progress, status, errMsg string, finished bool) {
|
|
if taskID == "" {
|
|
return
|
|
}
|
|
u := storage.ScrapeTaskUpdate{
|
|
Status: status,
|
|
BooksFound: p.BooksFound,
|
|
ChaptersScraped: p.ChaptersScraped,
|
|
ChaptersSkipped: p.ChaptersSkipped,
|
|
Errors: p.Errors,
|
|
ErrorMessage: errMsg,
|
|
}
|
|
if finished {
|
|
u.Finished = time.Now().UTC()
|
|
}
|
|
if updateErr := s.store.UpdateScrapeTask(ctx, taskID, u); updateErr != nil {
|
|
s.log.Warn("could not update scraping_tasks record", "task_id", taskID, "err", updateErr)
|
|
}
|
|
}
|
|
|
|
cfg.OnProgress = func(p orchestrator.Progress) {
|
|
flush(p, "running", "", false)
|
|
}
|
|
|
|
o := orchestrator.New(cfg, s.novel, s.log, s.store)
|
|
runErr := o.Run(ctx)
|
|
|
|
// After a successful full-catalogue run, refresh the ranking list.
|
|
if runErr == nil && cfg.SingleBookURL == "" {
|
|
s.log.Info("runAsync: starting ScrapeRanking after catalogue run")
|
|
rankCtx, rankCancel := context.WithTimeout(context.Background(), 30*time.Minute)
|
|
defer rankCancel()
|
|
rankEntries, rankErrs := s.novel.ScrapeRanking(rankCtx, 0)
|
|
rank := 1
|
|
for meta := range rankEntries {
|
|
item := storage.RankingItem{
|
|
Rank: rank,
|
|
Slug: meta.Slug,
|
|
Title: meta.Title,
|
|
Author: meta.Author,
|
|
Cover: meta.Cover,
|
|
Status: meta.Status,
|
|
Genres: meta.Genres,
|
|
SourceURL: meta.SourceURL,
|
|
}
|
|
if werr := s.store.WriteRankingItem(rankCtx, item); werr != nil {
|
|
s.log.Warn("runAsync: WriteRankingItem failed", "slug", meta.Slug, "err", werr)
|
|
}
|
|
rank++
|
|
}
|
|
if rerr := <-rankErrs; rerr != nil {
|
|
s.log.Warn("runAsync: ScrapeRanking finished with error", "err", rerr)
|
|
} else {
|
|
s.log.Info("runAsync: ScrapeRanking complete", "count", rank-1)
|
|
}
|
|
}
|
|
|
|
// Determine final status.
|
|
finalStatus := "done"
|
|
errMsg := ""
|
|
if runErr != nil {
|
|
s.log.Error("scrape job failed", "err", fmt.Sprintf("%v", runErr))
|
|
if ctx.Err() != nil {
|
|
finalStatus = "cancelled"
|
|
} else {
|
|
finalStatus = "failed"
|
|
}
|
|
errMsg = runErr.Error()
|
|
}
|
|
|
|
// Best-effort: read last known progress counters via a zero-value
|
|
// OnProgress — we don't have a snapshot here, so re-use whatever the
|
|
// last OnProgress call delivered (the orchestrator calls notify() at
|
|
// the very end, so this is always accurate after Run returns).
|
|
// We issue one final flush with the terminal status and finished time.
|
|
if taskID != "" {
|
|
// Re-fetch current counters by listing the task (cheapest path).
|
|
tasks, listErr := s.store.ListScrapeTasks(ctx)
|
|
var last storage.ScrapeTaskUpdate
|
|
if listErr == nil {
|
|
for _, t := range tasks {
|
|
if t.ID == taskID {
|
|
last = storage.ScrapeTaskUpdate{
|
|
BooksFound: t.BooksFound,
|
|
ChaptersScraped: t.ChaptersScraped,
|
|
ChaptersSkipped: t.ChaptersSkipped,
|
|
Errors: t.Errors,
|
|
}
|
|
break
|
|
}
|
|
}
|
|
}
|
|
last.Status = finalStatus
|
|
last.ErrorMessage = errMsg
|
|
last.Finished = time.Now().UTC()
|
|
if updateErr := s.store.UpdateScrapeTask(ctx, taskID, last); updateErr != nil {
|
|
s.log.Warn("could not finalize scraping_tasks record", "task_id", taskID, "err", updateErr)
|
|
}
|
|
}
|
|
}()
|
|
}
|
|
|
|
// ─── Scrape status API ────────────────────────────────────────────────────────
|
|
|
|
// handleScrapeStatus handles GET /api/scrape/status.
|
|
// Returns JSON: {"running": bool}
|
|
func (s *Server) handleScrapeStatus(w http.ResponseWriter, _ *http.Request) {
|
|
s.mu.Lock()
|
|
running := s.running
|
|
s.mu.Unlock()
|
|
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_ = json.NewEncoder(w).Encode(map[string]bool{"running": running})
|
|
}
|
|
|
|
// handleScrapeTasks handles GET /api/scrape/tasks.
|
|
// Returns JSON array of all scraping_tasks records, newest first.
|
|
func (s *Server) handleScrapeTasks(w http.ResponseWriter, r *http.Request) {
|
|
tasks, err := s.store.ListScrapeTasks(r.Context())
|
|
if err != nil {
|
|
s.log.Error("handleScrapeTasks: list failed", "err", err)
|
|
http.Error(w, `{"error":"failed to list tasks"}`, http.StatusInternalServerError)
|
|
return
|
|
}
|
|
if tasks == nil {
|
|
tasks = []storage.ScrapeTask{}
|
|
}
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_ = json.NewEncoder(w).Encode(tasks)
|
|
}
|
|
|
|
// handleReindex handles POST /api/reindex/{slug}.
|
|
// It rebuilds the chapters_idx PocketBase collection for the given book by
|
|
// walking its MinIO objects. Use this when chapters were scraped but the index
|
|
// is out of sync (e.g. after a failed UpsertChapterIdx during scraping).
|
|
func (s *Server) handleReindex(w http.ResponseWriter, r *http.Request) {
|
|
slug := r.PathValue("slug")
|
|
if slug == "" {
|
|
http.Error(w, `{"error":"missing slug"}`, http.StatusBadRequest)
|
|
return
|
|
}
|
|
|
|
type reindexer interface {
|
|
ReindexChapters(ctx context.Context, slug string) (int, error)
|
|
}
|
|
ri, ok := s.store.(reindexer)
|
|
if !ok {
|
|
http.Error(w, `{"error":"store does not support reindex"}`, http.StatusNotImplemented)
|
|
return
|
|
}
|
|
|
|
count, err := ri.ReindexChapters(r.Context(), slug)
|
|
if err != nil {
|
|
s.log.Error("reindex failed", "slug", slug, "indexed", count, "err", err)
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.WriteHeader(http.StatusInternalServerError)
|
|
_ = json.NewEncoder(w).Encode(map[string]interface{}{
|
|
"error": err.Error(),
|
|
"indexed": count,
|
|
})
|
|
return
|
|
}
|
|
|
|
s.log.Info("reindex complete", "slug", slug, "indexed", count)
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_ = json.NewEncoder(w).Encode(map[string]interface{}{
|
|
"slug": slug,
|
|
"indexed": count,
|
|
})
|
|
}
|