libnovel/scraper/internal/server/handlers_scrape.go

package server

import (
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"time"

	"github.com/libnovel/scraper/internal/orchestrator"
	"github.com/libnovel/scraper/internal/storage"
)

func (s *Server) handleScrapeCatalogue(w http.ResponseWriter, r *http.Request) {
	cfg := s.oCfg
	cfg.SingleBookURL = "" // full catalogue

	s.runAsync(w, cfg)
}

func (s *Server) handleScrapeBook(w http.ResponseWriter, r *http.Request) {
	var body struct {
		URL string `json:"url"`
	}
	if err := json.NewDecoder(r.Body).Decode(&body); err != nil || body.URL == "" {
		http.Error(w, `{"error":"request body must be JSON with \"url\" field"}`, http.StatusBadRequest)
		return
	}

	cfg := s.oCfg
	cfg.SingleBookURL = body.URL

	s.runAsync(w, cfg)
}

// runAsync launches an orchestrator in the background and returns 202 Accepted.
// Only one scrape job runs at a time; concurrent requests receive 409 Conflict.
func (s *Server) runAsync(w http.ResponseWriter, cfg orchestrator.Config) {
	s.mu.Lock()
	if s.running {
		s.mu.Unlock()
		http.Error(w, `{"error":"a scrape job is already running"}`, http.StatusConflict)
		return
	}
	s.running = true
	s.mu.Unlock()

	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(http.StatusAccepted)
	_ = json.NewEncoder(w).Encode(map[string]string{"status": "accepted"})

	go func() {
		defer func() {
			s.mu.Lock()
			s.running = false
			s.mu.Unlock()
		}()

		ctx, cancel := context.WithTimeout(context.Background(), 24*time.Hour)
		defer cancel()

		// Determine task kind and target.
		kind := "catalogue"
		targetURL := ""
		if cfg.SingleBookURL != "" {
			kind = "book"
			targetURL = cfg.SingleBookURL
		}

		// Create the task record in PocketBase.
		taskID, err := s.store.CreateScrapeTask(ctx, kind, targetURL)
		if err != nil {
			s.log.Warn("could not create scraping_tasks record", "err", err)
			// Non-fatal: continue without task tracking.
		}

		// flush pushes the latest counters to PocketBase (best-effort).
		flush := func(p orchestrator.Progress, status, errMsg string, finished bool) {
			if taskID == "" {
				return
			}
			u := storage.ScrapeTaskUpdate{
				Status:          status,
				BooksFound:      p.BooksFound,
				ChaptersScraped: p.ChaptersScraped,
				ChaptersSkipped: p.ChaptersSkipped,
				Errors:          p.Errors,
				ErrorMessage:    errMsg,
			}
			if finished {
				u.Finished = time.Now().UTC()
			}
			if updateErr := s.store.UpdateScrapeTask(ctx, taskID, u); updateErr != nil {
				s.log.Warn("could not update scraping_tasks record", "task_id", taskID, "err", updateErr)
			}
		}

		cfg.OnProgress = func(p orchestrator.Progress) {
			flush(p, "running", "", false)
		}

		o := orchestrator.New(cfg, s.novel, s.log, s.store)
		runErr := o.Run(ctx)

		// After a successful full-catalogue run, refresh the ranking list.
		if runErr == nil && cfg.SingleBookURL == "" {
			s.log.Info("runAsync: starting ScrapeRanking after catalogue run")
			rankCtx, rankCancel := context.WithTimeout(context.Background(), 30*time.Minute)
			defer rankCancel()
			rankEntries, rankErrs := s.novel.ScrapeRanking(rankCtx, 0)
			rank := 1
			for meta := range rankEntries {
				item := storage.RankingItem{
					Rank:      rank,
					Slug:      meta.Slug,
					Title:     meta.Title,
					Author:    meta.Author,
					Cover:     meta.Cover,
					Status:    meta.Status,
					Genres:    meta.Genres,
					SourceURL: meta.SourceURL,
				}
				if werr := s.store.WriteRankingItem(rankCtx, item); werr != nil {
					s.log.Warn("runAsync: WriteRankingItem failed", "slug", meta.Slug, "err", werr)
				}
				rank++
			}
			if rerr := <-rankErrs; rerr != nil {
				s.log.Warn("runAsync: ScrapeRanking finished with error", "err", rerr)
			} else {
				s.log.Info("runAsync: ScrapeRanking complete", "count", rank-1)
			}
		}

		// Determine final status.
		finalStatus := "done"
		errMsg := ""
		if runErr != nil {
			s.log.Error("scrape job failed", "err", fmt.Sprintf("%v", runErr))
			if ctx.Err() != nil {
				finalStatus = "cancelled"
			} else {
				finalStatus = "failed"
			}
			errMsg = runErr.Error()
		}

		// Best-effort: read last known progress counters via a zero-value
		// OnProgress — we don't have a snapshot here, so re-use whatever the
		// last OnProgress call delivered (the orchestrator calls notify() at
		// the very end, so this is always accurate after Run returns).
		// We issue one final flush with the terminal status and finished time.
		if taskID != "" {
			// Re-fetch current counters by listing the task (cheapest path).
			tasks, listErr := s.store.ListScrapeTasks(ctx)
			var last storage.ScrapeTaskUpdate
			if listErr == nil {
				for _, t := range tasks {
					if t.ID == taskID {
						last = storage.ScrapeTaskUpdate{
							BooksFound:      t.BooksFound,
							ChaptersScraped: t.ChaptersScraped,
							ChaptersSkipped: t.ChaptersSkipped,
							Errors:          t.Errors,
						}
						break
					}
				}
			}
			last.Status = finalStatus
			last.ErrorMessage = errMsg
			last.Finished = time.Now().UTC()
			if updateErr := s.store.UpdateScrapeTask(ctx, taskID, last); updateErr != nil {
				s.log.Warn("could not finalize scraping_tasks record", "task_id", taskID, "err", updateErr)
			}
		}
	}()
}

// ─── Scrape status API ────────────────────────────────────────────────────────

// handleScrapeStatus handles GET /api/scrape/status.
// Returns JSON: {"running": bool}
func (s *Server) handleScrapeStatus(w http.ResponseWriter, _ *http.Request) {
	s.mu.Lock()
	running := s.running
	s.mu.Unlock()

	w.Header().Set("Content-Type", "application/json")
	_ = json.NewEncoder(w).Encode(map[string]bool{"running": running})
}

// handleScrapeTasks handles GET /api/scrape/tasks.
// Returns JSON array of all scraping_tasks records, newest first.
func (s *Server) handleScrapeTasks(w http.ResponseWriter, r *http.Request) {
	tasks, err := s.store.ListScrapeTasks(r.Context())
	if err != nil {
		s.log.Error("handleScrapeTasks: list failed", "err", err)
		http.Error(w, `{"error":"failed to list tasks"}`, http.StatusInternalServerError)
		return
	}
	if tasks == nil {
		tasks = []storage.ScrapeTask{}
	}
	w.Header().Set("Content-Type", "application/json")
	_ = json.NewEncoder(w).Encode(tasks)
}

// handleReindex handles POST /api/reindex/{slug}.
// It rebuilds the chapters_idx PocketBase collection for the given book by
// walking its MinIO objects. Use this when chapters were scraped but the index
// is out of sync (e.g. after a failed UpsertChapterIdx during scraping).
func (s *Server) handleReindex(w http.ResponseWriter, r *http.Request) {
	slug := r.PathValue("slug")
	if slug == "" {
		http.Error(w, `{"error":"missing slug"}`, http.StatusBadRequest)
		return
	}

	type reindexer interface {
		ReindexChapters(ctx context.Context, slug string) (int, error)
	}
	ri, ok := s.store.(reindexer)
	if !ok {
		http.Error(w, `{"error":"store does not support reindex"}`, http.StatusNotImplemented)
		return
	}

	count, err := ri.ReindexChapters(r.Context(), slug)
	if err != nil {
		s.log.Error("reindex failed", "slug", slug, "indexed", count, "err", err)
		w.Header().Set("Content-Type", "application/json")
		w.WriteHeader(http.StatusInternalServerError)
		_ = json.NewEncoder(w).Encode(map[string]interface{}{
			"error":   err.Error(),
			"indexed": count,
		})
		return
	}

	s.log.Info("reindex complete", "slug", slug, "indexed", count)
	w.Header().Set("Content-Type", "application/json")
	_ = json.NewEncoder(w).Encode(map[string]interface{}{
		"slug":    slug,
		"indexed": count,
	})
}