fb5d22eb8d
- Extract scrape form and autocomplete JS from homeTmpl into new scrapeTmpl - Add handleScrape GET handler serving /scrape with ranking autocomplete - Register GET /scrape route in server.go - Replace inline scrape form on home page with '+ Add' flat button in header - handleHome no longer loads ranking items (only needed on /scrape)
600 lines
18 KiB
Go
600 lines
18 KiB
Go
// Package server exposes the scraper as an HTTP service.
|
|
//
|
|
// Endpoints:
|
|
//
|
|
// POST /scrape — enqueue a full catalogue scrape
|
|
// POST /scrape/book — enqueue a single-book scrape (JSON body: {"url":"..."})
|
|
// GET /health — liveness probe
|
|
package server
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"log/slog"
|
|
"net/http"
|
|
"os"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/libnovel/scraper/internal/orchestrator"
|
|
"github.com/libnovel/scraper/internal/scraper"
|
|
"github.com/libnovel/scraper/internal/writer"
|
|
)
|
|
|
|
// Server wraps an HTTP mux with the scraping endpoints.
|
|
type Server struct {
|
|
addr string
|
|
oCfg orchestrator.Config
|
|
novel scraper.NovelScraper
|
|
log *slog.Logger
|
|
writer *writer.Writer
|
|
mu sync.Mutex
|
|
running bool
|
|
rankingRunning bool
|
|
kokoroURL string // Kokoro-FastAPI base URL, e.g. http://kokoro:8880
|
|
kokoroVoice string // default voice, e.g. af_bella
|
|
|
|
// audioMu guards audioInFlight.
|
|
// audioInFlight maps an audio cache key to a channel that is closed when
|
|
// the in-flight Kokoro request for that key finishes (successfully or not).
|
|
// This prevents duplicate concurrent TTS generation for the same file.
|
|
audioMu sync.Mutex
|
|
audioInFlight map[string]chan struct{}
|
|
}
|
|
|
|
// New creates a new Server.
|
|
func New(addr string, oCfg orchestrator.Config, novel scraper.NovelScraper, log *slog.Logger, kokoroURL, kokoroVoice string) *Server {
|
|
return &Server{
|
|
addr: addr,
|
|
oCfg: oCfg,
|
|
novel: novel,
|
|
log: log,
|
|
writer: writer.New(oCfg.StaticRoot),
|
|
kokoroURL: kokoroURL,
|
|
kokoroVoice: kokoroVoice,
|
|
audioInFlight: make(map[string]chan struct{}),
|
|
}
|
|
}
|
|
|
|
// ListenAndServe starts the HTTP server and blocks until the provided context
|
|
// is cancelled.
|
|
func (s *Server) ListenAndServe(ctx context.Context) error {
|
|
mux := http.NewServeMux()
|
|
mux.HandleFunc("GET /health", s.handleHealth)
|
|
mux.HandleFunc("POST /scrape", s.handleScrapeCatalogue)
|
|
mux.HandleFunc("POST /scrape/book", s.handleScrapeBook)
|
|
// UI routes
|
|
mux.HandleFunc("GET /", s.handleHome)
|
|
mux.HandleFunc("GET /scrape", s.handleScrape)
|
|
mux.HandleFunc("GET /ranking", s.handleRanking)
|
|
mux.HandleFunc("POST /ranking/refresh", s.handleRankingRefresh)
|
|
mux.HandleFunc("GET /ranking/view", s.handleRankingView)
|
|
mux.HandleFunc("GET /books/{slug}", s.handleBook)
|
|
mux.HandleFunc("GET /books/{slug}/chapters/{n}", s.handleChapter)
|
|
mux.HandleFunc("GET /books/{slug}/chapters-page", s.handleBookChaptersPage)
|
|
mux.HandleFunc("POST /ui/scrape/book", s.handleUIScrapeBook)
|
|
mux.HandleFunc("GET /ui/scrape/status", s.handleUIScrapeStatus)
|
|
mux.HandleFunc("GET /ui/ranking/status", s.handleRankingStatus)
|
|
// Plain-text chapter content for browser-side TTS
|
|
mux.HandleFunc("GET /ui/chapter-text/{slug}/{n}", s.handleChapterText)
|
|
// Server-side audio generation and serving.
|
|
// Audio generation can take several minutes for long chapters, so wrap it
|
|
// in its own timeout handler instead of relying on the server WriteTimeout.
|
|
audioGenHandler := http.TimeoutHandler(
|
|
http.HandlerFunc(s.handleAudioGenerate),
|
|
10*time.Minute,
|
|
`{"error":"audio generation timed out"}`,
|
|
)
|
|
mux.Handle("POST /ui/audio/{slug}/{n}", audioGenHandler)
|
|
mux.HandleFunc("GET /ui/audio/{slug}/{n}/status", s.handleAudioStatus)
|
|
mux.HandleFunc("GET /ui/audio-file/{slug}/{n}", s.handleAudioFile)
|
|
mux.HandleFunc("GET /ui/audio-file/{slug}/{n}/part/{p}", s.handleAudioFilePart)
|
|
|
|
srv := &http.Server{
|
|
Addr: s.addr,
|
|
Handler: mux,
|
|
ReadTimeout: 15 * time.Second,
|
|
WriteTimeout: 60 * time.Second,
|
|
IdleTimeout: 60 * time.Second,
|
|
}
|
|
|
|
errCh := make(chan error, 1)
|
|
go func() { errCh <- srv.ListenAndServe() }()
|
|
|
|
s.log.Info("HTTP server listening", "addr", s.addr)
|
|
|
|
select {
|
|
case <-ctx.Done():
|
|
shutCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
defer cancel()
|
|
return srv.Shutdown(shutCtx)
|
|
case err := <-errCh:
|
|
return err
|
|
}
|
|
}
|
|
|
|
func (s *Server) handleHealth(w http.ResponseWriter, _ *http.Request) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_ = json.NewEncoder(w).Encode(map[string]string{"status": "ok"})
|
|
}
|
|
|
|
// handleChapterText returns the plain text of a chapter (markdown stripped)
|
|
// for browser-side TTS. The browser POSTs this directly to Kokoro-FastAPI.
|
|
func (s *Server) handleChapterText(w http.ResponseWriter, r *http.Request) {
|
|
slug := r.PathValue("slug")
|
|
n, err := strconv.Atoi(r.PathValue("n"))
|
|
if err != nil || n < 1 {
|
|
http.NotFound(w, r)
|
|
return
|
|
}
|
|
raw, err := s.writer.ReadChapter(slug, n)
|
|
if err != nil {
|
|
http.NotFound(w, r)
|
|
return
|
|
}
|
|
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
|
|
w.Header().Set("Cache-Control", "no-store")
|
|
fmt.Fprint(w, stripMarkdown(raw))
|
|
}
|
|
|
|
// ─── Chunked audio generation ────────────────────────────────────────────────
|
|
//
|
|
// handleAudioGenerate handles POST /ui/audio/{slug}/{n}.
|
|
//
|
|
// Flow:
|
|
// 1. If the merged MP3 already exists on disk → return it immediately.
|
|
// 2. Otherwise split the chapter text into up to audioParts equal-ish chunks
|
|
// (by paragraph), generate part 0 synchronously (so the browser can start
|
|
// playing right away), then launch a background goroutine that generates
|
|
// parts 1…N and merges them into the final file.
|
|
// 3. Response: {"url":"<part-0-url>","parts":N,"merged":false}
|
|
// (or {"url":"<final-url>","parts":1,"merged":true} on a cache hit).
|
|
//
|
|
// Deduplication: if another request is already generating the *merged* file
|
|
// for the same (slug,n,voice,speed) key, the new request blocks until it
|
|
// finishes and then serves the cached result.
|
|
const audioParts = 10
|
|
|
|
// handleAudioGenerate handles POST /ui/audio/{slug}/{n}.
|
|
func (s *Server) handleAudioGenerate(w http.ResponseWriter, r *http.Request) {
|
|
slug := r.PathValue("slug")
|
|
n, err := strconv.Atoi(r.PathValue("n"))
|
|
if err != nil || n < 1 {
|
|
http.Error(w, `{"error":"invalid chapter"}`, http.StatusBadRequest)
|
|
return
|
|
}
|
|
|
|
// Parse optional voice/speed from JSON body.
|
|
voice := s.kokoroVoice
|
|
speed := 1.0
|
|
var body struct {
|
|
Voice string `json:"voice"`
|
|
Speed float64 `json:"speed"`
|
|
}
|
|
if r.Body != nil {
|
|
_ = json.NewDecoder(r.Body).Decode(&body)
|
|
}
|
|
if body.Voice != "" {
|
|
voice = body.Voice
|
|
}
|
|
if body.Speed > 0 {
|
|
speed = body.Speed
|
|
}
|
|
|
|
audioPath := s.writer.AudioPath(slug, n, voice, speed)
|
|
|
|
// Fast path: merged file already on disk.
|
|
if _, err := os.Stat(audioPath); err == nil {
|
|
s.writeAudioResponse(w, slug, n, voice, speed, 1, true)
|
|
return
|
|
}
|
|
|
|
// Deduplicate concurrent generation requests for the same merged file.
|
|
cacheKey := fmt.Sprintf("%s/%d/%s/%.2f", slug, n, voice, speed)
|
|
|
|
s.audioMu.Lock()
|
|
if ch, ok := s.audioInFlight[cacheKey]; ok {
|
|
s.audioMu.Unlock()
|
|
select {
|
|
case <-ch:
|
|
case <-r.Context().Done():
|
|
http.Error(w, `{"error":"request cancelled"}`, http.StatusServiceUnavailable)
|
|
return
|
|
}
|
|
if _, err := os.Stat(audioPath); err == nil {
|
|
s.writeAudioResponse(w, slug, n, voice, speed, 1, true)
|
|
} else {
|
|
http.Error(w, `{"error":"audio generation failed"}`, http.StatusInternalServerError)
|
|
}
|
|
return
|
|
}
|
|
ch := make(chan struct{})
|
|
s.audioInFlight[cacheKey] = ch
|
|
s.audioMu.Unlock()
|
|
|
|
defer func() {
|
|
s.audioMu.Lock()
|
|
delete(s.audioInFlight, cacheKey)
|
|
s.audioMu.Unlock()
|
|
close(ch)
|
|
}()
|
|
|
|
// Load and validate chapter text.
|
|
raw, err := s.writer.ReadChapter(slug, n)
|
|
if err != nil {
|
|
http.Error(w, `{"error":"chapter not found"}`, http.StatusNotFound)
|
|
return
|
|
}
|
|
text := stripMarkdown(raw)
|
|
if text == "" {
|
|
http.Error(w, `{"error":"chapter text is empty"}`, http.StatusUnprocessableEntity)
|
|
return
|
|
}
|
|
if s.kokoroURL == "" {
|
|
http.Error(w, `{"error":"kokoro not configured"}`, http.StatusServiceUnavailable)
|
|
return
|
|
}
|
|
|
|
// Ensure audio dir exists.
|
|
if err := os.MkdirAll(s.writer.AudioDir(slug), 0o755); err != nil {
|
|
http.Error(w, `{"error":"failed to create audio dir"}`, http.StatusInternalServerError)
|
|
return
|
|
}
|
|
|
|
parts := splitTextIntoParts(text, audioParts)
|
|
totalParts := len(parts)
|
|
|
|
// Generate part 0 synchronously so the browser can start playing immediately.
|
|
if err := s.generateAudioPart(r.Context(), slug, n, voice, speed, 0, parts[0]); err != nil {
|
|
s.log.Error("part 0 generation failed", "err", err)
|
|
http.Error(w, `{"error":"part 0 generation failed"}`, http.StatusBadGateway)
|
|
return
|
|
}
|
|
|
|
if totalParts == 1 {
|
|
// Only one part — rename it to the final path directly.
|
|
partPath := s.writer.AudioPartPath(slug, n, voice, speed, 0)
|
|
if err := os.Rename(partPath, audioPath); err != nil {
|
|
http.Error(w, `{"error":"failed to save audio"}`, http.StatusInternalServerError)
|
|
return
|
|
}
|
|
s.log.Info("audio generated (single part)", "slug", slug, "chapter", n)
|
|
s.writeAudioResponse(w, slug, n, voice, speed, 1, true)
|
|
return
|
|
}
|
|
|
|
// Return part-0 URL immediately; generate the rest in the background.
|
|
s.writeAudioResponse(w, slug, n, voice, speed, totalParts, false)
|
|
|
|
// Background: generate parts 1…N then merge.
|
|
go func() {
|
|
bgCtx := context.Background()
|
|
for p := 1; p < totalParts; p++ {
|
|
if err := s.generateAudioPart(bgCtx, slug, n, voice, speed, p, parts[p]); err != nil {
|
|
s.log.Error("background part generation failed", "part", p, "err", err)
|
|
return
|
|
}
|
|
}
|
|
if err := s.mergeAudioParts(slug, n, voice, speed, totalParts); err != nil {
|
|
s.log.Error("audio merge failed", "slug", slug, "chapter", n, "err", err)
|
|
return
|
|
}
|
|
s.log.Info("audio merged", "slug", slug, "chapter", n, "parts", totalParts)
|
|
}()
|
|
}
|
|
|
|
// splitTextIntoParts divides text (paragraphs separated by blank lines) into
|
|
// at most n equal-ish chunks. Returns at least 1 element.
|
|
func splitTextIntoParts(text string, n int) []string {
|
|
// Split into paragraphs on blank lines.
|
|
raw := strings.Split(text, "\n\n")
|
|
var paras []string
|
|
for _, p := range raw {
|
|
p = strings.TrimSpace(p)
|
|
if p != "" {
|
|
paras = append(paras, p)
|
|
}
|
|
}
|
|
if len(paras) == 0 {
|
|
return []string{text}
|
|
}
|
|
if n > len(paras) {
|
|
n = len(paras)
|
|
}
|
|
if n < 1 {
|
|
n = 1
|
|
}
|
|
|
|
chunks := make([]string, n)
|
|
chunkSize := (len(paras) + n - 1) / n // ceiling division
|
|
for i := 0; i < n; i++ {
|
|
start := i * chunkSize
|
|
end := start + chunkSize
|
|
if start >= len(paras) {
|
|
// Fewer paragraphs than requested parts: return what we have.
|
|
chunks = chunks[:i]
|
|
break
|
|
}
|
|
if end > len(paras) {
|
|
end = len(paras)
|
|
}
|
|
chunks[i] = strings.Join(paras[start:end], "\n\n")
|
|
}
|
|
if len(chunks) == 0 {
|
|
return []string{text}
|
|
}
|
|
return chunks
|
|
}
|
|
|
|
// generateAudioPart calls Kokoro for a single text chunk and writes the result
|
|
// atomically to AudioPartPath(…, part).
|
|
func (s *Server) generateAudioPart(ctx context.Context, slug string, n int, voice string, speed float64, part int, text string) error {
|
|
reqBody, _ := json.Marshal(map[string]interface{}{
|
|
"model": "kokoro",
|
|
"input": text,
|
|
"voice": voice,
|
|
"response_format": "mp3",
|
|
"speed": speed,
|
|
"stream": false,
|
|
})
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodPost,
|
|
s.kokoroURL+"/v1/audio/speech", bytes.NewReader(reqBody))
|
|
if err != nil {
|
|
return fmt.Errorf("build request: %w", err)
|
|
}
|
|
req.Header.Set("Content-Type", "application/json")
|
|
|
|
resp, err := http.DefaultClient.Do(req)
|
|
if err != nil {
|
|
return fmt.Errorf("kokoro request: %w", err)
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
if resp.StatusCode != http.StatusOK {
|
|
body, _ := io.ReadAll(resp.Body)
|
|
return fmt.Errorf("kokoro status %d: %s", resp.StatusCode, string(body))
|
|
}
|
|
|
|
partPath := s.writer.AudioPartPath(slug, n, voice, speed, part)
|
|
tmpPath := partPath + ".tmp"
|
|
f, err := os.Create(tmpPath)
|
|
if err != nil {
|
|
return fmt.Errorf("create temp file: %w", err)
|
|
}
|
|
if _, err := io.Copy(f, resp.Body); err != nil {
|
|
f.Close()
|
|
os.Remove(tmpPath)
|
|
return fmt.Errorf("write audio: %w", err)
|
|
}
|
|
f.Close()
|
|
if err := os.Rename(tmpPath, partPath); err != nil {
|
|
os.Remove(tmpPath)
|
|
return fmt.Errorf("rename temp file: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// mergeAudioParts concatenates totalParts part files in order into AudioPath,
|
|
// then removes the individual part files.
|
|
func (s *Server) mergeAudioParts(slug string, n int, voice string, speed float64, totalParts int) error {
|
|
audioPath := s.writer.AudioPath(slug, n, voice, speed)
|
|
tmpPath := audioPath + ".tmp"
|
|
|
|
out, err := os.Create(tmpPath)
|
|
if err != nil {
|
|
return fmt.Errorf("create merged temp: %w", err)
|
|
}
|
|
|
|
for p := 0; p < totalParts; p++ {
|
|
partPath := s.writer.AudioPartPath(slug, n, voice, speed, p)
|
|
data, err := os.ReadFile(partPath)
|
|
if err != nil {
|
|
out.Close()
|
|
os.Remove(tmpPath)
|
|
return fmt.Errorf("read part %d: %w", p, err)
|
|
}
|
|
if _, err := out.Write(data); err != nil {
|
|
out.Close()
|
|
os.Remove(tmpPath)
|
|
return fmt.Errorf("write merged part %d: %w", p, err)
|
|
}
|
|
}
|
|
out.Close()
|
|
|
|
if err := os.Rename(tmpPath, audioPath); err != nil {
|
|
os.Remove(tmpPath)
|
|
return fmt.Errorf("rename merged: %w", err)
|
|
}
|
|
|
|
// Clean up part files (best-effort).
|
|
for p := 0; p < totalParts; p++ {
|
|
os.Remove(s.writer.AudioPartPath(slug, n, voice, speed, p))
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *Server) writeAudioResponse(w http.ResponseWriter, slug string, n int, voice string, speed float64, parts int, merged bool) {
|
|
var url string
|
|
if merged {
|
|
url = fmt.Sprintf("/ui/audio-file/%s/%d?voice=%s&speed=%.1f", slug, n, voice, speed)
|
|
} else {
|
|
url = fmt.Sprintf("/ui/audio-file/%s/%d/part/0?voice=%s&speed=%.1f", slug, n, voice, speed)
|
|
}
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_ = json.NewEncoder(w).Encode(map[string]interface{}{
|
|
"url": url,
|
|
"parts": parts,
|
|
"merged": merged,
|
|
})
|
|
}
|
|
|
|
// writeAudioURL is kept for backward compatibility (used by dedup waiters).
|
|
func (s *Server) writeAudioURL(w http.ResponseWriter, slug string, n int, voice string, speed float64) {
|
|
s.writeAudioResponse(w, slug, n, voice, speed, 1, true)
|
|
}
|
|
|
|
// handleAudioStatus handles GET /ui/audio/{slug}/{n}/status.
|
|
// Returns {"merged":true/false,"url":"..."} so the browser can poll for the
|
|
// merged file after receiving a parts response from handleAudioGenerate.
|
|
func (s *Server) handleAudioStatus(w http.ResponseWriter, r *http.Request) {
|
|
slug := r.PathValue("slug")
|
|
n, err := strconv.Atoi(r.PathValue("n"))
|
|
if err != nil || n < 1 {
|
|
http.Error(w, `{"error":"invalid chapter"}`, http.StatusBadRequest)
|
|
return
|
|
}
|
|
voice := r.URL.Query().Get("voice")
|
|
if voice == "" {
|
|
voice = s.kokoroVoice
|
|
}
|
|
speedStr := r.URL.Query().Get("speed")
|
|
speed := 1.0
|
|
if speedStr != "" {
|
|
if v, err := strconv.ParseFloat(speedStr, 64); err == nil && v > 0 {
|
|
speed = v
|
|
}
|
|
}
|
|
|
|
audioPath := s.writer.AudioPath(slug, n, voice, speed)
|
|
merged := false
|
|
if _, err := os.Stat(audioPath); err == nil {
|
|
merged = true
|
|
}
|
|
|
|
mergedURL := fmt.Sprintf("/ui/audio-file/%s/%d?voice=%s&speed=%.1f", slug, n, voice, speed)
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_ = json.NewEncoder(w).Encode(map[string]interface{}{
|
|
"merged": merged,
|
|
"url": mergedURL,
|
|
})
|
|
}
|
|
|
|
// handleAudioFile handles GET /ui/audio-file/{slug}/{n}.
|
|
// Serves the cached merged MP3 file identified by voice and speed query params.
|
|
func (s *Server) handleAudioFile(w http.ResponseWriter, r *http.Request) {
|
|
slug := r.PathValue("slug")
|
|
n, err := strconv.Atoi(r.PathValue("n"))
|
|
if err != nil || n < 1 {
|
|
http.NotFound(w, r)
|
|
return
|
|
}
|
|
voice := r.URL.Query().Get("voice")
|
|
if voice == "" {
|
|
voice = s.kokoroVoice
|
|
}
|
|
speedStr := r.URL.Query().Get("speed")
|
|
speed := 1.0
|
|
if speedStr != "" {
|
|
if v, err := strconv.ParseFloat(speedStr, 64); err == nil && v > 0 {
|
|
speed = v
|
|
}
|
|
}
|
|
|
|
audioPath := s.writer.AudioPath(slug, n, voice, speed)
|
|
if _, err := os.Stat(audioPath); err != nil {
|
|
http.NotFound(w, r)
|
|
return
|
|
}
|
|
|
|
w.Header().Set("Content-Type", "audio/mpeg")
|
|
w.Header().Set("Cache-Control", "public, max-age=86400")
|
|
http.ServeFile(w, r, audioPath)
|
|
}
|
|
|
|
// handleAudioFilePart handles GET /ui/audio-file/{slug}/{n}/part/{p}.
|
|
// Serves a specific MP3 part file.
|
|
func (s *Server) handleAudioFilePart(w http.ResponseWriter, r *http.Request) {
|
|
slug := r.PathValue("slug")
|
|
n, err := strconv.Atoi(r.PathValue("n"))
|
|
if err != nil || n < 1 {
|
|
http.NotFound(w, r)
|
|
return
|
|
}
|
|
p, err := strconv.Atoi(r.PathValue("p"))
|
|
if err != nil || p < 0 {
|
|
http.NotFound(w, r)
|
|
return
|
|
}
|
|
voice := r.URL.Query().Get("voice")
|
|
if voice == "" {
|
|
voice = s.kokoroVoice
|
|
}
|
|
speedStr := r.URL.Query().Get("speed")
|
|
speed := 1.0
|
|
if speedStr != "" {
|
|
if v, err := strconv.ParseFloat(speedStr, 64); err == nil && v > 0 {
|
|
speed = v
|
|
}
|
|
}
|
|
|
|
partPath := s.writer.AudioPartPath(slug, n, voice, speed, p)
|
|
if _, err := os.Stat(partPath); err != nil {
|
|
http.NotFound(w, r)
|
|
return
|
|
}
|
|
|
|
w.Header().Set("Content-Type", "audio/mpeg")
|
|
w.Header().Set("Cache-Control", "no-store")
|
|
http.ServeFile(w, r, partPath)
|
|
}
|
|
|
|
func (s *Server) handleScrapeCatalogue(w http.ResponseWriter, r *http.Request) {
|
|
cfg := s.oCfg
|
|
cfg.SingleBookURL = "" // full catalogue
|
|
|
|
s.runAsync(w, cfg)
|
|
}
|
|
|
|
func (s *Server) handleScrapeBook(w http.ResponseWriter, r *http.Request) {
|
|
var body struct {
|
|
URL string `json:"url"`
|
|
}
|
|
if err := json.NewDecoder(r.Body).Decode(&body); err != nil || body.URL == "" {
|
|
http.Error(w, `{"error":"request body must be JSON with \"url\" field"}`, http.StatusBadRequest)
|
|
return
|
|
}
|
|
|
|
cfg := s.oCfg
|
|
cfg.SingleBookURL = body.URL
|
|
|
|
s.runAsync(w, cfg)
|
|
}
|
|
|
|
// runAsync launches an orchestrator in the background and returns 202 Accepted.
|
|
// Only one scrape job runs at a time; concurrent requests receive 409 Conflict.
|
|
func (s *Server) runAsync(w http.ResponseWriter, cfg orchestrator.Config) {
|
|
s.mu.Lock()
|
|
if s.running {
|
|
s.mu.Unlock()
|
|
http.Error(w, `{"error":"a scrape job is already running"}`, http.StatusConflict)
|
|
return
|
|
}
|
|
s.running = true
|
|
s.mu.Unlock()
|
|
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.WriteHeader(http.StatusAccepted)
|
|
_ = json.NewEncoder(w).Encode(map[string]string{"status": "accepted"})
|
|
|
|
go func() {
|
|
defer func() {
|
|
s.mu.Lock()
|
|
s.running = false
|
|
s.mu.Unlock()
|
|
}()
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 24*time.Hour)
|
|
defer cancel()
|
|
|
|
o := orchestrator.New(cfg, s.novel, s.log)
|
|
if err := o.Run(ctx); err != nil {
|
|
s.log.Error("scrape job failed", "err", fmt.Sprintf("%v", err))
|
|
}
|
|
}()
|
|
}
|