583 lines
18 KiB
Go
583 lines
18 KiB
Go
// Package server exposes the scraper as an HTTP service.
|
|
//
|
|
// Endpoints:
|
|
//
|
|
// POST /scrape — enqueue a full catalogue scrape
|
|
// POST /scrape/book — enqueue a single-book scrape (JSON body: {"url":"..."})
|
|
// GET /health — liveness probe
|
|
package server
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"crypto/rand"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"log/slog"
|
|
"net/http"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/libnovel/scraper/internal/orchestrator"
|
|
"github.com/libnovel/scraper/internal/scraper"
|
|
"github.com/libnovel/scraper/internal/storage"
|
|
)
|
|
|
|
// Server wraps an HTTP mux with the scraping endpoints.
|
|
type Server struct {
|
|
addr string
|
|
oCfg orchestrator.Config
|
|
novel scraper.NovelScraper
|
|
log *slog.Logger
|
|
store storage.Store
|
|
mu sync.Mutex
|
|
running bool
|
|
rankingRunning bool
|
|
kokoroURL string // Kokoro-FastAPI base URL, e.g. http://kokoro:8880
|
|
kokoroVoice string // default voice, e.g. af_bella
|
|
|
|
// voiceMu guards cachedVoices.
|
|
voiceMu sync.RWMutex
|
|
cachedVoices []string // populated on first request from Kokoro /v1/audio/voices
|
|
|
|
// audioMu guards audioInFlight only.
|
|
// Completed audio filenames are persisted to the Store (PocketBase).
|
|
// audioInFlight deduplicates concurrent generation requests for the same key.
|
|
audioMu sync.Mutex
|
|
audioInFlight map[string]chan struct{} // cacheKey → closed when done
|
|
}
|
|
|
|
// New creates a new Server.
|
|
func New(addr string, oCfg orchestrator.Config, novel scraper.NovelScraper, log *slog.Logger, store storage.Store, kokoroURL, kokoroVoice string) *Server {
|
|
return &Server{
|
|
addr: addr,
|
|
oCfg: oCfg,
|
|
novel: novel,
|
|
log: log,
|
|
store: store,
|
|
kokoroURL: kokoroURL,
|
|
kokoroVoice: kokoroVoice,
|
|
audioInFlight: make(map[string]chan struct{}),
|
|
}
|
|
}
|
|
|
|
// voices returns the list of available Kokoro voices. On the first call it
|
|
// fetches GET /v1/audio/voices from the Kokoro service and caches the result.
|
|
// If the fetch fails (Kokoro not up yet, network error, etc.) it falls back to
|
|
// the hardcoded kokoroVoices list so the UI is never empty.
|
|
func (s *Server) voices() []string {
|
|
s.voiceMu.RLock()
|
|
cached := s.cachedVoices
|
|
s.voiceMu.RUnlock()
|
|
if len(cached) > 0 {
|
|
return cached
|
|
}
|
|
|
|
if s.kokoroURL != "" {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
defer cancel()
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, s.kokoroURL+"/v1/audio/voices", nil)
|
|
if err == nil {
|
|
req.Header.Set("Accept", "application/json")
|
|
resp, err := http.DefaultClient.Do(req)
|
|
if err == nil {
|
|
defer resp.Body.Close()
|
|
var payload struct {
|
|
Voices []string `json:"voices"`
|
|
}
|
|
if resp.StatusCode == http.StatusOK && json.NewDecoder(resp.Body).Decode(&payload) == nil && len(payload.Voices) > 0 {
|
|
s.voiceMu.Lock()
|
|
s.cachedVoices = payload.Voices
|
|
s.voiceMu.Unlock()
|
|
s.log.Info("fetched kokoro voices", "count", len(payload.Voices))
|
|
return payload.Voices
|
|
}
|
|
}
|
|
}
|
|
s.log.Warn("could not fetch kokoro voices, using built-in list")
|
|
}
|
|
|
|
return kokoroVoices
|
|
}
|
|
|
|
// ListenAndServe starts the HTTP server and blocks until the provided context
|
|
// is cancelled.
|
|
func (s *Server) ListenAndServe(ctx context.Context) error {
|
|
mux := http.NewServeMux()
|
|
mux.HandleFunc("GET /health", s.handleHealth)
|
|
mux.HandleFunc("POST /scrape", s.handleScrapeCatalogue)
|
|
mux.HandleFunc("POST /scrape/book", s.handleScrapeBook)
|
|
// Progress API
|
|
mux.HandleFunc("GET /api/progress", s.handleGetProgress)
|
|
mux.HandleFunc("POST /api/progress/{slug}", s.handleSetProgress)
|
|
mux.HandleFunc("DELETE /api/progress/{slug}", s.handleDeleteProgress)
|
|
// UI routes
|
|
mux.HandleFunc("GET /", s.handleHome)
|
|
mux.HandleFunc("GET /scrape", s.handleScrape)
|
|
mux.HandleFunc("GET /ranking", s.handleRanking)
|
|
mux.HandleFunc("POST /ranking/refresh", s.handleRankingRefresh)
|
|
mux.HandleFunc("GET /ranking/view", s.handleRankingView)
|
|
mux.HandleFunc("GET /books/{slug}", s.handleBook)
|
|
mux.HandleFunc("GET /books/{slug}/chapters/{n}", s.handleChapter)
|
|
mux.HandleFunc("GET /books/{slug}/chapters-page", s.handleBookChaptersPage)
|
|
mux.HandleFunc("POST /ui/scrape/book", s.handleUIScrapeBook)
|
|
mux.HandleFunc("GET /ui/scrape/status", s.handleUIScrapeStatus)
|
|
mux.HandleFunc("GET /ui/ranking/status", s.handleRankingStatus)
|
|
// Plain-text chapter content for browser-side TTS
|
|
mux.HandleFunc("GET /ui/chapter-text/{slug}/{n}", s.handleChapterText)
|
|
// Server-side audio generation via Kokoro /v1/audio/speech.
|
|
// Generation can take several minutes, so wrap in its own timeout handler.
|
|
audioGenHandler := http.TimeoutHandler(
|
|
http.HandlerFunc(s.handleAudioGenerate),
|
|
10*time.Minute,
|
|
`{"error":"audio generation timed out"}`,
|
|
)
|
|
mux.Handle("POST /ui/audio/{slug}/{n}", audioGenHandler)
|
|
// Proxy route: fetches the generated file from Kokoro /v1/download/{filename}.
|
|
mux.HandleFunc("GET /ui/audio-proxy/{slug}/{n}", s.handleAudioProxy)
|
|
|
|
srv := &http.Server{
|
|
Addr: s.addr,
|
|
Handler: mux,
|
|
ReadTimeout: 15 * time.Second,
|
|
WriteTimeout: 60 * time.Second,
|
|
IdleTimeout: 60 * time.Second,
|
|
}
|
|
|
|
errCh := make(chan error, 1)
|
|
go func() { errCh <- srv.ListenAndServe() }()
|
|
|
|
s.log.Info("HTTP server listening", "addr", s.addr)
|
|
|
|
select {
|
|
case <-ctx.Done():
|
|
shutCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
defer cancel()
|
|
return srv.Shutdown(shutCtx)
|
|
case err := <-errCh:
|
|
return err
|
|
}
|
|
}
|
|
|
|
func (s *Server) handleHealth(w http.ResponseWriter, _ *http.Request) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_ = json.NewEncoder(w).Encode(map[string]string{"status": "ok"})
|
|
}
|
|
|
|
// ─── Session cookie helpers ───────────────────────────────────────────────────
|
|
|
|
const sessionCookieName = "libnovel_session"
|
|
|
|
// sessionID returns the session ID from the request cookie, or "" if absent.
|
|
func sessionID(r *http.Request) string {
|
|
c, err := r.Cookie(sessionCookieName)
|
|
if err != nil {
|
|
return ""
|
|
}
|
|
return c.Value
|
|
}
|
|
|
|
// newSessionID generates a random 16-byte hex session ID.
|
|
func newSessionID() (string, error) {
|
|
b := make([]byte, 16)
|
|
if _, err := rand.Read(b); err != nil {
|
|
return "", err
|
|
}
|
|
return hex.EncodeToString(b), nil
|
|
}
|
|
|
|
// ensureSession issues a new session cookie if the request does not already
|
|
// carry one, and returns the session ID (either existing or newly issued).
|
|
func ensureSession(w http.ResponseWriter, r *http.Request) string {
|
|
if id := sessionID(r); id != "" {
|
|
return id
|
|
}
|
|
id, err := newSessionID()
|
|
if err != nil {
|
|
// Very unlikely, but fall back to a timestamp-based ID.
|
|
id = fmt.Sprintf("fallback-%d", time.Now().UnixNano())
|
|
}
|
|
http.SetCookie(w, &http.Cookie{
|
|
Name: sessionCookieName,
|
|
Value: id,
|
|
Path: "/",
|
|
HttpOnly: true,
|
|
SameSite: http.SameSiteLaxMode,
|
|
MaxAge: 365 * 24 * 60 * 60, // 1 year
|
|
})
|
|
return id
|
|
}
|
|
|
|
// ─── Reading progress API ─────────────────────────────────────────────────────
|
|
|
|
// handleGetProgress handles GET /api/progress.
|
|
// Returns JSON: {"slug": chapterNum, ...} merged with {"slug_ts": timestampMs, ...}
|
|
func (s *Server) handleGetProgress(w http.ResponseWriter, r *http.Request) {
|
|
sid := ensureSession(w, r)
|
|
entries, err := s.store.AllProgress(r.Context(), sid)
|
|
if err != nil {
|
|
s.log.Error("AllProgress failed", "err", err)
|
|
entries = nil
|
|
}
|
|
|
|
progress := make(map[string]interface{}, len(entries)*2)
|
|
for _, p := range entries {
|
|
progress[p.Slug] = p.Chapter
|
|
progress[p.Slug+"_ts"] = p.UpdatedAt.UnixMilli()
|
|
}
|
|
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_ = json.NewEncoder(w).Encode(progress)
|
|
}
|
|
|
|
// handleSetProgress handles POST /api/progress/{slug}.
|
|
// Body: {"chapter": N}
|
|
func (s *Server) handleSetProgress(w http.ResponseWriter, r *http.Request) {
|
|
sid := ensureSession(w, r)
|
|
slug := r.PathValue("slug")
|
|
if slug == "" {
|
|
http.Error(w, `{"error":"missing slug"}`, http.StatusBadRequest)
|
|
return
|
|
}
|
|
|
|
var body struct {
|
|
Chapter int `json:"chapter"`
|
|
}
|
|
if err := json.NewDecoder(r.Body).Decode(&body); err != nil || body.Chapter < 1 {
|
|
http.Error(w, `{"error":"invalid body"}`, http.StatusBadRequest)
|
|
return
|
|
}
|
|
|
|
p := storage.ReadingProgress{
|
|
Slug: slug,
|
|
Chapter: body.Chapter,
|
|
UpdatedAt: time.Now(),
|
|
}
|
|
if err := s.store.SetProgress(r.Context(), sid, p); err != nil {
|
|
s.log.Error("SetProgress failed", "slug", slug, "err", err)
|
|
http.Error(w, `{"error":"store error"}`, http.StatusInternalServerError)
|
|
return
|
|
}
|
|
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_ = json.NewEncoder(w).Encode(map[string]string{})
|
|
}
|
|
|
|
// handleDeleteProgress handles DELETE /api/progress/{slug}.
|
|
func (s *Server) handleDeleteProgress(w http.ResponseWriter, r *http.Request) {
|
|
sid := ensureSession(w, r)
|
|
slug := r.PathValue("slug")
|
|
if slug == "" {
|
|
http.Error(w, `{"error":"missing slug"}`, http.StatusBadRequest)
|
|
return
|
|
}
|
|
|
|
if err := s.store.DeleteProgress(r.Context(), sid, slug); err != nil {
|
|
s.log.Error("DeleteProgress failed", "slug", slug, "err", err)
|
|
// Non-fatal — treat as success.
|
|
}
|
|
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_ = json.NewEncoder(w).Encode(map[string]string{})
|
|
}
|
|
|
|
// handleChapterText returns the plain text of a chapter (markdown stripped)
|
|
// for browser-side TTS. The browser POSTs this directly to Kokoro-FastAPI.
|
|
func (s *Server) handleChapterText(w http.ResponseWriter, r *http.Request) {
|
|
slug := r.PathValue("slug")
|
|
n, err := strconv.Atoi(r.PathValue("n"))
|
|
if err != nil || n < 1 {
|
|
http.NotFound(w, r)
|
|
return
|
|
}
|
|
raw, err := s.store.ReadChapter(r.Context(), slug, n)
|
|
if err != nil {
|
|
http.NotFound(w, r)
|
|
return
|
|
}
|
|
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
|
|
w.Header().Set("Cache-Control", "no-store")
|
|
fmt.Fprint(w, stripMarkdown(raw))
|
|
}
|
|
|
|
// ─── Audio generation via Kokoro /v1/audio/speech ────────────────────────────
|
|
//
|
|
// handleAudioGenerate handles POST /ui/audio/{slug}/{n}.
|
|
//
|
|
// It calls Kokoro's POST /v1/audio/speech with return_download_link=true.
|
|
// Kokoro generates the audio, saves it to its own temp storage, and returns
|
|
// the download filename in the X-Download-Path response header.
|
|
// We cache that filename (in memory, keyed by slug/chapter/voice/speed) and
|
|
// return a proxy URL that the browser sets as audio.src.
|
|
//
|
|
// On a cache hit the proxy URL is returned immediately without re-generating.
|
|
// Concurrent requests for the same key are deduplicated.
|
|
func (s *Server) handleAudioGenerate(w http.ResponseWriter, r *http.Request) {
|
|
slug := r.PathValue("slug")
|
|
n, err := strconv.Atoi(r.PathValue("n"))
|
|
if err != nil || n < 1 {
|
|
http.Error(w, `{"error":"invalid chapter"}`, http.StatusBadRequest)
|
|
return
|
|
}
|
|
|
|
// Parse optional voice/speed from JSON body.
|
|
voice := s.kokoroVoice
|
|
speed := 1.0
|
|
var body struct {
|
|
Voice string `json:"voice"`
|
|
Speed float64 `json:"speed"`
|
|
}
|
|
if r.Body != nil {
|
|
_ = json.NewDecoder(r.Body).Decode(&body)
|
|
}
|
|
if body.Voice != "" {
|
|
voice = body.Voice
|
|
}
|
|
if body.Speed > 0 {
|
|
speed = body.Speed
|
|
}
|
|
|
|
cacheKey := fmt.Sprintf("%s/%d/%s/%.2f", slug, n, voice, speed)
|
|
|
|
// Fast path: already generated (check persistent store first).
|
|
if filename, ok := s.store.GetAudioCache(r.Context(), cacheKey); ok {
|
|
s.writeAudioResponse(w, slug, n, voice, speed, filename)
|
|
return
|
|
}
|
|
|
|
// Deduplicate concurrent generation for the same key.
|
|
s.audioMu.Lock()
|
|
if ch, ok := s.audioInFlight[cacheKey]; ok {
|
|
s.audioMu.Unlock()
|
|
select {
|
|
case <-ch:
|
|
case <-r.Context().Done():
|
|
http.Error(w, `{"error":"request cancelled"}`, http.StatusServiceUnavailable)
|
|
return
|
|
}
|
|
// Check store again after waiting.
|
|
if filename, ok := s.store.GetAudioCache(r.Context(), cacheKey); ok {
|
|
s.writeAudioResponse(w, slug, n, voice, speed, filename)
|
|
} else {
|
|
http.Error(w, `{"error":"audio generation failed"}`, http.StatusInternalServerError)
|
|
}
|
|
return
|
|
}
|
|
ch := make(chan struct{})
|
|
s.audioInFlight[cacheKey] = ch
|
|
s.audioMu.Unlock()
|
|
|
|
defer func() {
|
|
s.audioMu.Lock()
|
|
delete(s.audioInFlight, cacheKey)
|
|
s.audioMu.Unlock()
|
|
close(ch)
|
|
}()
|
|
|
|
// Load and validate chapter text.
|
|
raw, err := s.store.ReadChapter(r.Context(), slug, n)
|
|
if err != nil {
|
|
http.Error(w, `{"error":"chapter not found"}`, http.StatusNotFound)
|
|
return
|
|
}
|
|
text := stripMarkdown(raw)
|
|
if text == "" {
|
|
http.Error(w, `{"error":"chapter text is empty"}`, http.StatusUnprocessableEntity)
|
|
return
|
|
}
|
|
if s.kokoroURL == "" {
|
|
http.Error(w, `{"error":"kokoro not configured"}`, http.StatusServiceUnavailable)
|
|
return
|
|
}
|
|
|
|
// Call Kokoro POST /v1/audio/speech with return_download_link=true.
|
|
// Kokoro saves the generated audio to its own temp storage and returns the
|
|
// download path in the X-Download-Path response header.
|
|
filename, err := s.generateSpeech(r.Context(), text, voice, speed)
|
|
if err != nil {
|
|
s.log.Error("kokoro speech generation failed", "slug", slug, "chapter", n, "err", err)
|
|
http.Error(w, `{"error":"speech generation failed"}`, http.StatusBadGateway)
|
|
return
|
|
}
|
|
|
|
_ = s.store.SetAudioCache(r.Context(), cacheKey, filename)
|
|
|
|
s.log.Info("audio generated", "slug", slug, "chapter", n, "filename", filename)
|
|
s.writeAudioResponse(w, slug, n, voice, speed, filename)
|
|
}
|
|
|
|
// generateSpeech calls POST /v1/audio/speech on Kokoro with return_download_link=true
|
|
// and returns the filename from the X-Download-Path response header.
|
|
func (s *Server) generateSpeech(ctx context.Context, text, voice string, speed float64) (string, error) {
|
|
reqBody, _ := json.Marshal(map[string]interface{}{
|
|
"model": "kokoro",
|
|
"input": text,
|
|
"voice": voice,
|
|
"response_format": "mp3",
|
|
"speed": speed,
|
|
"stream": false,
|
|
"return_download_link": true,
|
|
})
|
|
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodPost,
|
|
s.kokoroURL+"/v1/audio/speech", bytes.NewReader(reqBody))
|
|
if err != nil {
|
|
return "", fmt.Errorf("build request: %w", err)
|
|
}
|
|
req.Header.Set("Content-Type", "application/json")
|
|
|
|
resp, err := http.DefaultClient.Do(req)
|
|
if err != nil {
|
|
return "", fmt.Errorf("kokoro request: %w", err)
|
|
}
|
|
defer resp.Body.Close()
|
|
// Drain body so the connection can be reused.
|
|
_, _ = io.Copy(io.Discard, resp.Body)
|
|
|
|
if resp.StatusCode != http.StatusOK {
|
|
return "", fmt.Errorf("kokoro status %d", resp.StatusCode)
|
|
}
|
|
|
|
// X-Download-Path is e.g. "/download/speech_abc123.mp3"
|
|
dlPath := resp.Header.Get("X-Download-Path")
|
|
if dlPath == "" {
|
|
return "", fmt.Errorf("kokoro did not return X-Download-Path header")
|
|
}
|
|
|
|
// Extract just the filename from the path.
|
|
filename := dlPath
|
|
if idx := strings.LastIndex(dlPath, "/"); idx >= 0 {
|
|
filename = dlPath[idx+1:]
|
|
}
|
|
if filename == "" {
|
|
return "", fmt.Errorf("empty filename in X-Download-Path: %q", dlPath)
|
|
}
|
|
return filename, nil
|
|
}
|
|
|
|
// writeAudioResponse writes the JSON response for a generated audio chapter.
|
|
// The URL points to our proxy handler which fetches from Kokoro on demand.
|
|
func (s *Server) writeAudioResponse(w http.ResponseWriter, slug string, n int, voice string, speed float64, filename string) {
|
|
proxyURL := fmt.Sprintf("/ui/audio-proxy/%s/%d?voice=%s&speed=%.1f", slug, n, voice, speed)
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_ = json.NewEncoder(w).Encode(map[string]interface{}{
|
|
"url": proxyURL,
|
|
"filename": filename,
|
|
})
|
|
}
|
|
|
|
// handleAudioProxy handles GET /ui/audio-proxy/{slug}/{n}.
|
|
// It looks up the Kokoro download filename for this chapter (voice/speed) and
|
|
// proxies GET /v1/download/{filename} from the Kokoro server back to the browser.
|
|
func (s *Server) handleAudioProxy(w http.ResponseWriter, r *http.Request) {
|
|
slug := r.PathValue("slug")
|
|
n, err := strconv.Atoi(r.PathValue("n"))
|
|
if err != nil || n < 1 {
|
|
http.NotFound(w, r)
|
|
return
|
|
}
|
|
voice := r.URL.Query().Get("voice")
|
|
if voice == "" {
|
|
voice = s.kokoroVoice
|
|
}
|
|
speedStr := r.URL.Query().Get("speed")
|
|
speed := 1.0
|
|
if speedStr != "" {
|
|
if v, err := strconv.ParseFloat(speedStr, 64); err == nil && v > 0 {
|
|
speed = v
|
|
}
|
|
}
|
|
|
|
cacheKey := fmt.Sprintf("%s/%d/%s/%.2f", slug, n, voice, speed)
|
|
filename, ok := s.store.GetAudioCache(r.Context(), cacheKey)
|
|
if !ok {
|
|
http.Error(w, "audio not generated yet", http.StatusNotFound)
|
|
return
|
|
}
|
|
|
|
kokoroURL := s.kokoroURL + "/v1/download/" + filename
|
|
req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, kokoroURL, nil)
|
|
if err != nil {
|
|
http.Error(w, "failed to build proxy request", http.StatusInternalServerError)
|
|
return
|
|
}
|
|
|
|
resp, err := http.DefaultClient.Do(req)
|
|
if err != nil {
|
|
http.Error(w, "kokoro download failed", http.StatusBadGateway)
|
|
return
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
if resp.StatusCode != http.StatusOK {
|
|
http.Error(w, fmt.Sprintf("kokoro returned %d", resp.StatusCode), http.StatusBadGateway)
|
|
return
|
|
}
|
|
|
|
w.Header().Set("Content-Type", "audio/mpeg")
|
|
w.Header().Set("Cache-Control", "public, max-age=3600")
|
|
if cl := resp.Header.Get("Content-Length"); cl != "" {
|
|
w.Header().Set("Content-Length", cl)
|
|
}
|
|
_, _ = io.Copy(w, resp.Body)
|
|
}
|
|
|
|
func (s *Server) handleScrapeCatalogue(w http.ResponseWriter, r *http.Request) {
|
|
cfg := s.oCfg
|
|
cfg.SingleBookURL = "" // full catalogue
|
|
|
|
s.runAsync(w, cfg)
|
|
}
|
|
|
|
func (s *Server) handleScrapeBook(w http.ResponseWriter, r *http.Request) {
|
|
var body struct {
|
|
URL string `json:"url"`
|
|
}
|
|
if err := json.NewDecoder(r.Body).Decode(&body); err != nil || body.URL == "" {
|
|
http.Error(w, `{"error":"request body must be JSON with \"url\" field"}`, http.StatusBadRequest)
|
|
return
|
|
}
|
|
|
|
cfg := s.oCfg
|
|
cfg.SingleBookURL = body.URL
|
|
|
|
s.runAsync(w, cfg)
|
|
}
|
|
|
|
// runAsync launches an orchestrator in the background and returns 202 Accepted.
|
|
// Only one scrape job runs at a time; concurrent requests receive 409 Conflict.
|
|
func (s *Server) runAsync(w http.ResponseWriter, cfg orchestrator.Config) {
|
|
s.mu.Lock()
|
|
if s.running {
|
|
s.mu.Unlock()
|
|
http.Error(w, `{"error":"a scrape job is already running"}`, http.StatusConflict)
|
|
return
|
|
}
|
|
s.running = true
|
|
s.mu.Unlock()
|
|
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.WriteHeader(http.StatusAccepted)
|
|
_ = json.NewEncoder(w).Encode(map[string]string{"status": "accepted"})
|
|
|
|
go func() {
|
|
defer func() {
|
|
s.mu.Lock()
|
|
s.running = false
|
|
s.mu.Unlock()
|
|
}()
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 24*time.Hour)
|
|
defer cancel()
|
|
|
|
o := orchestrator.New(cfg, s.novel, s.log, s.store)
|
|
if err := o.Run(ctx); err != nil {
|
|
s.log.Error("scrape job failed", "err", fmt.Sprintf("%v", err))
|
|
}
|
|
}()
|
|
}
|