refactor: audit, split server.go, add unit tests, and fix latent bugs
- Remove dead code: browser cdp/content_scrape strategies, writer package, printUsage, downloadAndStoreCoverCLI in main.go - Fix bugs: defer-in-loop in pocketbase deleteWhere, listAll() pagination hard cap removed, splitChapterTitle off-by-one in date extraction - Split server.go (~1700 lines) into focused handler files: handlers_audio, handlers_browse, handlers_progress, handlers_ranking, handlers_scrape - Export htmlutil.AttrVal/TextContent/ResolveURL; add storage/coverutil.go to consolidate duplicate helpers - Flatten deeply nested conditionals: voices() early-return guards, ScrapeCatalogue next-link double attr scan, chapterNumberFromKey dead strings.Cut line, splitChapterTitle double-nested unit/suffix loop - Add unit tests: htmlutil (9 funcs), novelfire ScrapeMetadata (3 cases), orchestrator Run (5 cases), storage chapterNumberFromKey/splitChapterTitle (22 cases); all pass with go build/vet/test clean
This commit is contained in:
Admin
@@ -10,10 +10,6 @@
|
||||
//
|
||||
// Environment variables:
|
||||
//
|
||||
// BROWSERLESS_URL Browserless base URL (default: http://localhost:3030)
|
||||
// BROWSERLESS_TOKEN Browserless API token (default: "")
|
||||
// BROWSERLESS_STRATEGY content | scrape | cdp | direct (default: direct; chapter list+text always use direct HTTP)
|
||||
// BROWSERLESS_MAX_CONCURRENT Max simultaneous browser sessions (default: 5)
|
||||
// SCRAPER_WORKERS Chapter goroutine count (default: NumCPU)
|
||||
// SCRAPER_HTTP_ADDR HTTP listen address (default: :8080)
|
||||
// KOKORO_URL Kokoro-FastAPI base URL (default: "")
|
||||
@@ -33,9 +29,7 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"os/signal"
|
||||
@@ -48,6 +42,7 @@ import (
|
||||
"github.com/libnovel/scraper/internal/browser"
|
||||
"github.com/libnovel/scraper/internal/novelfire"
|
||||
"github.com/libnovel/scraper/internal/orchestrator"
|
||||
"github.com/libnovel/scraper/internal/scraper/htmlutil"
|
||||
"github.com/libnovel/scraper/internal/server"
|
||||
"github.com/libnovel/scraper/internal/storage"
|
||||
)
|
||||
@@ -343,7 +338,7 @@ func runSaveBrowse(ctx context.Context, args []string, store storage.Store, log
|
||||
|
||||
// Download cover image in the background (best-effort).
|
||||
if novel.coverURL != "" {
|
||||
go downloadAndStoreCoverCLI(store, log, coverKey, novel.coverURL)
|
||||
go storage.DownloadAndStoreCover(store, log, coverKey, novel.coverURL)
|
||||
}
|
||||
}
|
||||
if len(novels) > 0 {
|
||||
@@ -420,9 +415,9 @@ func parseSaveBrowseListings(htmlBytes []byte, novelFireBase string) []novelList
|
||||
// Extract cover URL from data-src or src on img tags.
|
||||
if cur.coverURL == "" && strings.Contains(trimmed, "<img") {
|
||||
if src := extractAttr(trimmed, "data-src"); src != "" {
|
||||
cur.coverURL = resolveURL(src, novelFireBase)
|
||||
cur.coverURL = htmlutil.ResolveURL(novelFireBase, src)
|
||||
} else if src := extractAttr(trimmed, "src"); src != "" && !strings.Contains(src, "data:") {
|
||||
cur.coverURL = resolveURL(src, novelFireBase)
|
||||
cur.coverURL = htmlutil.ResolveURL(novelFireBase, src)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -465,62 +460,6 @@ func extractAttr(tag, attr string) string {
|
||||
return rest[:end]
|
||||
}
|
||||
|
||||
// resolveURL ensures the URL is absolute, prepending novelFireBase if needed.
|
||||
func resolveURL(src, base string) string {
|
||||
if strings.HasPrefix(src, "http") {
|
||||
return src
|
||||
}
|
||||
return base + src
|
||||
}
|
||||
|
||||
// downloadAndStoreCoverCLI fetches a cover image and stores it in MinIO.
|
||||
// Errors are logged but not propagated — this is a best-effort background task.
|
||||
func downloadAndStoreCoverCLI(store storage.Store, log *slog.Logger, key, imageURL string) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
// Skip if already stored.
|
||||
if _, _, ok, _ := store.GetBrowseAsset(ctx, key); ok {
|
||||
return
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, imageURL, nil)
|
||||
if err != nil {
|
||||
log.Warn("save-browse: cover build request failed", "url", imageURL, "err", err)
|
||||
return
|
||||
}
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; libnovel-scraper/1.0)")
|
||||
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
log.Warn("save-browse: cover fetch failed", "url", imageURL, "err", err)
|
||||
return
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
log.Warn("save-browse: cover non-200", "url", imageURL, "status", resp.StatusCode)
|
||||
return
|
||||
}
|
||||
|
||||
data, readErr := io.ReadAll(resp.Body)
|
||||
if readErr != nil {
|
||||
log.Warn("save-browse: cover read body failed", "url", imageURL, "err", readErr)
|
||||
return
|
||||
}
|
||||
|
||||
contentType := resp.Header.Get("Content-Type")
|
||||
if contentType == "" {
|
||||
contentType = "image/jpeg"
|
||||
}
|
||||
|
||||
if putErr := store.SaveBrowseAsset(ctx, key, data, contentType); putErr != nil {
|
||||
log.Warn("save-browse: SaveBrowseAsset failed", "key", key, "err", putErr)
|
||||
return
|
||||
}
|
||||
log.Debug("save-browse: cover stored", "key", key, "bytes", len(data))
|
||||
}
|
||||
|
||||
func envOr(key, fallback string) string {
|
||||
if v := os.Getenv(key); v != "" {
|
||||
return v
|
||||
@@ -543,17 +482,22 @@ Commands:
|
||||
--max-pages <n> pages to capture (default: 5)
|
||||
|
||||
Environment variables:
|
||||
BROWSERLESS_URL Browserless base URL (default: http://localhost:3030)
|
||||
BROWSERLESS_TOKEN API token (default: "")
|
||||
BROWSERLESS_STRATEGY content|scrape|cdp|direct (default: direct; chapter list+text always use direct HTTP)
|
||||
BROWSERLESS_MAX_CONCURRENT Max simultaneous sessions (default: 5)
|
||||
BROWSERLESS_TIMEOUT HTTP request timeout sec (default: 90)
|
||||
SCRAPER_WORKERS Chapter goroutines (default: NumCPU = %d)
|
||||
SCRAPER_STATIC_ROOT Output directory (default: ./static/books)
|
||||
SCRAPER_HTTP_ADDR HTTP listen address (default: :8080)
|
||||
SCRAPER_TIMEOUT HTTP request timeout sec (default: 90)
|
||||
KOKORO_URL Kokoro-FastAPI base URL (default: "", TTS disabled)
|
||||
KOKORO_VOICE Default TTS voice (default: af_bella)
|
||||
POCKETBASE_URL PocketBase base URL (default: http://localhost:8090)
|
||||
POCKETBASE_ADMIN_EMAIL PocketBase admin email (default: admin@libnovel.local)
|
||||
POCKETBASE_ADMIN_PASSWORD PocketBase admin password (default: changeme123)
|
||||
MINIO_ENDPOINT MinIO endpoint host:port (default: localhost:9000)
|
||||
MINIO_ACCESS_KEY MinIO access key (default: admin)
|
||||
MINIO_SECRET_KEY MinIO secret key (default: changeme123)
|
||||
MINIO_USE_SSL MinIO TLS (default: false)
|
||||
MINIO_BUCKET_CHAPTERS Chapter objects bucket (default: libnovel-chapters)
|
||||
MINIO_BUCKET_AUDIO Audio objects bucket (default: libnovel-audio)
|
||||
MINIO_BUCKET_BROWSE Browse snapshots bucket (default: libnovel-browse)
|
||||
BROWSERLESS_URL Browserless WS endpoint (default: http://localhost:3030)
|
||||
SINGLEFILE_PATH Path to single-file CLI (default: single-file)
|
||||
LOG_LEVEL debug|info|warn|error (default: info)
|
||||
`, runtime.NumCPU())
|
||||
|
||||
@@ -3,10 +3,8 @@ module github.com/libnovel/scraper
|
||||
go 1.25.0
|
||||
|
||||
require (
|
||||
github.com/gorilla/websocket v1.5.3
|
||||
github.com/minio/minio-go/v7 v7.0.98
|
||||
golang.org/x/net v0.51.0
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
)
|
||||
|
||||
require (
|
||||
@@ -27,4 +25,5 @@ require (
|
||||
golang.org/x/crypto v0.48.0 // indirect
|
||||
golang.org/x/sys v0.41.0 // indirect
|
||||
golang.org/x/text v0.34.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
)
|
||||
|
||||
@@ -6,8 +6,6 @@ github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
|
||||
github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
|
||||
github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
||||
github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk=
|
||||
github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
|
||||
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
|
||||
@@ -1,137 +0,0 @@
|
||||
package browser
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/gorilla/websocket"
|
||||
)
|
||||
|
||||
// cdpClient implements BrowserClient using the CDP WebSocket endpoint.
|
||||
type cdpClient struct {
|
||||
cfg Config
|
||||
sem chan struct{}
|
||||
}
|
||||
|
||||
// NewCDPClient returns a BrowserClient that uses CDP WebSocket sessions.
|
||||
func NewCDPClient(cfg Config) BrowserClient {
|
||||
if cfg.Timeout == 0 {
|
||||
cfg.Timeout = 60 * time.Second
|
||||
}
|
||||
return &cdpClient{cfg: cfg, sem: makeSem(cfg.MaxConcurrent)}
|
||||
}
|
||||
|
||||
func (c *cdpClient) Strategy() Strategy { return StrategyCDP }
|
||||
|
||||
func (c *cdpClient) GetContent(_ context.Context, _ ContentRequest) (string, error) {
|
||||
return "", fmt.Errorf("CDP client does not support /content; use NewContentClient")
|
||||
}
|
||||
|
||||
func (c *cdpClient) ScrapePage(_ context.Context, _ ScrapeRequest) (ScrapeResponse, error) {
|
||||
return ScrapeResponse{}, fmt.Errorf("CDP client does not support /scrape; use NewScrapeClient")
|
||||
}
|
||||
|
||||
// CDPSession opens a WebSocket to the Browserless /devtools/browser endpoint,
|
||||
// navigates to pageURL, and invokes fn with a live CDPConn.
|
||||
func (c *cdpClient) CDPSession(ctx context.Context, pageURL string, fn CDPSessionFunc) error {
|
||||
if err := acquire(ctx, c.sem); err != nil {
|
||||
return fmt.Errorf("cdp: semaphore: %w", err)
|
||||
}
|
||||
defer release(c.sem)
|
||||
|
||||
// Build WebSocket URL: ws://host:port/devtools/browser?token=...&url=...
|
||||
wsURL := strings.Replace(c.cfg.BaseURL, "http://", "ws://", 1)
|
||||
wsURL = strings.Replace(wsURL, "https://", "wss://", 1)
|
||||
wsURL += "/devtools/browser"
|
||||
sep := "?"
|
||||
if c.cfg.Token != "" {
|
||||
wsURL += sep + "token=" + c.cfg.Token
|
||||
sep = "&"
|
||||
}
|
||||
wsURL += sep + "url=" + pageURL
|
||||
|
||||
dialer := websocket.Dialer{
|
||||
HandshakeTimeout: 15 * time.Second,
|
||||
Proxy: http.ProxyFromEnvironment,
|
||||
}
|
||||
|
||||
conn, _, err := dialer.DialContext(ctx, wsURL, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cdp: dial %s: %w", wsURL, err)
|
||||
}
|
||||
|
||||
cdp := &cdpConn{ws: conn}
|
||||
defer cdp.Close()
|
||||
|
||||
return fn(ctx, cdp)
|
||||
}
|
||||
|
||||
// ─── cdpConn ─────────────────────────────────────────────────────────────────
|
||||
|
||||
type cdpConn struct {
|
||||
ws *websocket.Conn
|
||||
counter atomic.Int64
|
||||
}
|
||||
|
||||
type cdpRequest struct {
|
||||
ID int64 `json:"id"`
|
||||
Method string `json:"method"`
|
||||
Params map[string]any `json:"params,omitempty"`
|
||||
}
|
||||
|
||||
type cdpResponse struct {
|
||||
ID int64 `json:"id"`
|
||||
Result map[string]any `json:"result,omitempty"`
|
||||
Error *struct {
|
||||
Code int `json:"code"`
|
||||
Message string `json:"message"`
|
||||
} `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
func (c *cdpConn) Send(ctx context.Context, method string, params map[string]any) (map[string]any, error) {
|
||||
id := c.counter.Add(1)
|
||||
|
||||
req := cdpRequest{ID: id, Method: method, Params: params}
|
||||
data, err := json.Marshal(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cdp send: marshal: %w", err)
|
||||
}
|
||||
|
||||
if dl, ok := ctx.Deadline(); ok {
|
||||
_ = c.ws.SetWriteDeadline(dl)
|
||||
}
|
||||
if err := c.ws.WriteMessage(websocket.TextMessage, data); err != nil {
|
||||
return nil, fmt.Errorf("cdp send: write: %w", err)
|
||||
}
|
||||
|
||||
// Read messages until we find the response matching our id.
|
||||
for {
|
||||
if dl, ok := ctx.Deadline(); ok {
|
||||
_ = c.ws.SetReadDeadline(dl)
|
||||
}
|
||||
_, msg, err := c.ws.ReadMessage()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cdp send: read: %w", err)
|
||||
}
|
||||
var resp cdpResponse
|
||||
if err := json.Unmarshal(msg, &resp); err != nil {
|
||||
continue // skip non-JSON frames (events etc.)
|
||||
}
|
||||
if resp.ID != id {
|
||||
continue // event or different command reply
|
||||
}
|
||||
if resp.Error != nil {
|
||||
return nil, fmt.Errorf("cdp error %d: %s", resp.Error.Code, resp.Error.Message)
|
||||
}
|
||||
return resp.Result, nil
|
||||
}
|
||||
}
|
||||
|
||||
func (c *cdpConn) Close() error {
|
||||
return c.ws.Close()
|
||||
}
|
||||
@@ -55,6 +55,8 @@ func release(sem chan struct{}) {
|
||||
}
|
||||
}
|
||||
|
||||
// ─── /content client ──────────────────────────────────────────────────────────
|
||||
|
||||
// contentClient implements BrowserClient using the /content endpoint.
|
||||
type contentClient struct {
|
||||
cfg Config
|
||||
@@ -121,75 +123,5 @@ func (c *contentClient) ScrapePage(_ context.Context, _ ScrapeRequest) (ScrapeRe
|
||||
}
|
||||
|
||||
func (c *contentClient) CDPSession(_ context.Context, _ string, _ CDPSessionFunc) error {
|
||||
return fmt.Errorf("content client does not support CDP; use NewCDPClient")
|
||||
}
|
||||
|
||||
// ─── /scrape client ───────────────────────────────────────────────────────────
|
||||
|
||||
type scrapeClient struct {
|
||||
cfg Config
|
||||
http *http.Client
|
||||
sem chan struct{}
|
||||
}
|
||||
|
||||
// NewScrapeClient returns a BrowserClient that uses POST /scrape.
|
||||
func NewScrapeClient(cfg Config) BrowserClient {
|
||||
if cfg.Timeout == 0 {
|
||||
cfg.Timeout = 90 * time.Second
|
||||
}
|
||||
return &scrapeClient{
|
||||
cfg: cfg,
|
||||
http: &http.Client{Timeout: cfg.Timeout},
|
||||
sem: makeSem(cfg.MaxConcurrent),
|
||||
}
|
||||
}
|
||||
|
||||
func (c *scrapeClient) Strategy() Strategy { return StrategyScrape }
|
||||
|
||||
func (c *scrapeClient) GetContent(_ context.Context, _ ContentRequest) (string, error) {
|
||||
return "", fmt.Errorf("scrape client does not support /content; use NewContentClient")
|
||||
}
|
||||
|
||||
func (c *scrapeClient) ScrapePage(ctx context.Context, req ScrapeRequest) (ScrapeResponse, error) {
|
||||
if err := acquire(ctx, c.sem); err != nil {
|
||||
return ScrapeResponse{}, fmt.Errorf("scrape: semaphore: %w", err)
|
||||
}
|
||||
defer release(c.sem)
|
||||
|
||||
body, err := json.Marshal(req)
|
||||
if err != nil {
|
||||
return ScrapeResponse{}, fmt.Errorf("scrape: marshal request: %w", err)
|
||||
}
|
||||
|
||||
url := c.cfg.BaseURL + "/scrape"
|
||||
if c.cfg.Token != "" {
|
||||
url += "?token=" + c.cfg.Token
|
||||
}
|
||||
|
||||
httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return ScrapeResponse{}, fmt.Errorf("scrape: build request: %w", err)
|
||||
}
|
||||
httpReq.Header.Set("Content-Type", "application/json")
|
||||
|
||||
resp, err := c.http.Do(httpReq)
|
||||
if err != nil {
|
||||
return ScrapeResponse{}, fmt.Errorf("scrape: do request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
return ScrapeResponse{}, fmt.Errorf("scrape: unexpected status %d: %s", resp.StatusCode, b)
|
||||
}
|
||||
|
||||
var result ScrapeResponse
|
||||
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
||||
return ScrapeResponse{}, fmt.Errorf("scrape: decode response: %w", err)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (c *scrapeClient) CDPSession(_ context.Context, _ string, _ CDPSessionFunc) error {
|
||||
return fmt.Errorf("scrape client does not support CDP; use NewCDPClient")
|
||||
return fmt.Errorf("content client does not support CDP")
|
||||
}
|
||||
@@ -802,13 +802,6 @@ func assertURLAccessibleWithRetry(t *testing.T, ctx context.Context, url, label
|
||||
|
||||
// ─── stdlib helpers ───────────────────────────────────────────────────────────
|
||||
|
||||
func min(a, b int) int {
|
||||
if a < b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func chapterNumbers(refs []scraper.ChapterRef) []int {
|
||||
ns := make([]int, len(refs))
|
||||
for i, r := range refs {
|
||||
|
||||
@@ -2,12 +2,9 @@ package novelfire
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/libnovel/scraper/internal/scraper"
|
||||
"github.com/libnovel/scraper/internal/writer"
|
||||
)
|
||||
|
||||
// rankingPage1HTML is a realistic mock of the popular genre listing page
|
||||
@@ -149,60 +146,3 @@ func TestScrapeRanking_EmptyPage(t *testing.T) {
|
||||
t.Errorf("expected 0 entries for empty page, got %d", len(entries))
|
||||
}
|
||||
}
|
||||
|
||||
// TestWriteRanking_RoundTrip verifies WriteRanking → ReadRankingItems
|
||||
// faithfully reconstructs the original slice.
|
||||
func TestWriteRanking_RoundTrip(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
w := writer.New(dir)
|
||||
|
||||
items := []writer.RankingItem{
|
||||
{Rank: 1, Slug: "the-iron-throne", Title: "The Iron Throne", Status: "Ongoing",
|
||||
Genres: []string{"Fantasy", "Action"}, SourceURL: "https://novelfire.net/book/the-iron-throne"},
|
||||
{Rank: 2, Slug: "shadow-mage", Title: "Shadow Mage", Status: "Completed",
|
||||
Genres: []string{"Magic"}, SourceURL: "https://novelfire.net/book/shadow-mage"},
|
||||
}
|
||||
|
||||
if err := w.WriteRanking(items); err != nil {
|
||||
t.Fatalf("WriteRanking failed: %v", err)
|
||||
}
|
||||
|
||||
rankingFile := filepath.Join(dir, "ranking.json")
|
||||
if _, err := os.Stat(rankingFile); err != nil {
|
||||
t.Fatalf("ranking.json not created: %v", err)
|
||||
}
|
||||
|
||||
got, err := w.ReadRankingItems()
|
||||
if err != nil {
|
||||
t.Fatalf("ReadRankingItems failed: %v", err)
|
||||
}
|
||||
if len(got) != len(items) {
|
||||
t.Fatalf("expected %d items, got %d", len(items), len(got))
|
||||
}
|
||||
for i, want := range items {
|
||||
if got[i].Rank != want.Rank {
|
||||
t.Errorf("item[%d].Rank = %d, want %d", i, got[i].Rank, want.Rank)
|
||||
}
|
||||
if got[i].Slug != want.Slug {
|
||||
t.Errorf("item[%d].Slug = %q, want %q", i, got[i].Slug, want.Slug)
|
||||
}
|
||||
if got[i].Title != want.Title {
|
||||
t.Errorf("item[%d].Title = %q, want %q", i, got[i].Title, want.Title)
|
||||
}
|
||||
if got[i].Status != want.Status {
|
||||
t.Errorf("item[%d].Status = %q, want %q", i, got[i].Status, want.Status)
|
||||
}
|
||||
if len(got[i].Genres) != len(want.Genres) {
|
||||
t.Errorf("item[%d].Genres len = %d, want %d", i, len(got[i].Genres), len(want.Genres))
|
||||
} else {
|
||||
for j, g := range want.Genres {
|
||||
if got[i].Genres[j] != g {
|
||||
t.Errorf("item[%d].Genres[%d] = %q, want %q", i, j, got[i].Genres[j], g)
|
||||
}
|
||||
}
|
||||
}
|
||||
if got[i].SourceURL != want.SourceURL {
|
||||
t.Errorf("item[%d].SourceURL = %q, want %q", i, got[i].SourceURL, want.SourceURL)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -155,18 +155,8 @@ func (s *Scraper) ScrapeCatalogue(ctx context.Context) (<-chan scraper.Catalogue
|
||||
}
|
||||
nextHref := ""
|
||||
for _, a := range htmlutil.FindAll(root, scraper.Selector{Tag: "a", Multiple: true}) {
|
||||
isNext := false
|
||||
for _, attr := range a.Attr {
|
||||
if attr.Key == "rel" && attr.Val == "next" {
|
||||
isNext = true
|
||||
}
|
||||
}
|
||||
if isNext {
|
||||
for _, attr := range a.Attr {
|
||||
if attr.Key == "href" {
|
||||
nextHref = attr.Val
|
||||
}
|
||||
}
|
||||
if htmlutil.AttrVal(a, "rel") == "next" {
|
||||
nextHref = htmlutil.AttrVal(a, "href")
|
||||
break
|
||||
}
|
||||
}
|
||||
@@ -683,20 +673,8 @@ func (s *Scraper) ScrapeChapterText(ctx context.Context, ref scraper.ChapterRef)
|
||||
|
||||
// ─── helpers ─────────────────────────────────────────────────────────────────
|
||||
|
||||
func resolveURL(base, href string) string {
|
||||
if strings.HasPrefix(href, "http://") || strings.HasPrefix(href, "https://") {
|
||||
return href
|
||||
}
|
||||
b, err := url.Parse(base)
|
||||
if err != nil {
|
||||
return base + href
|
||||
}
|
||||
ref, err := url.Parse(href)
|
||||
if err != nil {
|
||||
return base + href
|
||||
}
|
||||
return b.ResolveReference(ref).String()
|
||||
}
|
||||
// resolveURL is a thin alias over htmlutil.ResolveURL kept for readability.
|
||||
func resolveURL(base, href string) string { return htmlutil.ResolveURL(base, href) }
|
||||
|
||||
func slugFromURL(bookURL string) string {
|
||||
u, err := url.Parse(bookURL)
|
||||
|
||||
@@ -141,6 +141,84 @@ func TestChapterNumberFromURL(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// ── ScrapeMetadata ────────────────────────────────────────────────────────────
|
||||
|
||||
func TestScrapeMetadata_ParsesFields(t *testing.T) {
|
||||
html := `<!DOCTYPE html><html><body>
|
||||
<h1 class="novel-title">The Iron Throne</h1>
|
||||
<span class="author"><a>Jane Doe</a></span>
|
||||
<figure class="cover"><img src="https://cdn.example.com/cover.jpg"></figure>
|
||||
<span class="status">Ongoing</span>
|
||||
<div class="genres"><a>Fantasy</a><a>Action</a></div>
|
||||
<div class="summary"><p>A sweeping epic set in a magical world.</p></div>
|
||||
<span class="chapter-count">42 Chapters</span>
|
||||
</body></html>`
|
||||
|
||||
s := newScraper(html)
|
||||
meta, err := s.ScrapeMetadata(context.Background(), "https://novelfire.net/book/the-iron-throne")
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if meta.Slug != "the-iron-throne" {
|
||||
t.Errorf("Slug = %q, want %q", meta.Slug, "the-iron-throne")
|
||||
}
|
||||
if meta.Title != "The Iron Throne" {
|
||||
t.Errorf("Title = %q, want %q", meta.Title, "The Iron Throne")
|
||||
}
|
||||
if meta.Author != "Jane Doe" {
|
||||
t.Errorf("Author = %q, want %q", meta.Author, "Jane Doe")
|
||||
}
|
||||
if meta.Cover != "https://cdn.example.com/cover.jpg" {
|
||||
t.Errorf("Cover = %q, want %q", meta.Cover, "https://cdn.example.com/cover.jpg")
|
||||
}
|
||||
if meta.Status != "Ongoing" {
|
||||
t.Errorf("Status = %q, want %q", meta.Status, "Ongoing")
|
||||
}
|
||||
if len(meta.Genres) != 2 || meta.Genres[0] != "Fantasy" || meta.Genres[1] != "Action" {
|
||||
t.Errorf("Genres = %v, want [Fantasy Action]", meta.Genres)
|
||||
}
|
||||
if !strings.Contains(meta.Summary, "sweeping epic") {
|
||||
t.Errorf("Summary = %q, want it to contain 'sweeping epic'", meta.Summary)
|
||||
}
|
||||
if meta.TotalChapters != 42 {
|
||||
t.Errorf("TotalChapters = %d, want 42", meta.TotalChapters)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScrapeMetadata_RelativeCoverURL(t *testing.T) {
|
||||
html := `<!DOCTYPE html><html><body>
|
||||
<h1 class="novel-title">Relative Cover</h1>
|
||||
<figure class="cover"><img src="/images/cover.jpg"></figure>
|
||||
</body></html>`
|
||||
|
||||
s := newScraper(html)
|
||||
meta, err := s.ScrapeMetadata(context.Background(), "https://novelfire.net/book/relative-cover")
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
// Relative cover URL should be resolved against the base domain.
|
||||
if !strings.HasPrefix(meta.Cover, "https://novelfire.net") {
|
||||
t.Errorf("Cover = %q, expected it to be resolved to an absolute URL", meta.Cover)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScrapeMetadata_MissingFields(t *testing.T) {
|
||||
// Minimal page — everything absent; should succeed without panicking.
|
||||
html := `<!DOCTYPE html><html><body></body></html>`
|
||||
|
||||
s := newScraper(html)
|
||||
meta, err := s.ScrapeMetadata(context.Background(), "https://novelfire.net/book/empty-novel")
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if meta.Slug != "empty-novel" {
|
||||
t.Errorf("Slug = %q, want %q", meta.Slug, "empty-novel")
|
||||
}
|
||||
if meta.TotalChapters != 0 {
|
||||
t.Errorf("TotalChapters = %d, want 0 for missing chapter-count", meta.TotalChapters)
|
||||
}
|
||||
}
|
||||
|
||||
// ── ScrapeChapterList (position vs URL numbering) ─────────────────────────────
|
||||
|
||||
// TestScrapeChapterList_NumbersFromURL verifies that when the chapter list HTML
|
||||
|
||||
312
scraper/internal/orchestrator/orchestrator_test.go
Normal file
312
scraper/internal/orchestrator/orchestrator_test.go
Normal file
@@ -0,0 +1,312 @@
|
||||
package orchestrator
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/libnovel/scraper/internal/scraper"
|
||||
"github.com/libnovel/scraper/internal/storage"
|
||||
"io"
|
||||
"log/slog"
|
||||
)
|
||||
|
||||
// ── mock NovelScraper ─────────────────────────────────────────────────────────
|
||||
|
||||
type mockScraper struct {
|
||||
catalogue []scraper.CatalogueEntry
|
||||
meta scraper.BookMeta
|
||||
metaErr error
|
||||
chapters []scraper.ChapterRef
|
||||
chapterTextFn func(ref scraper.ChapterRef) (scraper.Chapter, error)
|
||||
}
|
||||
|
||||
func (m *mockScraper) SourceName() string { return "mock" }
|
||||
|
||||
func (m *mockScraper) ScrapeCatalogue(_ context.Context) (<-chan scraper.CatalogueEntry, <-chan error) {
|
||||
entries := make(chan scraper.CatalogueEntry, len(m.catalogue))
|
||||
errs := make(chan error, 1)
|
||||
for _, e := range m.catalogue {
|
||||
entries <- e
|
||||
}
|
||||
close(entries)
|
||||
close(errs)
|
||||
return entries, errs
|
||||
}
|
||||
|
||||
func (m *mockScraper) ScrapeMetadata(_ context.Context, _ string) (scraper.BookMeta, error) {
|
||||
return m.meta, m.metaErr
|
||||
}
|
||||
|
||||
func (m *mockScraper) ScrapeChapterList(_ context.Context, _ string) ([]scraper.ChapterRef, error) {
|
||||
return m.chapters, nil
|
||||
}
|
||||
|
||||
func (m *mockScraper) ScrapeChapterText(_ context.Context, ref scraper.ChapterRef) (scraper.Chapter, error) {
|
||||
if m.chapterTextFn != nil {
|
||||
return m.chapterTextFn(ref)
|
||||
}
|
||||
return scraper.Chapter{Ref: ref, Text: "stub text"}, nil
|
||||
}
|
||||
|
||||
func (m *mockScraper) ScrapeRanking(_ context.Context, _ int) (<-chan scraper.BookMeta, <-chan error) {
|
||||
ch := make(chan scraper.BookMeta)
|
||||
errs := make(chan error)
|
||||
close(ch)
|
||||
close(errs)
|
||||
return ch, errs
|
||||
}
|
||||
|
||||
// ── mock Store ────────────────────────────────────────────────────────────────
|
||||
|
||||
// mockStore records which methods were called; only implements what the
|
||||
// orchestrator touches. All other methods panic so unexpected calls surface
|
||||
// as test failures rather than silent no-ops.
|
||||
type mockStore struct {
|
||||
mu sync.Mutex
|
||||
writtenMeta []scraper.BookMeta
|
||||
writtenChapters []scraper.Chapter
|
||||
existingSlugs map[string]map[int]bool // slug → chapterNum → exists
|
||||
}
|
||||
|
||||
func newMockStore() *mockStore {
|
||||
return &mockStore{existingSlugs: make(map[string]map[int]bool)}
|
||||
}
|
||||
|
||||
func (s *mockStore) ChapterExists(_ context.Context, slug string, ref scraper.ChapterRef) bool {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if m, ok := s.existingSlugs[slug]; ok {
|
||||
return m[ref.Number]
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (s *mockStore) WriteChapter(_ context.Context, slug string, ch scraper.Chapter) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.writtenChapters = append(s.writtenChapters, ch)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *mockStore) WriteMetadata(_ context.Context, meta scraper.BookMeta) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.writtenMeta = append(s.writtenMeta, meta)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Unimplemented Store methods — panic so accidental calls surface immediately.
|
||||
func (s *mockStore) ReadMetadata(_ context.Context, _ string) (scraper.BookMeta, bool, error) {
|
||||
panic("ReadMetadata not expected")
|
||||
}
|
||||
func (s *mockStore) ListBooks(_ context.Context) ([]scraper.BookMeta, error) {
|
||||
panic("ListBooks not expected")
|
||||
}
|
||||
func (s *mockStore) LocalSlugs(_ context.Context) (map[string]bool, error) {
|
||||
panic("LocalSlugs not expected")
|
||||
}
|
||||
func (s *mockStore) MetadataMtime(_ context.Context, _ string) int64 { return 0 }
|
||||
func (s *mockStore) ReadChapter(_ context.Context, _ string, _ int) (string, error) {
|
||||
panic("ReadChapter not expected")
|
||||
}
|
||||
func (s *mockStore) ListChapters(_ context.Context, _ string) ([]storage.ChapterInfo, error) {
|
||||
panic("ListChapters not expected")
|
||||
}
|
||||
func (s *mockStore) CountChapters(_ context.Context, _ string) int { return 0 }
|
||||
func (s *mockStore) ReindexChapters(_ context.Context, _ string) (int, error) {
|
||||
panic("ReindexChapters not expected")
|
||||
}
|
||||
func (s *mockStore) WriteRankingItem(_ context.Context, _ storage.RankingItem) error { return nil }
|
||||
func (s *mockStore) ReadRankingItems(_ context.Context) ([]storage.RankingItem, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (s *mockStore) RankingFreshEnough(_ context.Context, _ time.Duration) (bool, error) {
|
||||
return false, nil
|
||||
}
|
||||
func (s *mockStore) GetAudioCache(_ context.Context, _ string) (string, bool) { return "", false }
|
||||
func (s *mockStore) SetAudioCache(_ context.Context, _, _ string) error { return nil }
|
||||
func (s *mockStore) PutAudio(_ context.Context, _ string, _ []byte) error { return nil }
|
||||
func (s *mockStore) GetProgress(_ context.Context, _, _ string) (storage.ReadingProgress, bool) {
|
||||
return storage.ReadingProgress{}, false
|
||||
}
|
||||
func (s *mockStore) SetProgress(_ context.Context, _ string, _ storage.ReadingProgress) error {
|
||||
return nil
|
||||
}
|
||||
func (s *mockStore) AllProgress(_ context.Context, _ string) ([]storage.ReadingProgress, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (s *mockStore) DeleteProgress(_ context.Context, _, _ string) error { return nil }
|
||||
func (s *mockStore) AudioObjectKey(_ string, _ int, _ string) string { return "" }
|
||||
|
||||
func (s *mockStore) AudioExists(_ context.Context, _ string) bool { return false }
|
||||
func (s *mockStore) PresignChapter(_ context.Context, _ string, _ int, _ time.Duration) (string, error) {
|
||||
return "", nil
|
||||
}
|
||||
func (s *mockStore) PresignAudio(_ context.Context, _ string, _ time.Duration) (string, error) {
|
||||
return "", nil
|
||||
}
|
||||
func (s *mockStore) SaveBrowsePage(_ context.Context, _, _ string) error { return nil }
|
||||
func (s *mockStore) GetBrowsePage(_ context.Context, _ string) (string, bool, error) {
|
||||
return "", false, nil
|
||||
}
|
||||
func (s *mockStore) BrowseHTMLKey(_ string, _ int) string { return "" }
|
||||
func (s *mockStore) BrowseCoverKey(_, _ string) string { return "" }
|
||||
func (s *mockStore) SaveBrowseAsset(_ context.Context, _ string, _ []byte, _ string) error {
|
||||
return nil
|
||||
}
|
||||
func (s *mockStore) GetBrowseAsset(_ context.Context, _ string) ([]byte, string, bool, error) {
|
||||
return nil, "", false, nil
|
||||
}
|
||||
func (s *mockStore) CreateScrapeTask(_ context.Context, _, _ string) (string, error) {
|
||||
return "task-id", nil
|
||||
}
|
||||
func (s *mockStore) UpdateScrapeTask(_ context.Context, _ string, _ storage.ScrapeTaskUpdate) error {
|
||||
return nil
|
||||
}
|
||||
func (s *mockStore) ListScrapeTasks(_ context.Context) ([]storage.ScrapeTask, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// ── helpers ───────────────────────────────────────────────────────────────────
|
||||
|
||||
func discardLogger() *slog.Logger {
|
||||
return slog.New(slog.NewTextHandler(io.Discard, nil))
|
||||
}
|
||||
|
||||
// ── tests ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
// TestRun_SingleBook verifies the happy-path single-book scrape: metadata is
|
||||
// persisted and all chapters are written to the store.
|
||||
func TestRun_SingleBook(t *testing.T) {
|
||||
novel := &mockScraper{
|
||||
meta: scraper.BookMeta{Slug: "the-iron-throne", Title: "The Iron Throne"},
|
||||
chapters: []scraper.ChapterRef{
|
||||
{Number: 1, Title: "Chapter 1", URL: "https://example.com/book/ch-1"},
|
||||
{Number: 2, Title: "Chapter 2", URL: "https://example.com/book/ch-2"},
|
||||
{Number: 3, Title: "Chapter 3", URL: "https://example.com/book/ch-3"},
|
||||
},
|
||||
}
|
||||
store := newMockStore()
|
||||
|
||||
o := New(Config{Workers: 2, SingleBookURL: "https://example.com/book/the-iron-throne"}, novel, discardLogger(), store)
|
||||
if err := o.Run(context.Background()); err != nil {
|
||||
t.Fatalf("Run() returned error: %v", err)
|
||||
}
|
||||
|
||||
store.mu.Lock()
|
||||
defer store.mu.Unlock()
|
||||
|
||||
if len(store.writtenMeta) != 1 {
|
||||
t.Errorf("writtenMeta count = %d, want 1", len(store.writtenMeta))
|
||||
}
|
||||
if len(store.writtenChapters) != 3 {
|
||||
t.Errorf("writtenChapters count = %d, want 3", len(store.writtenChapters))
|
||||
}
|
||||
}
|
||||
|
||||
// TestRun_SingleBook_SkipsExistingChapters verifies that chapters already in
|
||||
// the store are not re-scraped.
|
||||
func TestRun_SingleBook_SkipsExistingChapters(t *testing.T) {
|
||||
novel := &mockScraper{
|
||||
meta: scraper.BookMeta{Slug: "test-novel", Title: "Test Novel"},
|
||||
chapters: []scraper.ChapterRef{
|
||||
{Number: 1, Title: "Chapter 1"},
|
||||
{Number: 2, Title: "Chapter 2"},
|
||||
},
|
||||
}
|
||||
store := newMockStore()
|
||||
// Mark chapter 1 as already existing.
|
||||
store.existingSlugs["test-novel"] = map[int]bool{1: true}
|
||||
|
||||
o := New(Config{Workers: 1, SingleBookURL: "https://example.com/book/test-novel"}, novel, discardLogger(), store)
|
||||
if err := o.Run(context.Background()); err != nil {
|
||||
t.Fatalf("Run() returned error: %v", err)
|
||||
}
|
||||
|
||||
store.mu.Lock()
|
||||
defer store.mu.Unlock()
|
||||
|
||||
// Only chapter 2 should have been written; chapter 1 was skipped.
|
||||
if len(store.writtenChapters) != 1 {
|
||||
t.Errorf("writtenChapters count = %d, want 1 (skipped ch1)", len(store.writtenChapters))
|
||||
}
|
||||
if store.writtenChapters[0].Ref.Number != 2 {
|
||||
t.Errorf("expected chapter 2 to be written, got chapter %d", store.writtenChapters[0].Ref.Number)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRun_CatalogueMode verifies that catalogue mode processes all books.
|
||||
func TestRun_CatalogueMode(t *testing.T) {
|
||||
novel := &mockScraper{
|
||||
catalogue: []scraper.CatalogueEntry{
|
||||
{Title: "Book A", URL: "https://example.com/book/a"},
|
||||
{Title: "Book B", URL: "https://example.com/book/b"},
|
||||
},
|
||||
meta: scraper.BookMeta{Slug: "book-slug", Title: "A Book"},
|
||||
chapters: []scraper.ChapterRef{{Number: 1, Title: "Chapter 1"}},
|
||||
}
|
||||
store := newMockStore()
|
||||
|
||||
o := New(Config{Workers: 2}, novel, discardLogger(), store)
|
||||
if err := o.Run(context.Background()); err != nil {
|
||||
t.Fatalf("Run() returned error: %v", err)
|
||||
}
|
||||
|
||||
store.mu.Lock()
|
||||
defer store.mu.Unlock()
|
||||
|
||||
// 2 books → 2 metadata writes, 2 chapter writes (one chapter per book).
|
||||
if len(store.writtenMeta) != 2 {
|
||||
t.Errorf("writtenMeta count = %d, want 2", len(store.writtenMeta))
|
||||
}
|
||||
if len(store.writtenChapters) != 2 {
|
||||
t.Errorf("writtenChapters count = %d, want 2", len(store.writtenChapters))
|
||||
}
|
||||
}
|
||||
|
||||
// TestRun_OnProgress_Called verifies that the OnProgress callback fires at
|
||||
// least once upon completion.
|
||||
func TestRun_OnProgress_Called(t *testing.T) {
|
||||
novel := &mockScraper{
|
||||
meta: scraper.BookMeta{Slug: "progress-book", Title: "Progress Book"},
|
||||
chapters: []scraper.ChapterRef{{Number: 1, Title: "Chapter 1"}},
|
||||
}
|
||||
store := newMockStore()
|
||||
|
||||
var callCount int
|
||||
o := New(Config{
|
||||
Workers: 1,
|
||||
SingleBookURL: "https://example.com/book/progress-book",
|
||||
OnProgress: func(_ Progress) {
|
||||
callCount++
|
||||
},
|
||||
}, novel, discardLogger(), store)
|
||||
|
||||
if err := o.Run(context.Background()); err != nil {
|
||||
t.Fatalf("Run() returned error: %v", err)
|
||||
}
|
||||
if callCount == 0 {
|
||||
t.Error("OnProgress was never called")
|
||||
}
|
||||
}
|
||||
|
||||
// TestRun_ContextCancelled verifies that Run returns a non-nil error when the
|
||||
// context is cancelled before work completes.
|
||||
func TestRun_ContextCancelled(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel() // cancel immediately
|
||||
|
||||
novel := &mockScraper{
|
||||
meta: scraper.BookMeta{Slug: "cancel-book", Title: "Cancel Book"},
|
||||
chapters: []scraper.ChapterRef{{Number: 1}},
|
||||
}
|
||||
store := newMockStore()
|
||||
|
||||
o := New(Config{Workers: 1, SingleBookURL: "https://example.com/book/cancel-book"}, novel, discardLogger(), store)
|
||||
err := o.Run(ctx)
|
||||
if err == nil {
|
||||
t.Error("expected non-nil error when context is cancelled, got nil")
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@
|
||||
package htmlutil
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
@@ -10,6 +11,24 @@ import (
|
||||
"golang.org/x/net/html"
|
||||
)
|
||||
|
||||
// ResolveURL returns an absolute URL. If href is already absolute it is
|
||||
// returned unchanged. Otherwise it is resolved against base using standard
|
||||
// URL resolution (handles relative paths, absolute paths, etc.).
|
||||
func ResolveURL(base, href string) string {
|
||||
if strings.HasPrefix(href, "http://") || strings.HasPrefix(href, "https://") {
|
||||
return href
|
||||
}
|
||||
b, err := url.Parse(base)
|
||||
if err != nil {
|
||||
return base + href
|
||||
}
|
||||
ref, err := url.Parse(href)
|
||||
if err != nil {
|
||||
return base + href
|
||||
}
|
||||
return b.ResolveReference(ref).String()
|
||||
}
|
||||
|
||||
// ParseHTML parses raw HTML and returns the root node.
|
||||
func ParseHTML(raw string) (*html.Node, error) {
|
||||
return html.Parse(strings.NewReader(raw))
|
||||
@@ -48,8 +67,8 @@ matched:
|
||||
return true
|
||||
}
|
||||
|
||||
// attrVal returns the value of attribute key from node n.
|
||||
func attrVal(n *html.Node, key string) string {
|
||||
// AttrVal returns the value of attribute key from node n.
|
||||
func AttrVal(n *html.Node, key string) string {
|
||||
for _, a := range n.Attr {
|
||||
if a.Key == key {
|
||||
return a.Val
|
||||
@@ -58,8 +77,11 @@ func attrVal(n *html.Node, key string) string {
|
||||
return ""
|
||||
}
|
||||
|
||||
// textContent returns the concatenated text content of all descendant text nodes.
|
||||
func textContent(n *html.Node) string {
|
||||
// attrVal is an unexported alias kept for internal use within this package.
|
||||
func attrVal(n *html.Node, key string) string { return AttrVal(n, key) }
|
||||
|
||||
// TextContent returns the concatenated text content of all descendant text nodes.
|
||||
func TextContent(n *html.Node) string {
|
||||
var sb strings.Builder
|
||||
var walk func(*html.Node)
|
||||
walk = func(cur *html.Node) {
|
||||
@@ -74,6 +96,9 @@ func textContent(n *html.Node) string {
|
||||
return strings.TrimSpace(sb.String())
|
||||
}
|
||||
|
||||
// textContent is an unexported alias kept for internal use within this package.
|
||||
func textContent(n *html.Node) string { return TextContent(n) }
|
||||
|
||||
// FindFirst returns the first node matching sel within root.
|
||||
func FindFirst(root *html.Node, sel scraper.Selector) *html.Node {
|
||||
var found *html.Node
|
||||
|
||||
221
scraper/internal/scraper/htmlutil/htmlutil_test.go
Normal file
221
scraper/internal/scraper/htmlutil/htmlutil_test.go
Normal file
@@ -0,0 +1,221 @@
|
||||
package htmlutil
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/libnovel/scraper/internal/scraper"
|
||||
)
|
||||
|
||||
// ── ResolveURL ────────────────────────────────────────────────────────────────
|
||||
|
||||
func TestResolveURL(t *testing.T) {
|
||||
cases := []struct{ base, href, want string }{
|
||||
// Already absolute → unchanged.
|
||||
{"https://example.com", "https://other.com/page", "https://other.com/page"},
|
||||
{"https://example.com", "http://other.com/page", "http://other.com/page"},
|
||||
// Absolute path.
|
||||
{"https://example.com", "/book/slug", "https://example.com/book/slug"},
|
||||
// Relative path.
|
||||
{"https://example.com/genre/all", "page?p=2", "https://example.com/genre/page?p=2"},
|
||||
// Empty href → base itself.
|
||||
{"https://example.com", "", "https://example.com"},
|
||||
}
|
||||
for _, c := range cases {
|
||||
got := ResolveURL(c.base, c.href)
|
||||
if got != c.want {
|
||||
t.Errorf("ResolveURL(%q, %q) = %q, want %q", c.base, c.href, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── AttrVal ───────────────────────────────────────────────────────────────────
|
||||
|
||||
func TestAttrVal(t *testing.T) {
|
||||
root, err := ParseHTML(`<html><body><a href="/book/slug" class="link">text</a></body></html>`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a := FindFirst(root, scraper.Selector{Tag: "a"})
|
||||
if a == nil {
|
||||
t.Fatal("expected to find <a>")
|
||||
}
|
||||
if got := AttrVal(a, "href"); got != "/book/slug" {
|
||||
t.Errorf("AttrVal href = %q, want %q", got, "/book/slug")
|
||||
}
|
||||
if got := AttrVal(a, "class"); got != "link" {
|
||||
t.Errorf("AttrVal class = %q, want %q", got, "link")
|
||||
}
|
||||
if got := AttrVal(a, "missing"); got != "" {
|
||||
t.Errorf("AttrVal missing = %q, want empty", got)
|
||||
}
|
||||
}
|
||||
|
||||
// ── TextContent ───────────────────────────────────────────────────────────────
|
||||
|
||||
func TestTextContent(t *testing.T) {
|
||||
root, err := ParseHTML(`<html><body><p>Hello <b>world</b></p></body></html>`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p := FindFirst(root, scraper.Selector{Tag: "p"})
|
||||
if p == nil {
|
||||
t.Fatal("expected to find <p>")
|
||||
}
|
||||
if got := TextContent(p); got != "Hello world" {
|
||||
t.Errorf("TextContent = %q, want %q", got, "Hello world")
|
||||
}
|
||||
}
|
||||
|
||||
// ── FindFirst / FindAll ───────────────────────────────────────────────────────
|
||||
|
||||
func TestFindFirst_ByTag(t *testing.T) {
|
||||
root, _ := ParseHTML(`<html><body><h1>Title</h1><h2>Sub</h2></body></html>`)
|
||||
n := FindFirst(root, scraper.Selector{Tag: "h1"})
|
||||
if n == nil {
|
||||
t.Fatal("expected to find <h1>")
|
||||
}
|
||||
if TextContent(n) != "Title" {
|
||||
t.Errorf("h1 text = %q, want %q", TextContent(n), "Title")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFindFirst_ByClass(t *testing.T) {
|
||||
root, _ := ParseHTML(`<html><body><span class="author foo">JR</span></body></html>`)
|
||||
n := FindFirst(root, scraper.Selector{Tag: "span", Class: "author"})
|
||||
if n == nil {
|
||||
t.Fatal("expected to find span.author")
|
||||
}
|
||||
if TextContent(n) != "JR" {
|
||||
t.Errorf("author text = %q, want %q", TextContent(n), "JR")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFindFirst_ByID(t *testing.T) {
|
||||
root, _ := ParseHTML(`<html><body><div id="content"><p>text</p></div></body></html>`)
|
||||
n := FindFirst(root, scraper.Selector{ID: "content"})
|
||||
if n == nil {
|
||||
t.Fatal("expected to find #content")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFindFirst_NoMatch(t *testing.T) {
|
||||
root, _ := ParseHTML(`<html><body><p>nothing</p></body></html>`)
|
||||
n := FindFirst(root, scraper.Selector{Tag: "h1"})
|
||||
if n != nil {
|
||||
t.Errorf("expected nil for missing tag, got %v", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFindAll_Multiple(t *testing.T) {
|
||||
root, _ := ParseHTML(`<html><body>
|
||||
<li class="novel-item">A</li>
|
||||
<li class="novel-item">B</li>
|
||||
<li class="other">C</li>
|
||||
</body></html>`)
|
||||
nodes := FindAll(root, scraper.Selector{Tag: "li", Class: "novel-item"})
|
||||
if len(nodes) != 2 {
|
||||
t.Errorf("FindAll novel-item = %d, want 2", len(nodes))
|
||||
}
|
||||
}
|
||||
|
||||
// ── ExtractFirst / ExtractAll ─────────────────────────────────────────────────
|
||||
|
||||
func TestExtractFirst_TextNode(t *testing.T) {
|
||||
root, _ := ParseHTML(`<html><body><h1 class="novel-title">Shadow Slave</h1></body></html>`)
|
||||
got := ExtractFirst(root, scraper.Selector{Tag: "h1", Class: "novel-title"})
|
||||
if got != "Shadow Slave" {
|
||||
t.Errorf("ExtractFirst title = %q, want %q", got, "Shadow Slave")
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractFirst_AttrNode(t *testing.T) {
|
||||
root, _ := ParseHTML(`<html><body><img src="/covers/slug.jpg"></body></html>`)
|
||||
got := ExtractFirst(root, scraper.Selector{Tag: "img", Attr: "src"})
|
||||
if got != "/covers/slug.jpg" {
|
||||
t.Errorf("ExtractFirst img src = %q, want %q", got, "/covers/slug.jpg")
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractFirst_Missing(t *testing.T) {
|
||||
root, _ := ParseHTML(`<html><body></body></html>`)
|
||||
got := ExtractFirst(root, scraper.Selector{Tag: "h1"})
|
||||
if got != "" {
|
||||
t.Errorf("ExtractFirst missing = %q, want empty", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractAll_Genres(t *testing.T) {
|
||||
root, _ := ParseHTML(`<html><body>
|
||||
<div class="genres">
|
||||
<a href="/genre/action">Action</a>
|
||||
<a href="/genre/fantasy">Fantasy</a>
|
||||
</div>
|
||||
</body></html>`)
|
||||
genresNode := FindFirst(root, scraper.Selector{Tag: "div", Class: "genres"})
|
||||
if genresNode == nil {
|
||||
t.Fatal("expected genres div")
|
||||
}
|
||||
genres := ExtractAll(genresNode, scraper.Selector{Tag: "a"})
|
||||
if len(genres) != 2 {
|
||||
t.Fatalf("genres = %v, want 2", genres)
|
||||
}
|
||||
if genres[0] != "Action" || genres[1] != "Fantasy" {
|
||||
t.Errorf("genres = %v, want [Action Fantasy]", genres)
|
||||
}
|
||||
}
|
||||
|
||||
// ── NodeToMarkdown ────────────────────────────────────────────────────────────
|
||||
|
||||
func TestNodeToMarkdown_Paragraphs(t *testing.T) {
|
||||
root, _ := ParseHTML(`<html><body><div id="content">
|
||||
<p>First paragraph.</p>
|
||||
<p>Second paragraph.</p>
|
||||
</div></body></html>`)
|
||||
container := FindFirst(root, scraper.Selector{ID: "content"})
|
||||
if container == nil {
|
||||
t.Fatal("missing #content")
|
||||
}
|
||||
md := NodeToMarkdown(container)
|
||||
if md == "" {
|
||||
t.Fatal("NodeToMarkdown returned empty string")
|
||||
}
|
||||
for _, want := range []string{"First paragraph", "Second paragraph"} {
|
||||
if !strings.Contains(md, want) {
|
||||
t.Errorf("NodeToMarkdown missing %q in:\n%s", want, md)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNodeToMarkdown_Bold(t *testing.T) {
|
||||
root, _ := ParseHTML(`<html><body><div id="content"><p>He was <strong>very</strong> strong.</p></div></body></html>`)
|
||||
container := FindFirst(root, scraper.Selector{ID: "content"})
|
||||
md := NodeToMarkdown(container)
|
||||
if !strings.Contains(md, "**very**") {
|
||||
t.Errorf("NodeToMarkdown should wrap <strong> in **, got:\n%s", md)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNodeToMarkdown_ScriptStripped(t *testing.T) {
|
||||
root, _ := ParseHTML(`<html><body><div id="content"><p>Good</p><script>alert(1)</script></div></body></html>`)
|
||||
container := FindFirst(root, scraper.Selector{ID: "content"})
|
||||
md := NodeToMarkdown(container)
|
||||
if strings.Contains(md, "alert") {
|
||||
t.Errorf("NodeToMarkdown should strip <script> content, got:\n%s", md)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNodeToMarkdown_CollapseBlankLines(t *testing.T) {
|
||||
root, _ := ParseHTML(`<html><body><div id="content">
|
||||
<p>A</p>
|
||||
<p></p>
|
||||
<p></p>
|
||||
<p>B</p>
|
||||
</div></body></html>`)
|
||||
container := FindFirst(root, scraper.Selector{ID: "content"})
|
||||
md := NodeToMarkdown(container)
|
||||
// Should not have more than one consecutive blank line.
|
||||
if strings.Contains(md, "\n\n\n") {
|
||||
t.Errorf("NodeToMarkdown should collapse triple newlines, got:\n%q", md)
|
||||
}
|
||||
}
|
||||
541
scraper/internal/server/handlers_audio.go
Normal file
541
scraper/internal/server/handlers_audio.go
Normal file
@@ -0,0 +1,541 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ─── Audio generation via Kokoro /v1/audio/speech ────────────────────────────
|
||||
//
|
||||
// handleAudioGenerate handles POST /api/audio/{slug}/{n}.
|
||||
//
|
||||
// It calls Kokoro's POST /v1/audio/speech with return_download_link=true.
|
||||
// Kokoro generates the audio, saves it to its own temp storage, and returns
|
||||
// the download filename in the X-Download-Path response header.
|
||||
// We cache that filename (in memory, keyed by slug/chapter/voice) and
|
||||
// return a proxy URL that the browser sets as audio.src.
|
||||
//
|
||||
// TTS is always generated at speed 1.0; playback speed is controlled
|
||||
// client-side via the <audio> element's playbackRate.
|
||||
//
|
||||
// On a cache hit the proxy URL is returned immediately without re-generating.
|
||||
// Concurrent requests for the same key are deduplicated.
|
||||
func (s *Server) handleAudioGenerate(w http.ResponseWriter, r *http.Request) {
|
||||
slug := r.PathValue("slug")
|
||||
n, err := strconv.Atoi(r.PathValue("n"))
|
||||
if err != nil || n < 1 {
|
||||
http.Error(w, `{"error":"invalid chapter"}`, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
// Parse optional voice from JSON body. Speed is intentionally ignored —
|
||||
// TTS is always generated at 1.0; playback speed is applied client-side.
|
||||
voice := s.kokoroVoice
|
||||
var body struct {
|
||||
Voice string `json:"voice"`
|
||||
MaxChars int `json:"max_chars"`
|
||||
}
|
||||
if r.Body != nil {
|
||||
_ = json.NewDecoder(r.Body).Decode(&body)
|
||||
}
|
||||
if body.Voice != "" {
|
||||
voice = body.Voice
|
||||
}
|
||||
|
||||
cacheKey := fmt.Sprintf("%s/%d/%s", slug, n, voice)
|
||||
|
||||
// Fast path: already generated (check persistent store first).
|
||||
if filename, ok := s.store.GetAudioCache(r.Context(), cacheKey); ok {
|
||||
s.writeAudioResponse(w, slug, n, voice, filename)
|
||||
return
|
||||
}
|
||||
|
||||
// Deduplicate concurrent generation for the same key.
|
||||
s.audioMu.Lock()
|
||||
if ch, ok := s.audioInFlight[cacheKey]; ok {
|
||||
s.audioMu.Unlock()
|
||||
select {
|
||||
case <-ch:
|
||||
case <-r.Context().Done():
|
||||
http.Error(w, `{"error":"request cancelled"}`, http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
// Check store again after waiting.
|
||||
if filename, ok := s.store.GetAudioCache(r.Context(), cacheKey); ok {
|
||||
s.writeAudioResponse(w, slug, n, voice, filename)
|
||||
} else {
|
||||
http.Error(w, `{"error":"audio generation failed"}`, http.StatusInternalServerError)
|
||||
}
|
||||
return
|
||||
}
|
||||
ch := make(chan struct{})
|
||||
s.audioInFlight[cacheKey] = ch
|
||||
s.audioMu.Unlock()
|
||||
|
||||
defer func() {
|
||||
s.audioMu.Lock()
|
||||
delete(s.audioInFlight, cacheKey)
|
||||
s.audioMu.Unlock()
|
||||
close(ch)
|
||||
}()
|
||||
|
||||
// Load and validate chapter text.
|
||||
raw, err := s.store.ReadChapter(r.Context(), slug, n)
|
||||
if err != nil {
|
||||
http.Error(w, `{"error":"chapter not found"}`, http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
text := stripMarkdown(raw)
|
||||
if text == "" {
|
||||
http.Error(w, `{"error":"chapter text is empty"}`, http.StatusUnprocessableEntity)
|
||||
return
|
||||
}
|
||||
if body.MaxChars > 0 && len([]rune(text)) > body.MaxChars {
|
||||
text = string([]rune(text)[:body.MaxChars])
|
||||
}
|
||||
if s.kokoroURL == "" {
|
||||
http.Error(w, `{"error":"kokoro not configured"}`, http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
|
||||
// Call Kokoro POST /v1/audio/speech at speed 1.0.
|
||||
// Kokoro saves the generated audio to its own temp storage and returns the
|
||||
// download path in the X-Download-Path response header.
|
||||
filename, err := s.generateSpeech(r.Context(), text, voice, 1.0)
|
||||
if err != nil {
|
||||
s.log.Error("kokoro speech generation failed", "slug", slug, "chapter", n, "err", err)
|
||||
http.Error(w, `{"error":"speech generation failed"}`, http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
|
||||
if err := s.store.SetAudioCache(r.Context(), cacheKey, filename); err != nil {
|
||||
s.log.Warn("audio cache write failed", "slug", slug, "chapter", n, "cache_key", cacheKey, "err", err)
|
||||
}
|
||||
|
||||
// Download generated audio from Kokoro and persist to MinIO synchronously
|
||||
// so that the presigned URL returned to the client is immediately valid.
|
||||
minioKey := s.store.AudioObjectKey(slug, n, voice)
|
||||
audioData, dlErr := s.downloadFromKokoro(r.Context(), filename)
|
||||
if dlErr != nil {
|
||||
s.log.Warn("audio MinIO upload skipped: kokoro download failed",
|
||||
"slug", slug, "chapter", n, "filename", filename, "err", dlErr)
|
||||
} else if putErr := s.store.PutAudio(r.Context(), minioKey, audioData); putErr != nil {
|
||||
s.log.Warn("audio MinIO upload failed",
|
||||
"slug", slug, "chapter", n, "key", minioKey, "err", putErr)
|
||||
// upload failure is non-fatal; the client can still stream via Kokoro proxy
|
||||
} else {
|
||||
s.log.Info("audio uploaded to MinIO", "slug", slug, "chapter", n, "key", minioKey)
|
||||
}
|
||||
|
||||
s.log.Info("audio generated", "slug", slug, "chapter", n, "filename", filename)
|
||||
s.writeAudioResponse(w, slug, n, voice, filename)
|
||||
}
|
||||
|
||||
// generateSpeech calls POST /v1/audio/speech on Kokoro with return_download_link=true
|
||||
// and returns the filename from the X-Download-Path response header.
|
||||
func (s *Server) generateSpeech(ctx context.Context, text, voice string, speed float64) (string, error) {
|
||||
reqBody, _ := json.Marshal(map[string]interface{}{
|
||||
"model": "kokoro",
|
||||
"input": text,
|
||||
"voice": voice,
|
||||
"response_format": "mp3",
|
||||
"speed": speed,
|
||||
"stream": false,
|
||||
"return_download_link": true,
|
||||
})
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost,
|
||||
s.kokoroURL+"/v1/audio/speech", bytes.NewReader(reqBody))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("build request: %w", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("kokoro request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
// Drain body so the connection can be reused.
|
||||
_, _ = io.Copy(io.Discard, resp.Body)
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return "", fmt.Errorf("kokoro status %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
// X-Download-Path is e.g. "/download/speech_abc123.mp3"
|
||||
dlPath := resp.Header.Get("X-Download-Path")
|
||||
if dlPath == "" {
|
||||
return "", fmt.Errorf("kokoro did not return X-Download-Path header")
|
||||
}
|
||||
|
||||
// Extract just the filename from the path.
|
||||
filename := dlPath
|
||||
if idx := strings.LastIndex(dlPath, "/"); idx >= 0 {
|
||||
filename = dlPath[idx+1:]
|
||||
}
|
||||
if filename == "" {
|
||||
return "", fmt.Errorf("empty filename in X-Download-Path: %q", dlPath)
|
||||
}
|
||||
return filename, nil
|
||||
}
|
||||
|
||||
// downloadFromKokoro downloads a generated audio file from Kokoro's temp storage
|
||||
// using GET /v1/download/{filename} and returns the raw bytes.
|
||||
func (s *Server) downloadFromKokoro(ctx context.Context, filename string) ([]byte, error) {
|
||||
url := s.kokoroURL + "/v1/download/" + filename
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("build download request: %w", err)
|
||||
}
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("kokoro download request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("kokoro download status %d", resp.StatusCode)
|
||||
}
|
||||
data, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read kokoro download body: %w", err)
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
// writeAudioResponse writes the JSON response for a generated audio chapter.
|
||||
// The URL points to our proxy handler GET /api/audio-proxy/{slug}/{n}.
|
||||
func (s *Server) writeAudioResponse(w http.ResponseWriter, slug string, n int, voice string, filename string) {
|
||||
proxyURL := fmt.Sprintf("/api/audio-proxy/%s/%d?voice=%s", slug, n, voice)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"url": proxyURL,
|
||||
"filename": filename,
|
||||
})
|
||||
}
|
||||
|
||||
// handleAudioProxy handles GET /api/audio-proxy/{slug}/{n}.
|
||||
// It looks up the Kokoro download filename for this chapter (voice) and
|
||||
// proxies GET /v1/download/{filename} from the Kokoro server back to the browser.
|
||||
func (s *Server) handleAudioProxy(w http.ResponseWriter, r *http.Request) {
|
||||
slug := r.PathValue("slug")
|
||||
n, err := strconv.Atoi(r.PathValue("n"))
|
||||
if err != nil || n < 1 {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
voice := r.URL.Query().Get("voice")
|
||||
if voice == "" {
|
||||
voice = s.kokoroVoice
|
||||
}
|
||||
|
||||
cacheKey := fmt.Sprintf("%s/%d/%s", slug, n, voice)
|
||||
filename, ok := s.store.GetAudioCache(r.Context(), cacheKey)
|
||||
if !ok {
|
||||
http.Error(w, "audio not generated yet", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
kokoroURL := s.kokoroURL + "/v1/download/" + filename
|
||||
req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, kokoroURL, nil)
|
||||
if err != nil {
|
||||
http.Error(w, "failed to build proxy request", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
http.Error(w, "kokoro download failed", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
http.Error(w, fmt.Sprintf("kokoro returned %d", resp.StatusCode), http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "audio/mpeg")
|
||||
w.Header().Set("Cache-Control", "public, max-age=3600")
|
||||
if cl := resp.Header.Get("Content-Length"); cl != "" {
|
||||
w.Header().Set("Content-Length", cl)
|
||||
}
|
||||
_, _ = io.Copy(w, resp.Body)
|
||||
}
|
||||
|
||||
// ─── Presigned URL handlers ───────────────────────────────────────────────────
|
||||
|
||||
// handlePresignChapter handles GET /api/presign/chapter/{slug}/{n}.
|
||||
// Returns a short-lived presigned MinIO URL for the chapter markdown object.
|
||||
// The SvelteKit server uses this to fetch chapter content server-side.
|
||||
func (s *Server) handlePresignChapter(w http.ResponseWriter, r *http.Request) {
|
||||
slug := r.PathValue("slug")
|
||||
n, err := strconv.Atoi(r.PathValue("n"))
|
||||
if err != nil || n < 1 || slug == "" {
|
||||
http.Error(w, `{"error":"invalid params"}`, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
url, err := s.store.PresignChapter(r.Context(), slug, n, 15*time.Minute)
|
||||
if err != nil {
|
||||
s.log.Error("presign chapter failed", "slug", slug, "n", n, "err", err)
|
||||
http.Error(w, `{"error":"presign failed"}`, http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]string{"url": url})
|
||||
}
|
||||
|
||||
// handlePresignAudio handles GET /api/presign/audio/{slug}/{n}.
|
||||
// Returns a presigned MinIO URL for the audio object (if it has been generated).
|
||||
// Query params: voice (optional, defaults to server default).
|
||||
func (s *Server) handlePresignAudio(w http.ResponseWriter, r *http.Request) {
|
||||
slug := r.PathValue("slug")
|
||||
n, err := strconv.Atoi(r.PathValue("n"))
|
||||
if err != nil || n < 1 || slug == "" {
|
||||
http.Error(w, `{"error":"invalid params"}`, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
voice := r.URL.Query().Get("voice")
|
||||
if voice == "" {
|
||||
voice = s.kokoroVoice
|
||||
}
|
||||
|
||||
key := s.store.AudioObjectKey(slug, n, voice)
|
||||
|
||||
// Return 404 when the object hasn't been uploaded yet — the client treats
|
||||
// this as "audio not ready" and will either poll or trigger generation.
|
||||
if !s.store.AudioExists(r.Context(), key) {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
url, err := s.store.PresignAudio(r.Context(), key, 1*time.Hour)
|
||||
if err != nil {
|
||||
s.log.Error("presign audio failed", "slug", slug, "n", n, "err", err)
|
||||
http.Error(w, `{"error":"presign failed"}`, http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]string{"url": url})
|
||||
}
|
||||
|
||||
// ─── Voices API ───────────────────────────────────────────────────────────────
|
||||
|
||||
// handleVoices handles GET /api/voices.
|
||||
// Returns the list of available Kokoro voices as JSON: {"voices": [...]}
|
||||
func (s *Server) handleVoices(w http.ResponseWriter, _ *http.Request) {
|
||||
voices := s.voices()
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]interface{}{"voices": voices})
|
||||
}
|
||||
|
||||
// ─── Voice sample generation ──────────────────────────────────────────────────
|
||||
|
||||
// voiceSampleText is the short passage used for voice sample previews.
|
||||
const voiceSampleText = "The ancient library held secrets older than memory itself, its dust-laden shelves stretching upward into shadow. She reached for the worn leather spine, fingers trembling with anticipation."
|
||||
|
||||
// voiceSampleKey returns the MinIO object key for a voice sample.
|
||||
// Key: _voice-samples/{voice}.mp3
|
||||
func voiceSampleKey(voice string) string {
|
||||
safe := strings.Map(func(r rune) rune {
|
||||
if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') ||
|
||||
(r >= '0' && r <= '9') || r == '_' || r == '-' {
|
||||
return r
|
||||
}
|
||||
return '_'
|
||||
}, voice)
|
||||
return fmt.Sprintf("_voice-samples/%s.mp3", safe)
|
||||
}
|
||||
|
||||
// warmVoiceSamples runs at startup in a background goroutine.
|
||||
// It generates a short audio sample for every available Kokoro voice that
|
||||
// doesn't already have one in MinIO, so the UI voice selector has playable
|
||||
// previews without requiring a manual trigger.
|
||||
// It respects ctx cancellation and waits up to 30 s for Kokoro to become
|
||||
// reachable before giving up.
|
||||
func (s *Server) warmVoiceSamples(ctx context.Context) {
|
||||
if s.kokoroURL == "" {
|
||||
return
|
||||
}
|
||||
|
||||
// Wait for Kokoro to be reachable (it may still be starting up).
|
||||
deadline := time.Now().Add(30 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
req, _ := http.NewRequestWithContext(ctx, http.MethodGet, s.kokoroURL+"/v1/audio/voices", nil)
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err == nil {
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode == http.StatusOK {
|
||||
break
|
||||
}
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(3 * time.Second):
|
||||
}
|
||||
}
|
||||
|
||||
voices := s.voices()
|
||||
s.log.Info("warming voice samples", "voices", len(voices))
|
||||
|
||||
generated, skipped, failed := 0, 0, 0
|
||||
for _, voice := range voices {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
|
||||
key := voiceSampleKey(voice)
|
||||
if s.store.AudioExists(ctx, key) {
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
|
||||
filename, err := s.generateSpeech(ctx, voiceSampleText, voice, 1.0)
|
||||
if err != nil {
|
||||
s.log.Warn("voice sample warmup: generation failed", "voice", voice, "err", err)
|
||||
failed++
|
||||
continue
|
||||
}
|
||||
|
||||
audioData, err := s.downloadFromKokoro(ctx, filename)
|
||||
if err != nil {
|
||||
s.log.Warn("voice sample warmup: download failed", "voice", voice, "err", err)
|
||||
failed++
|
||||
continue
|
||||
}
|
||||
|
||||
if err := s.store.PutAudio(ctx, key, audioData); err != nil {
|
||||
s.log.Warn("voice sample warmup: upload failed", "voice", voice, "key", key, "err", err)
|
||||
failed++
|
||||
continue
|
||||
}
|
||||
|
||||
s.log.Debug("voice sample warmed", "voice", voice)
|
||||
generated++
|
||||
}
|
||||
|
||||
s.log.Info("voice sample warmup complete",
|
||||
"generated", generated, "skipped", skipped, "failed", failed)
|
||||
}
|
||||
|
||||
// handleGenerateVoiceSamples handles POST /api/audio/voice-samples.
|
||||
// It generates short audio samples for each available voice and stores them
|
||||
// in the audio MinIO bucket so the UI can play them during voice selection.
|
||||
// Already-generated samples are skipped (idempotent).
|
||||
// Optional JSON body: {"voices": ["af_bella", ...]} to generate a subset.
|
||||
// Returns: {"generated": [...], "skipped": [...], "failed": [...]}
|
||||
func (s *Server) handleGenerateVoiceSamples(w http.ResponseWriter, r *http.Request) {
|
||||
if s.kokoroURL == "" {
|
||||
http.Error(w, `{"error":"kokoro not configured"}`, http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
|
||||
// Parse optional voice list from body.
|
||||
var body struct {
|
||||
Voices []string `json:"voices"`
|
||||
}
|
||||
if r.Body != nil {
|
||||
_ = json.NewDecoder(r.Body).Decode(&body)
|
||||
}
|
||||
|
||||
targetVoices := body.Voices
|
||||
if len(targetVoices) == 0 {
|
||||
targetVoices = s.voices()
|
||||
}
|
||||
|
||||
type result struct {
|
||||
Generated []string `json:"generated"`
|
||||
Skipped []string `json:"skipped"`
|
||||
Failed []string `json:"failed"`
|
||||
}
|
||||
var res result
|
||||
|
||||
for _, voice := range targetVoices {
|
||||
key := voiceSampleKey(voice)
|
||||
|
||||
// Skip if already uploaded.
|
||||
if s.store.AudioExists(r.Context(), key) {
|
||||
res.Skipped = append(res.Skipped, voice)
|
||||
s.log.Debug("voice sample already exists, skipping", "voice", voice)
|
||||
continue
|
||||
}
|
||||
|
||||
// Generate via Kokoro (speed 1.0 for samples).
|
||||
filename, err := s.generateSpeech(r.Context(), voiceSampleText, voice, 1.0)
|
||||
if err != nil {
|
||||
s.log.Warn("voice sample generation failed", "voice", voice, "err", err)
|
||||
res.Failed = append(res.Failed, voice)
|
||||
continue
|
||||
}
|
||||
|
||||
// Download from Kokoro and upload to MinIO.
|
||||
audioData, dlErr := s.downloadFromKokoro(r.Context(), filename)
|
||||
if dlErr != nil {
|
||||
s.log.Warn("voice sample kokoro download failed", "voice", voice, "err", dlErr)
|
||||
res.Failed = append(res.Failed, voice)
|
||||
continue
|
||||
}
|
||||
|
||||
if putErr := s.store.PutAudio(r.Context(), key, audioData); putErr != nil {
|
||||
s.log.Warn("voice sample MinIO upload failed", "voice", voice, "key", key, "err", putErr)
|
||||
res.Failed = append(res.Failed, voice)
|
||||
continue
|
||||
}
|
||||
|
||||
s.log.Info("voice sample generated", "voice", voice, "key", key)
|
||||
res.Generated = append(res.Generated, voice)
|
||||
}
|
||||
|
||||
if res.Generated == nil {
|
||||
res.Generated = []string{}
|
||||
}
|
||||
if res.Skipped == nil {
|
||||
res.Skipped = []string{}
|
||||
}
|
||||
if res.Failed == nil {
|
||||
res.Failed = []string{}
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(res)
|
||||
}
|
||||
|
||||
// handlePresignVoiceSample handles GET /api/presign/voice-sample/{voice}.
|
||||
// Returns a presigned URL for the voice sample audio file stored in MinIO.
|
||||
// Returns 404 if the sample has not been generated yet.
|
||||
func (s *Server) handlePresignVoiceSample(w http.ResponseWriter, r *http.Request) {
|
||||
voice := r.PathValue("voice")
|
||||
if voice == "" {
|
||||
http.Error(w, `{"error":"missing voice"}`, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
key := voiceSampleKey(voice)
|
||||
|
||||
if !s.store.AudioExists(r.Context(), key) {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
url, err := s.store.PresignAudio(r.Context(), key, 1*time.Hour)
|
||||
if err != nil {
|
||||
s.log.Error("presign voice sample failed", "voice", voice, "err", err)
|
||||
http.Error(w, `{"error":"presign failed"}`, http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]string{"url": url})
|
||||
}
|
||||
476
scraper/internal/server/handlers_browse.go
Normal file
476
scraper/internal/server/handlers_browse.go
Normal file
@@ -0,0 +1,476 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/libnovel/scraper/internal/storage"
|
||||
"golang.org/x/net/html"
|
||||
|
||||
"github.com/libnovel/scraper/internal/scraper/htmlutil"
|
||||
)
|
||||
|
||||
// ─── Browse API ───────────────────────────────────────────────────────────────
|
||||
|
||||
// NovelListing represents a single novel entry from the novelfire browse page.
|
||||
type NovelListing struct {
|
||||
Slug string `json:"slug"`
|
||||
Title string `json:"title"`
|
||||
Cover string `json:"cover"`
|
||||
Rank string `json:"rank"`
|
||||
Rating string `json:"rating"`
|
||||
Chapters string `json:"chapters"`
|
||||
URL string `json:"url"`
|
||||
}
|
||||
|
||||
const novelFireBase = "https://novelfire.net"
|
||||
const novelFireDomain = "novelfire.net"
|
||||
|
||||
// handleBrowse handles GET /api/browse.
|
||||
// Query params:
|
||||
//
|
||||
// page (default 1)
|
||||
// genre (default "all")
|
||||
// sort (default "popular")
|
||||
// status (default "all")
|
||||
// type (default "all-novel")
|
||||
//
|
||||
// Returns JSON: {"novels":[...], "page": N, "hasNext": bool}
|
||||
//
|
||||
// Cache strategy: check MinIO browse bucket first (key: {domain}/html/page-N.html);
|
||||
// if a snapshot exists, parse it and return structured JSON.
|
||||
// On a cache miss, fetch live from novelfire.net, return the result, and
|
||||
// trigger a background SingleFile snapshot + ranking population.
|
||||
func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) {
|
||||
q := r.URL.Query()
|
||||
page := q.Get("page")
|
||||
if page == "" {
|
||||
page = "1"
|
||||
}
|
||||
genre := q.Get("genre")
|
||||
if genre == "" {
|
||||
genre = "all"
|
||||
}
|
||||
sortBy := q.Get("sort")
|
||||
if sortBy == "" {
|
||||
sortBy = "popular"
|
||||
}
|
||||
status := q.Get("status")
|
||||
if status == "" {
|
||||
status = "all"
|
||||
}
|
||||
novelType := q.Get("type")
|
||||
if novelType == "" {
|
||||
novelType = "all-novel"
|
||||
}
|
||||
|
||||
pageNum, _ := strconv.Atoi(page)
|
||||
if pageNum <= 0 {
|
||||
pageNum = 1
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 45*time.Second)
|
||||
defer cancel()
|
||||
|
||||
// ── Cache-first: try MinIO snapshot (new key layout) ─────────────────
|
||||
cacheKey := s.store.BrowseHTMLKey(novelFireDomain, pageNum)
|
||||
if html, ok, err := s.store.GetBrowsePage(ctx, cacheKey); err == nil && ok && len(html) > 0 {
|
||||
novels, hasNext := parseBrowsePage(strings.NewReader(html))
|
||||
s.log.Debug("browse: served from cache", "key", cacheKey)
|
||||
// Still fire background ranking population in case PocketBase ranking
|
||||
// records are missing (e.g. after a schema reset / fresh deploy).
|
||||
targetURLForRanking := fmt.Sprintf("%s/genre-%s/sort-%s/status-%s/%s?page=%s",
|
||||
novelFireBase, genre, sortBy, status, novelType, page)
|
||||
s.triggerDirectScrape(cacheKey, targetURLForRanking)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Header().Set("Cache-Control", "public, max-age=300")
|
||||
_ = json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"novels": novels,
|
||||
"page": pageNum,
|
||||
"hasNext": hasNext,
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
// ── Live fallback: direct fetch from novelfire.net ───────────────────
|
||||
// Build URL: /genre-{genre}/sort-{sort}/status-{status}/{type}?page={page}
|
||||
targetURL := fmt.Sprintf("%s/genre-%s/sort-%s/status-%s/%s?page=%s",
|
||||
novelFireBase, genre, sortBy, status, novelType, page)
|
||||
|
||||
var novels []NovelListing
|
||||
var hasNext bool
|
||||
var fetchErr error
|
||||
for attempt := 1; attempt <= 3; attempt++ {
|
||||
if attempt > 1 {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
http.Error(w, `{"error":"request cancelled"}`, http.StatusServiceUnavailable)
|
||||
return
|
||||
case <-time.After(time.Duration(attempt) * time.Second):
|
||||
}
|
||||
}
|
||||
|
||||
var req *http.Request
|
||||
req, fetchErr = http.NewRequestWithContext(ctx, http.MethodGet, targetURL, nil)
|
||||
if fetchErr != nil {
|
||||
http.Error(w, `{"error":"failed to build request"}`, http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
|
||||
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
|
||||
req.Header.Set("Accept-Language", "en-US,en;q=0.9")
|
||||
// Do NOT set Accept-Encoding manually: Go's http.Transport handles
|
||||
// transparent gzip decompression only when it adds the header itself.
|
||||
// If we set it explicitly, Transport disables auto-decompression and
|
||||
// parseBrowsePage receives raw gzip bytes instead of HTML.
|
||||
req.Header.Set("Cache-Control", "no-cache")
|
||||
req.Header.Set("Pragma", "no-cache")
|
||||
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
fetchErr = err
|
||||
s.log.Warn("browse fetch failed, retrying", "url", targetURL, "attempt", attempt, "err", err)
|
||||
continue
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
_, _ = io.Copy(io.Discard, resp.Body)
|
||||
resp.Body.Close()
|
||||
fetchErr = fmt.Errorf("upstream returned %d", resp.StatusCode)
|
||||
s.log.Warn("browse upstream error, retrying", "url", targetURL, "attempt", attempt, "status", resp.StatusCode)
|
||||
continue
|
||||
}
|
||||
|
||||
novels, hasNext = parseBrowsePage(resp.Body)
|
||||
resp.Body.Close()
|
||||
fetchErr = nil
|
||||
break
|
||||
}
|
||||
if fetchErr != nil {
|
||||
s.log.Error("browse fetch failed after retries", "url", targetURL, "err", fetchErr)
|
||||
// ── In-memory fallback: use cached result from a prior successful fetch ──
|
||||
s.browseMemCacheMu.RLock()
|
||||
entry, memHit := s.browseMemCache[cacheKey]
|
||||
s.browseMemCacheMu.RUnlock()
|
||||
if memHit {
|
||||
s.log.Warn("browse: upstream unavailable, serving stale in-memory cache",
|
||||
"key", cacheKey, "age", time.Since(entry.cachedAt).Round(time.Second))
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Header().Set("Cache-Control", "public, max-age=60")
|
||||
_ = json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"novels": entry.novels,
|
||||
"page": pageNum,
|
||||
"hasNext": entry.hasNext,
|
||||
})
|
||||
return
|
||||
}
|
||||
http.Error(w, fmt.Sprintf(`{"error":"%s"}`, fetchErr.Error()), http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
|
||||
// ── Populate in-memory cache with the fresh upstream result ──────────
|
||||
if len(novels) > 0 {
|
||||
s.browseMemCacheMu.Lock()
|
||||
s.browseMemCache[cacheKey] = browseCacheEntry{
|
||||
novels: novels,
|
||||
hasNext: hasNext,
|
||||
cachedAt: time.Now(),
|
||||
}
|
||||
s.browseMemCacheMu.Unlock()
|
||||
}
|
||||
|
||||
// ── Background: fetch and cache page directly from novelfire.net ─────
|
||||
// Fire-and-forget: stores raw HTML in MinIO and populates the ranking
|
||||
// collection in PocketBase (no browser/SingleFile needed).
|
||||
s.triggerDirectScrape(cacheKey, targetURL)
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Header().Set("Cache-Control", "public, max-age=300")
|
||||
_ = json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"novels": novels,
|
||||
"page": pageNum,
|
||||
"hasNext": hasNext,
|
||||
})
|
||||
}
|
||||
|
||||
// triggerDirectScrape fires a background goroutine that:
|
||||
// 1. Fetches pageURL directly from novelfire.net using Go's HTTP client
|
||||
// (no browser/SingleFile needed — the page is server-rendered HTML).
|
||||
// 2. Stores the raw HTML in MinIO at cacheKey so future requests are served
|
||||
// from cache without hitting the origin.
|
||||
// 3. Parses the HTML to extract novel listings.
|
||||
// 4. For each listing, upserts a ranking record in PocketBase (rank, slug,
|
||||
// title, cover key, source_url).
|
||||
// 5. Fires a separate goroutine per cover image to download and store it at
|
||||
// {domain}/assets/book-covers/{slug}.jpg in MinIO.
|
||||
//
|
||||
// It is a no-op when a refresh for this cache key is already in progress.
|
||||
// The goroutine uses a fresh context so it outlives the HTTP request.
|
||||
func (s *Server) triggerDirectScrape(cacheKey, pageURL string) {
|
||||
s.browseMu.Lock()
|
||||
if _, inflight := s.browseInFlight[cacheKey]; inflight {
|
||||
s.browseMu.Unlock()
|
||||
return
|
||||
}
|
||||
s.browseInFlight[cacheKey] = struct{}{}
|
||||
s.browseMu.Unlock()
|
||||
|
||||
go func() {
|
||||
defer func() {
|
||||
s.browseMu.Lock()
|
||||
delete(s.browseInFlight, cacheKey)
|
||||
s.browseMu.Unlock()
|
||||
}()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, pageURL, nil)
|
||||
if err != nil {
|
||||
s.log.Warn("triggerDirectScrape: build request failed", "key", cacheKey, "err", err)
|
||||
return
|
||||
}
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
|
||||
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
|
||||
req.Header.Set("Accept-Language", "en-US,en;q=0.9")
|
||||
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
s.log.Warn("triggerDirectScrape: fetch failed", "key", cacheKey, "err", err)
|
||||
return
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
s.log.Warn("triggerDirectScrape: non-200 response", "key", cacheKey, "status", resp.StatusCode)
|
||||
return
|
||||
}
|
||||
|
||||
htmlBytes, readErr := io.ReadAll(resp.Body)
|
||||
if readErr != nil {
|
||||
s.log.Warn("triggerDirectScrape: read body failed", "key", cacheKey, "err", readErr)
|
||||
return
|
||||
}
|
||||
if len(htmlBytes) == 0 {
|
||||
s.log.Warn("triggerDirectScrape: empty response body", "key", cacheKey)
|
||||
return
|
||||
}
|
||||
|
||||
// Store the HTML in MinIO so subsequent requests are cache-hits.
|
||||
if putErr := s.store.SaveBrowsePage(ctx, cacheKey, string(htmlBytes)); putErr != nil {
|
||||
s.log.Warn("triggerDirectScrape: SaveBrowsePage failed", "key", cacheKey, "err", putErr)
|
||||
// Non-fatal: continue to populate PocketBase/covers even if MinIO write fails.
|
||||
} else {
|
||||
s.log.Info("triggerDirectScrape: cached browse page", "key", cacheKey, "bytes", len(htmlBytes))
|
||||
}
|
||||
|
||||
// Parse to extract novel listings.
|
||||
novels, _ := parseBrowsePage(strings.NewReader(string(htmlBytes)))
|
||||
if len(novels) == 0 {
|
||||
s.log.Warn("triggerDirectScrape: no novels parsed", "key", cacheKey)
|
||||
return
|
||||
}
|
||||
|
||||
// Upsert each novel into PocketBase ranking and kick off cover downloads.
|
||||
for i, novel := range novels {
|
||||
rank := i + 1
|
||||
coverKey := s.store.BrowseCoverKey(novelFireDomain, novel.Slug)
|
||||
|
||||
item := storage.RankingItem{
|
||||
Rank: rank,
|
||||
Slug: novel.Slug,
|
||||
Title: novel.Title,
|
||||
Cover: coverKey, // stored as MinIO key; UI fetches via /api/cover/...
|
||||
SourceURL: novel.URL,
|
||||
}
|
||||
if werr := s.store.WriteRankingItem(ctx, item); werr != nil {
|
||||
s.log.Warn("triggerDirectScrape: WriteRankingItem failed",
|
||||
"slug", novel.Slug, "err", werr)
|
||||
}
|
||||
|
||||
if novel.Cover != "" {
|
||||
go s.downloadAndStoreCover(coverKey, novel.Cover)
|
||||
}
|
||||
}
|
||||
|
||||
s.log.Info("triggerDirectScrape: ranking populated", "count", len(novels), "key", cacheKey)
|
||||
}()
|
||||
}
|
||||
|
||||
// warmBrowseCache checks whether the browse cache for page 1 is populated in
|
||||
// MinIO and, if not, triggers a background direct scrape. This is called
|
||||
// once on server startup so the first user request is likely served from cache.
|
||||
func (s *Server) warmBrowseCache() {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
cacheKey := s.store.BrowseHTMLKey(novelFireDomain, 1)
|
||||
if _, ok, err := s.store.GetBrowsePage(ctx, cacheKey); err == nil && ok {
|
||||
s.log.Debug("warmBrowseCache: page 1 already cached, skipping")
|
||||
return
|
||||
}
|
||||
|
||||
targetURL := fmt.Sprintf("%s/genre-all/sort-popular/status-all/all-novel?page=1", novelFireBase)
|
||||
s.log.Info("warmBrowseCache: page 1 not cached, triggering background scrape")
|
||||
s.triggerDirectScrape(cacheKey, targetURL)
|
||||
}
|
||||
|
||||
// downloadAndStoreCover delegates to storage.DownloadAndStoreCover.
|
||||
func (s *Server) downloadAndStoreCover(key, imageURL string) {
|
||||
storage.DownloadAndStoreCover(s.store, s.log, key, imageURL)
|
||||
}
|
||||
|
||||
// parseBrowsePage parses the novelfire HTML and extracts novel listings.
|
||||
// Returns novels and whether a "next page" link was found.
|
||||
func parseBrowsePage(r io.Reader) ([]NovelListing, bool) {
|
||||
doc, err := html.Parse(r)
|
||||
if err != nil {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
var novels []NovelListing
|
||||
hasNext := false
|
||||
|
||||
var walk func(*html.Node)
|
||||
walk = func(n *html.Node) {
|
||||
if n.Type == html.ElementNode {
|
||||
switch n.Data {
|
||||
case "li":
|
||||
if hasClass(n, "novel-item") {
|
||||
if novel, ok := parseNovelItem(n); ok {
|
||||
novels = append(novels, novel)
|
||||
}
|
||||
}
|
||||
// pagination li with class "next"
|
||||
if hasClass(n, "next") {
|
||||
hasNext = true
|
||||
}
|
||||
case "a":
|
||||
// Detect "next" pagination link
|
||||
if hasClass(n, "next") || attrVal(n, "rel") == "next" {
|
||||
hasNext = true
|
||||
}
|
||||
// Also check aria-label="Next"
|
||||
if attrVal(n, "aria-label") == "Next" {
|
||||
hasNext = true
|
||||
}
|
||||
}
|
||||
}
|
||||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||
walk(c)
|
||||
}
|
||||
}
|
||||
walk(doc)
|
||||
return novels, hasNext
|
||||
}
|
||||
|
||||
// parseNovelItem extracts a NovelListing from a <li class="novel-item"> node.
|
||||
func parseNovelItem(li *html.Node) (NovelListing, bool) {
|
||||
var novel NovelListing
|
||||
|
||||
var walk func(*html.Node)
|
||||
walk = func(n *html.Node) {
|
||||
if n.Type == html.ElementNode {
|
||||
switch n.Data {
|
||||
case "a":
|
||||
href := attrVal(n, "href")
|
||||
if strings.HasPrefix(href, "/book/") {
|
||||
slug := strings.TrimPrefix(href, "/book/")
|
||||
slug = strings.TrimSuffix(slug, "/")
|
||||
if novel.Slug == "" {
|
||||
novel.Slug = slug
|
||||
novel.URL = novelFireBase + href
|
||||
}
|
||||
}
|
||||
case "img":
|
||||
// lazy-loaded covers use data-src
|
||||
src := attrVal(n, "data-src")
|
||||
if src == "" {
|
||||
src = attrVal(n, "src")
|
||||
}
|
||||
if src != "" && novel.Cover == "" {
|
||||
if !strings.HasPrefix(src, "http") {
|
||||
src = novelFireBase + src
|
||||
}
|
||||
novel.Cover = src
|
||||
}
|
||||
case "h4":
|
||||
if hasClass(n, "novel-title") && novel.Title == "" {
|
||||
novel.Title = strings.TrimSpace(textContent(n))
|
||||
}
|
||||
case "span":
|
||||
cls := attrVal(n, "class")
|
||||
if strings.Contains(cls, "_bl") && novel.Rank == "" {
|
||||
novel.Rank = strings.TrimSpace(textContent(n))
|
||||
}
|
||||
if strings.Contains(cls, "_br") && novel.Rating == "" {
|
||||
novel.Rating = strings.TrimSpace(textContent(n))
|
||||
}
|
||||
}
|
||||
}
|
||||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||
walk(c)
|
||||
}
|
||||
}
|
||||
walk(li)
|
||||
|
||||
// Extract chapter count from the novel stats text (contains "N Chapters")
|
||||
novel.Chapters = extractChapters(li)
|
||||
|
||||
if novel.Slug == "" || novel.Title == "" {
|
||||
return novel, false
|
||||
}
|
||||
return novel, true
|
||||
}
|
||||
|
||||
// extractChapters finds the chapter count text within a novel-item node.
|
||||
func extractChapters(n *html.Node) string {
|
||||
var result string
|
||||
var walk func(*html.Node)
|
||||
walk = func(node *html.Node) {
|
||||
if node.Type == html.ElementNode {
|
||||
cls := attrVal(node, "class")
|
||||
if strings.Contains(cls, "novel-stats") || strings.Contains(cls, "chapter") {
|
||||
txt := strings.TrimSpace(textContent(node))
|
||||
if strings.Contains(txt, "Chapter") || strings.Contains(txt, "chapter") {
|
||||
// Extract just the numeric part if possible
|
||||
result = txt
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
for c := node.FirstChild; c != nil; c = c.NextSibling {
|
||||
walk(c)
|
||||
}
|
||||
}
|
||||
walk(n)
|
||||
return result
|
||||
}
|
||||
|
||||
// hasClass reports whether an HTML node has the given CSS class.
|
||||
func hasClass(n *html.Node, cls string) bool {
|
||||
for _, a := range n.Attr {
|
||||
if a.Key == "class" {
|
||||
for _, c := range strings.Fields(a.Val) {
|
||||
if c == cls {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// attrVal returns the value of an attribute on an HTML node, or "".
|
||||
// Delegates to htmlutil.AttrVal.
|
||||
func attrVal(n *html.Node, key string) string { return htmlutil.AttrVal(n, key) }
|
||||
|
||||
// textContent returns the concatenated text content of a node and its descendants.
|
||||
// Delegates to htmlutil.TextContent.
|
||||
func textContent(n *html.Node) string { return htmlutil.TextContent(n) }
|
||||
103
scraper/internal/server/handlers_progress.go
Normal file
103
scraper/internal/server/handlers_progress.go
Normal file
@@ -0,0 +1,103 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/libnovel/scraper/internal/storage"
|
||||
)
|
||||
|
||||
// ─── Reading progress API ─────────────────────────────────────────────────────
|
||||
|
||||
// handleGetProgress handles GET /api/progress.
|
||||
// Returns JSON: {"slug": chapterNum, ...} merged with {"slug_ts": timestampMs, ...}
|
||||
func (s *Server) handleGetProgress(w http.ResponseWriter, r *http.Request) {
|
||||
sid := ensureSession(w, r)
|
||||
entries, err := s.store.AllProgress(r.Context(), sid)
|
||||
if err != nil {
|
||||
s.log.Error("AllProgress failed", "err", err)
|
||||
entries = nil
|
||||
}
|
||||
|
||||
progress := make(map[string]interface{}, len(entries)*2)
|
||||
for _, p := range entries {
|
||||
progress[p.Slug] = p.Chapter
|
||||
progress[p.Slug+"_ts"] = p.UpdatedAt.UnixMilli()
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(progress)
|
||||
}
|
||||
|
||||
// handleSetProgress handles POST /api/progress/{slug}.
|
||||
// Body: {"chapter": N}
|
||||
func (s *Server) handleSetProgress(w http.ResponseWriter, r *http.Request) {
|
||||
sid := ensureSession(w, r)
|
||||
slug := r.PathValue("slug")
|
||||
if slug == "" {
|
||||
http.Error(w, `{"error":"missing slug"}`, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
var body struct {
|
||||
Chapter int `json:"chapter"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil || body.Chapter < 1 {
|
||||
http.Error(w, `{"error":"invalid body"}`, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
p := storage.ReadingProgress{
|
||||
Slug: slug,
|
||||
Chapter: body.Chapter,
|
||||
UpdatedAt: time.Now(),
|
||||
}
|
||||
if err := s.store.SetProgress(r.Context(), sid, p); err != nil {
|
||||
s.log.Error("SetProgress failed", "slug", slug, "err", err)
|
||||
http.Error(w, `{"error":"store error"}`, http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]string{})
|
||||
}
|
||||
|
||||
// handleDeleteProgress handles DELETE /api/progress/{slug}.
|
||||
func (s *Server) handleDeleteProgress(w http.ResponseWriter, r *http.Request) {
|
||||
sid := ensureSession(w, r)
|
||||
slug := r.PathValue("slug")
|
||||
if slug == "" {
|
||||
http.Error(w, `{"error":"missing slug"}`, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
if err := s.store.DeleteProgress(r.Context(), sid, slug); err != nil {
|
||||
s.log.Error("DeleteProgress failed", "slug", slug, "err", err)
|
||||
// Non-fatal — treat as success.
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]string{})
|
||||
}
|
||||
|
||||
// handleChapterText returns the plain text of a chapter (markdown stripped)
|
||||
// for server-side audio generation. Called by handleAudioGenerate internally.
|
||||
func (s *Server) handleChapterText(w http.ResponseWriter, r *http.Request) {
|
||||
slug := r.PathValue("slug")
|
||||
n, err := strconv.Atoi(r.PathValue("n"))
|
||||
if err != nil || n < 1 {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
raw, err := s.store.ReadChapter(r.Context(), slug, n)
|
||||
if err != nil {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
|
||||
w.Header().Set("Cache-Control", "no-store")
|
||||
fmt.Fprint(w, stripMarkdown(raw))
|
||||
}
|
||||
86
scraper/internal/server/handlers_ranking.go
Normal file
86
scraper/internal/server/handlers_ranking.go
Normal file
@@ -0,0 +1,86 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/libnovel/scraper/internal/storage"
|
||||
)
|
||||
|
||||
// handleGetRanking returns all ranking items sorted by rank ascending.
|
||||
// Cover fields that hold a MinIO object key (e.g. "novelfire.net/assets/book-covers/slug.jpg")
|
||||
// are rewritten to a /api/cover/{key} proxy URL so the UI can fetch them
|
||||
// without knowing about the internal MinIO topology.
|
||||
func (s *Server) handleGetRanking(w http.ResponseWriter, r *http.Request) {
|
||||
items, err := s.store.ReadRankingItems(r.Context())
|
||||
if err != nil {
|
||||
s.log.Error("ranking read failed", "err", err)
|
||||
http.Error(w, `{"error":"failed to read ranking"}`, http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if items == nil {
|
||||
items = []storage.RankingItem{}
|
||||
}
|
||||
// Rewrite cover keys to proxy URLs.
|
||||
// Keys stored by triggerDirectScrape look like:
|
||||
// "novelfire.net/assets/book-covers/shadow-slave.jpg"
|
||||
// We expose them as:
|
||||
// "/api/cover/novelfire.net/shadow-slave"
|
||||
// (the handler strips the domain and slug from the path, reconstructs the key)
|
||||
for i := range items {
|
||||
cover := items[i].Cover
|
||||
if cover != "" && !strings.HasPrefix(cover, "http") {
|
||||
// cover is a MinIO key; extract domain + slug for the proxy path.
|
||||
// Key format: {domain}/assets/book-covers/{slug}.jpg
|
||||
parts := strings.SplitN(cover, "/assets/book-covers/", 2)
|
||||
if len(parts) == 2 {
|
||||
domain := parts[0]
|
||||
slug := strings.TrimSuffix(parts[1], ".jpg")
|
||||
items[i].Cover = "/api/cover/" + domain + "/" + slug
|
||||
}
|
||||
}
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(items)
|
||||
}
|
||||
|
||||
// handleGetCover proxies a cover image stored in the MinIO browse bucket.
|
||||
// Route: GET /api/cover/{domain}/{slug}
|
||||
// It reconstructs the MinIO key as {domain}/assets/book-covers/{slug}.jpg,
|
||||
// fetches the object, and streams it to the client.
|
||||
// Returns 404 if not yet downloaded, allowing the UI to fall back to the
|
||||
// original source URL.
|
||||
func (s *Server) handleGetCover(w http.ResponseWriter, r *http.Request) {
|
||||
domain := r.PathValue("domain")
|
||||
slug := r.PathValue("slug")
|
||||
if domain == "" || slug == "" {
|
||||
http.Error(w, "missing domain or slug", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
key := s.store.BrowseCoverKey(domain, slug)
|
||||
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
data, contentType, ok, err := s.store.GetBrowseAsset(ctx, key)
|
||||
if err != nil {
|
||||
s.log.Warn("handleGetCover: GetBrowseAsset error", "key", key, "err", err)
|
||||
http.Error(w, "storage error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if !ok {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
if contentType == "" {
|
||||
contentType = "image/jpeg"
|
||||
}
|
||||
w.Header().Set("Content-Type", contentType)
|
||||
w.Header().Set("Cache-Control", "public, max-age=86400")
|
||||
_, _ = w.Write(data)
|
||||
}
|
||||
247
scraper/internal/server/handlers_scrape.go
Normal file
247
scraper/internal/server/handlers_scrape.go
Normal file
@@ -0,0 +1,247 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/libnovel/scraper/internal/orchestrator"
|
||||
"github.com/libnovel/scraper/internal/storage"
|
||||
)
|
||||
|
||||
func (s *Server) handleScrapeCatalogue(w http.ResponseWriter, r *http.Request) {
|
||||
cfg := s.oCfg
|
||||
cfg.SingleBookURL = "" // full catalogue
|
||||
|
||||
s.runAsync(w, cfg)
|
||||
}
|
||||
|
||||
func (s *Server) handleScrapeBook(w http.ResponseWriter, r *http.Request) {
|
||||
var body struct {
|
||||
URL string `json:"url"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil || body.URL == "" {
|
||||
http.Error(w, `{"error":"request body must be JSON with \"url\" field"}`, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
cfg := s.oCfg
|
||||
cfg.SingleBookURL = body.URL
|
||||
|
||||
s.runAsync(w, cfg)
|
||||
}
|
||||
|
||||
// runAsync launches an orchestrator in the background and returns 202 Accepted.
|
||||
// Only one scrape job runs at a time; concurrent requests receive 409 Conflict.
|
||||
func (s *Server) runAsync(w http.ResponseWriter, cfg orchestrator.Config) {
|
||||
s.mu.Lock()
|
||||
if s.running {
|
||||
s.mu.Unlock()
|
||||
http.Error(w, `{"error":"a scrape job is already running"}`, http.StatusConflict)
|
||||
return
|
||||
}
|
||||
s.running = true
|
||||
s.mu.Unlock()
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusAccepted)
|
||||
_ = json.NewEncoder(w).Encode(map[string]string{"status": "accepted"})
|
||||
|
||||
go func() {
|
||||
defer func() {
|
||||
s.mu.Lock()
|
||||
s.running = false
|
||||
s.mu.Unlock()
|
||||
}()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 24*time.Hour)
|
||||
defer cancel()
|
||||
|
||||
// Determine task kind and target.
|
||||
kind := "catalogue"
|
||||
targetURL := ""
|
||||
if cfg.SingleBookURL != "" {
|
||||
kind = "book"
|
||||
targetURL = cfg.SingleBookURL
|
||||
}
|
||||
|
||||
// Create the task record in PocketBase.
|
||||
taskID, err := s.store.CreateScrapeTask(ctx, kind, targetURL)
|
||||
if err != nil {
|
||||
s.log.Warn("could not create scraping_tasks record", "err", err)
|
||||
// Non-fatal: continue without task tracking.
|
||||
}
|
||||
|
||||
// flush pushes the latest counters to PocketBase (best-effort).
|
||||
flush := func(p orchestrator.Progress, status, errMsg string, finished bool) {
|
||||
if taskID == "" {
|
||||
return
|
||||
}
|
||||
u := storage.ScrapeTaskUpdate{
|
||||
Status: status,
|
||||
BooksFound: p.BooksFound,
|
||||
ChaptersScraped: p.ChaptersScraped,
|
||||
ChaptersSkipped: p.ChaptersSkipped,
|
||||
Errors: p.Errors,
|
||||
ErrorMessage: errMsg,
|
||||
}
|
||||
if finished {
|
||||
u.Finished = time.Now().UTC()
|
||||
}
|
||||
if updateErr := s.store.UpdateScrapeTask(ctx, taskID, u); updateErr != nil {
|
||||
s.log.Warn("could not update scraping_tasks record", "task_id", taskID, "err", updateErr)
|
||||
}
|
||||
}
|
||||
|
||||
cfg.OnProgress = func(p orchestrator.Progress) {
|
||||
flush(p, "running", "", false)
|
||||
}
|
||||
|
||||
o := orchestrator.New(cfg, s.novel, s.log, s.store)
|
||||
runErr := o.Run(ctx)
|
||||
|
||||
// After a successful full-catalogue run, refresh the ranking list.
|
||||
if runErr == nil && cfg.SingleBookURL == "" {
|
||||
s.log.Info("runAsync: starting ScrapeRanking after catalogue run")
|
||||
rankCtx, rankCancel := context.WithTimeout(context.Background(), 30*time.Minute)
|
||||
defer rankCancel()
|
||||
rankEntries, rankErrs := s.novel.ScrapeRanking(rankCtx, 0)
|
||||
rank := 1
|
||||
for meta := range rankEntries {
|
||||
item := storage.RankingItem{
|
||||
Rank: rank,
|
||||
Slug: meta.Slug,
|
||||
Title: meta.Title,
|
||||
Author: meta.Author,
|
||||
Cover: meta.Cover,
|
||||
Status: meta.Status,
|
||||
Genres: meta.Genres,
|
||||
SourceURL: meta.SourceURL,
|
||||
}
|
||||
if werr := s.store.WriteRankingItem(rankCtx, item); werr != nil {
|
||||
s.log.Warn("runAsync: WriteRankingItem failed", "slug", meta.Slug, "err", werr)
|
||||
}
|
||||
rank++
|
||||
}
|
||||
if rerr := <-rankErrs; rerr != nil {
|
||||
s.log.Warn("runAsync: ScrapeRanking finished with error", "err", rerr)
|
||||
} else {
|
||||
s.log.Info("runAsync: ScrapeRanking complete", "count", rank-1)
|
||||
}
|
||||
}
|
||||
|
||||
// Determine final status.
|
||||
finalStatus := "done"
|
||||
errMsg := ""
|
||||
if runErr != nil {
|
||||
s.log.Error("scrape job failed", "err", fmt.Sprintf("%v", runErr))
|
||||
if ctx.Err() != nil {
|
||||
finalStatus = "cancelled"
|
||||
} else {
|
||||
finalStatus = "failed"
|
||||
}
|
||||
errMsg = runErr.Error()
|
||||
}
|
||||
|
||||
// Best-effort: read last known progress counters via a zero-value
|
||||
// OnProgress — we don't have a snapshot here, so re-use whatever the
|
||||
// last OnProgress call delivered (the orchestrator calls notify() at
|
||||
// the very end, so this is always accurate after Run returns).
|
||||
// We issue one final flush with the terminal status and finished time.
|
||||
if taskID != "" {
|
||||
// Re-fetch current counters by listing the task (cheapest path).
|
||||
tasks, listErr := s.store.ListScrapeTasks(ctx)
|
||||
var last storage.ScrapeTaskUpdate
|
||||
if listErr == nil {
|
||||
for _, t := range tasks {
|
||||
if t.ID == taskID {
|
||||
last = storage.ScrapeTaskUpdate{
|
||||
BooksFound: t.BooksFound,
|
||||
ChaptersScraped: t.ChaptersScraped,
|
||||
ChaptersSkipped: t.ChaptersSkipped,
|
||||
Errors: t.Errors,
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
last.Status = finalStatus
|
||||
last.ErrorMessage = errMsg
|
||||
last.Finished = time.Now().UTC()
|
||||
if updateErr := s.store.UpdateScrapeTask(ctx, taskID, last); updateErr != nil {
|
||||
s.log.Warn("could not finalize scraping_tasks record", "task_id", taskID, "err", updateErr)
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// ─── Scrape status API ────────────────────────────────────────────────────────
|
||||
|
||||
// handleScrapeStatus handles GET /api/scrape/status.
|
||||
// Returns JSON: {"running": bool}
|
||||
func (s *Server) handleScrapeStatus(w http.ResponseWriter, _ *http.Request) {
|
||||
s.mu.Lock()
|
||||
running := s.running
|
||||
s.mu.Unlock()
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]bool{"running": running})
|
||||
}
|
||||
|
||||
// handleScrapeTasks handles GET /api/scrape/tasks.
|
||||
// Returns JSON array of all scraping_tasks records, newest first.
|
||||
func (s *Server) handleScrapeTasks(w http.ResponseWriter, r *http.Request) {
|
||||
tasks, err := s.store.ListScrapeTasks(r.Context())
|
||||
if err != nil {
|
||||
s.log.Error("handleScrapeTasks: list failed", "err", err)
|
||||
http.Error(w, `{"error":"failed to list tasks"}`, http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if tasks == nil {
|
||||
tasks = []storage.ScrapeTask{}
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(tasks)
|
||||
}
|
||||
|
||||
// handleReindex handles POST /api/reindex/{slug}.
|
||||
// It rebuilds the chapters_idx PocketBase collection for the given book by
|
||||
// walking its MinIO objects. Use this when chapters were scraped but the index
|
||||
// is out of sync (e.g. after a failed UpsertChapterIdx during scraping).
|
||||
func (s *Server) handleReindex(w http.ResponseWriter, r *http.Request) {
|
||||
slug := r.PathValue("slug")
|
||||
if slug == "" {
|
||||
http.Error(w, `{"error":"missing slug"}`, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
type reindexer interface {
|
||||
ReindexChapters(ctx context.Context, slug string) (int, error)
|
||||
}
|
||||
ri, ok := s.store.(reindexer)
|
||||
if !ok {
|
||||
http.Error(w, `{"error":"store does not support reindex"}`, http.StatusNotImplemented)
|
||||
return
|
||||
}
|
||||
|
||||
count, err := ri.ReindexChapters(r.Context(), slug)
|
||||
if err != nil {
|
||||
s.log.Error("reindex failed", "slug", slug, "indexed", count, "err", err)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
_ = json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
"indexed": count,
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
s.log.Info("reindex complete", "slug", slug, "indexed", count)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"slug": slug,
|
||||
"indexed": count,
|
||||
})
|
||||
}
|
||||
@@ -335,6 +335,9 @@ func TestServer_PresignChapter_NotFound(t *testing.T) {
|
||||
// MinIO presign on a non-existent key returns an error; server returns 500.
|
||||
// (Some MinIO versions return a valid presigned URL anyway, which is also acceptable.)
|
||||
t.Logf("presign non-existent chapter status: %d", resp.StatusCode)
|
||||
if resp.StatusCode != http.StatusInternalServerError && resp.StatusCode != http.StatusOK {
|
||||
t.Errorf("status = %d, want 500 or 200", resp.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
// TestServer_ChapterText writes a chapter and verifies
|
||||
@@ -395,13 +398,6 @@ func mapKeys(m map[string]interface{}) []string {
|
||||
return keys
|
||||
}
|
||||
|
||||
func min(a, b int) int {
|
||||
if a < b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// cookieJar is a minimal http.CookieJar that stores cookies by host.
|
||||
type cookieJar struct {
|
||||
cookies map[string][]*http.Cookie
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
59
scraper/internal/storage/coverutil.go
Normal file
59
scraper/internal/storage/coverutil.go
Normal file
@@ -0,0 +1,59 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"time"
|
||||
)
|
||||
|
||||
// DownloadAndStoreCover fetches the image at imageURL and stores it in the
|
||||
// store under key. Errors are logged but not returned — this is best-effort.
|
||||
// If the asset is already present the download is skipped.
|
||||
func DownloadAndStoreCover(store Store, log *slog.Logger, key, imageURL string) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
// Skip if already stored.
|
||||
if _, _, ok, _ := store.GetBrowseAsset(ctx, key); ok {
|
||||
return
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, imageURL, nil)
|
||||
if err != nil {
|
||||
log.Warn("cover: build request failed", "key", key, "url", imageURL, "err", err)
|
||||
return
|
||||
}
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; libnovel-scraper/1.0)")
|
||||
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
log.Warn("cover: fetch failed", "key", key, "url", imageURL, "err", fmt.Errorf("%w", err))
|
||||
return
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
log.Warn("cover: non-200 response", "key", key, "url", imageURL, "status", resp.StatusCode)
|
||||
return
|
||||
}
|
||||
|
||||
data, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
log.Warn("cover: read body failed", "key", key, "url", imageURL, "err", err)
|
||||
return
|
||||
}
|
||||
|
||||
contentType := resp.Header.Get("Content-Type")
|
||||
if contentType == "" {
|
||||
contentType = "image/jpeg"
|
||||
}
|
||||
|
||||
if err := store.SaveBrowseAsset(ctx, key, data, contentType); err != nil {
|
||||
log.Warn("cover: SaveBrowseAsset failed", "key", key, "err", err)
|
||||
return
|
||||
}
|
||||
log.Debug("cover: stored", "key", key, "bytes", len(data))
|
||||
}
|
||||
@@ -193,7 +193,6 @@ func (h *HybridStore) ReindexChapters(ctx context.Context, slug string) (int, er
|
||||
// form "{slug}/vol-N/lo-hi/chapter-N.md".
|
||||
func chapterNumberFromKey(key string) int {
|
||||
// Grab the filename portion after the last '/'.
|
||||
_, file, _ := strings.Cut(key, "/")
|
||||
parts := strings.Split(key, "/")
|
||||
if len(parts) == 0 {
|
||||
return 0
|
||||
@@ -206,7 +205,6 @@ func chapterNumberFromKey(key string) int {
|
||||
if err != nil || n <= 0 {
|
||||
return 0
|
||||
}
|
||||
_ = file
|
||||
return n
|
||||
}
|
||||
|
||||
@@ -443,24 +441,35 @@ func splitChapterTitle(raw string) (title, date string) {
|
||||
raw = strings.TrimSpace(raw[idx:])
|
||||
}
|
||||
}
|
||||
// Detect trailing relative date.
|
||||
// Detect trailing relative date. Build a flat list of all suffixes once
|
||||
// to avoid a double-nested loop.
|
||||
units := []string{"second", "minute", "hour", "day", "week", "month", "year"}
|
||||
lower := strings.ToLower(raw)
|
||||
suffixes := make([]string, 0, len(units)*2)
|
||||
for _, u := range units {
|
||||
for _, suffix := range []string{u + "s ago", u + " ago"} {
|
||||
if idx := strings.LastIndex(lower, suffix); idx > 0 {
|
||||
// Find start of date token (digit before the unit).
|
||||
start := strings.LastIndex(raw[:idx], " ")
|
||||
if start < 0 {
|
||||
start = 0
|
||||
}
|
||||
numPart := strings.TrimSpace(raw[start:idx])
|
||||
fields := strings.Fields(numPart)
|
||||
if len(fields) > 0 {
|
||||
if _, err := strconv.Atoi(fields[0]); err == nil {
|
||||
return strings.TrimSpace(raw[:start]), strings.TrimSpace(raw[start : idx+len(suffix)])
|
||||
}
|
||||
}
|
||||
suffixes = append(suffixes, u+"s ago", u+" ago")
|
||||
}
|
||||
lower := strings.ToLower(raw)
|
||||
for _, suffix := range suffixes {
|
||||
idx := strings.LastIndex(lower, suffix)
|
||||
if idx <= 0 {
|
||||
continue
|
||||
}
|
||||
// Find start of the numeric token that precedes the unit.
|
||||
// Strip any whitespace that separates the number from the unit so
|
||||
// that LastIndex finds the space before the digit, not the one
|
||||
// between the digit and the unit word.
|
||||
before := strings.TrimRight(raw[:idx], " \t")
|
||||
start := strings.LastIndex(before, " ")
|
||||
if start < 0 {
|
||||
start = 0
|
||||
} else {
|
||||
start++ // advance past the space to point at the digit
|
||||
}
|
||||
numPart := strings.TrimSpace(raw[start:idx])
|
||||
fields := strings.Fields(numPart)
|
||||
if len(fields) > 0 {
|
||||
if _, err := strconv.Atoi(fields[0]); err == nil {
|
||||
return strings.TrimSpace(raw[:start]), strings.TrimSpace(raw[start : idx+len(suffix)])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,6 +13,7 @@ package storage
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -48,7 +49,7 @@ func newTestHybridStore(t *testing.T) *HybridStore {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
hs, err := NewHybridStore(ctx, pbCfg, minioCfg)
|
||||
hs, err := NewHybridStore(ctx, pbCfg, minioCfg, slog.Default())
|
||||
if err != nil {
|
||||
t.Fatalf("NewHybridStore: %v", err)
|
||||
}
|
||||
@@ -418,7 +419,7 @@ func TestHybridStore_PresignAudio(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
key := hs.AudioObjectKey(slug, 1, "af_bella", 1.0)
|
||||
key := hs.AudioObjectKey(slug, 1, "af_bella")
|
||||
fakeAudio := []byte("ID3\x03\x00\x00\x00\x00\x00\x00hybrid-presign-audio-test")
|
||||
|
||||
if err := hs.minio.PutAudio(ctx, key, fakeAudio); err != nil {
|
||||
@@ -470,10 +471,3 @@ func TestHybridStore_AudioCache(t *testing.T) {
|
||||
}
|
||||
|
||||
// ─── helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
func min(a, b int) int {
|
||||
if a < b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
77
scraper/internal/storage/hybrid_unit_test.go
Normal file
77
scraper/internal/storage/hybrid_unit_test.go
Normal file
@@ -0,0 +1,77 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
// ── chapterNumberFromKey ──────────────────────────────────────────────────────
|
||||
|
||||
func TestChapterNumberFromKey(t *testing.T) {
|
||||
cases := []struct {
|
||||
key string
|
||||
want int
|
||||
}{
|
||||
// Standard four-segment key.
|
||||
{"my-novel/vol-0/1-50/chapter-1.md", 1},
|
||||
{"my-novel/vol-0/1-50/chapter-42.md", 42},
|
||||
{"my-novel/vol-0/51-100/chapter-99.md", 99},
|
||||
// Large chapter numbers.
|
||||
{"some-novel/vol-1/1001-1050/chapter-1024.md", 1024},
|
||||
// Nested deeper paths should still work (last segment used).
|
||||
{"a/b/c/d/chapter-7.md", 7},
|
||||
// Malformed / unexpected inputs — should return 0 without panicking.
|
||||
{"chapter-notanumber.md", 0},
|
||||
{"", 0},
|
||||
// No .md extension — TrimSuffix is a no-op; TrimPrefix still strips
|
||||
// "chapter-", so the number is parsed successfully.
|
||||
{"no-md-extension/chapter-5", 5},
|
||||
{"my-novel/vol-0/1-50/chapter-0.md", 0}, // 0 is invalid (chapters are 1-based)
|
||||
{"my-novel/vol-0/1-50/chapter--1.md", 0},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
got := chapterNumberFromKey(tc.key)
|
||||
if got != tc.want {
|
||||
t.Errorf("chapterNumberFromKey(%q) = %d, want %d", tc.key, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── splitChapterTitle ─────────────────────────────────────────────────────────
|
||||
|
||||
func TestSplitChapterTitle(t *testing.T) {
|
||||
cases := []struct {
|
||||
raw string
|
||||
wantTitle string
|
||||
wantDate string
|
||||
}{
|
||||
// No date — title is returned as-is.
|
||||
{"The Great Battle", "The Great Battle", ""},
|
||||
// Leading numeric index is stripped.
|
||||
{"42 The Great Battle", "The Great Battle", ""},
|
||||
// Relative date with plural unit.
|
||||
{"The Storm Arrives 3 days ago", "The Storm Arrives", "3 days ago"},
|
||||
// Singular unit.
|
||||
{"A New Hope 1 week ago", "A New Hope", "1 week ago"},
|
||||
// Minutes and seconds.
|
||||
{"Flash Fight 5 minutes ago", "Flash Fight", "5 minutes ago"},
|
||||
{"Quick Strike 30 seconds ago", "Quick Strike", "30 seconds ago"},
|
||||
// Months and years.
|
||||
{"Old Chapter 2 months ago", "Old Chapter", "2 months ago"},
|
||||
{"Ancient Story 1 year ago", "Ancient Story", "1 year ago"},
|
||||
// Leading index AND trailing date.
|
||||
{"5 The Final Chapter 2 hours ago", "The Final Chapter", "2 hours ago"},
|
||||
// Extra whitespace.
|
||||
{" The Calm ", "The Calm", ""},
|
||||
// Empty string.
|
||||
{"", "", ""},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
title, date := splitChapterTitle(tc.raw)
|
||||
if title != tc.wantTitle || date != tc.wantDate {
|
||||
t.Errorf("splitChapterTitle(%q) = (%q, %q), want (%q, %q)",
|
||||
tc.raw, title, date, tc.wantTitle, tc.wantDate)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -16,6 +16,7 @@ package storage
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -66,7 +67,7 @@ func newTestPocketBaseStore(t *testing.T) *PocketBaseStore {
|
||||
AdminEmail: envOr("POCKETBASE_ADMIN_EMAIL", "admin@libnovel.local"),
|
||||
AdminPassword: envOr("POCKETBASE_ADMIN_PASSWORD", "changeme123"),
|
||||
}
|
||||
store := NewPocketBaseStore(cfg)
|
||||
store := NewPocketBaseStore(cfg, slog.Default())
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
if err := store.EnsureCollections(ctx); err != nil {
|
||||
@@ -216,7 +217,7 @@ func TestMinioClient_PresignChapter(t *testing.T) {
|
||||
func TestMinioClient_AudioRoundTrip(t *testing.T) {
|
||||
mc := newTestMinioClient(t)
|
||||
slug := testSlug(t)
|
||||
key := AudioObjectKey(slug, 1, "af_bella", 1.0)
|
||||
key := AudioObjectKey(slug, 1, "af_bella")
|
||||
|
||||
// Use minimal fake MP3 bytes (just a recognisable prefix).
|
||||
fakeAudio := []byte("ID3\x03\x00\x00\x00\x00\x00\x00integration-test-audio")
|
||||
@@ -256,7 +257,7 @@ func TestMinioClient_AudioRoundTrip(t *testing.T) {
|
||||
func TestMinioClient_PresignAudio(t *testing.T) {
|
||||
mc := newTestMinioClient(t)
|
||||
slug := testSlug(t)
|
||||
key := AudioObjectKey(slug, 1, "af_bella", 1.0)
|
||||
key := AudioObjectKey(slug, 1, "af_bella")
|
||||
fakeAudio := []byte("ID3\x03\x00\x00\x00\x00\x00\x00presign-audio-test")
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
|
||||
@@ -163,33 +163,48 @@ func (p *pbClient) listOne(ctx context.Context, collection, filter string) (map[
|
||||
return result.Items[0], nil
|
||||
}
|
||||
|
||||
// listAll returns all records (up to 500) from a collection matching filter.
|
||||
// listAll returns all records from a collection matching filter by paginating
|
||||
// through all pages (PocketBase default page size is capped at 500).
|
||||
func (p *pbClient) listAll(ctx context.Context, collection, filter, sort string) ([]map[string]interface{}, error) {
|
||||
q := url.Values{}
|
||||
if filter != "" {
|
||||
q.Set("filter", filter)
|
||||
const perPage = 500
|
||||
var all []map[string]interface{}
|
||||
|
||||
for page := 1; ; page++ {
|
||||
q := url.Values{}
|
||||
if filter != "" {
|
||||
q.Set("filter", filter)
|
||||
}
|
||||
if sort != "" {
|
||||
q.Set("sort", sort)
|
||||
}
|
||||
q.Set("perPage", fmt.Sprintf("%d", perPage))
|
||||
q.Set("page", fmt.Sprintf("%d", page))
|
||||
path := fmt.Sprintf("/api/collections/%s/records?%s", collection, q.Encode())
|
||||
resp, err := p.do(ctx, http.MethodGet, path, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
return nil, fmt.Errorf("pocketbase: listAll %s: status %d: %s", collection, resp.StatusCode, b)
|
||||
}
|
||||
var result struct {
|
||||
Page int `json:"page"`
|
||||
TotalPages int `json:"totalPages"`
|
||||
Items []map[string]interface{} `json:"items"`
|
||||
}
|
||||
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
||||
resp.Body.Close()
|
||||
return nil, fmt.Errorf("pocketbase: listAll %s: decode: %w", collection, err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
all = append(all, result.Items...)
|
||||
if page >= result.TotalPages || len(result.Items) == 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
if sort != "" {
|
||||
q.Set("sort", sort)
|
||||
}
|
||||
q.Set("perPage", "500")
|
||||
path := fmt.Sprintf("/api/collections/%s/records?%s", collection, q.Encode())
|
||||
resp, err := p.do(ctx, http.MethodGet, path, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
return nil, fmt.Errorf("pocketbase: listAll %s: status %d: %s", collection, resp.StatusCode, b)
|
||||
}
|
||||
var result struct {
|
||||
Items []map[string]interface{} `json:"items"`
|
||||
}
|
||||
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
||||
return nil, fmt.Errorf("pocketbase: listAll %s: decode: %w", collection, err)
|
||||
}
|
||||
return result.Items, nil
|
||||
return all, nil
|
||||
}
|
||||
|
||||
// upsert creates a record; if one matching filter already exists it updates it.
|
||||
@@ -238,11 +253,12 @@ func (p *pbClient) deleteWhere(ctx context.Context, collection, filter string) e
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusNoContent && resp.StatusCode != http.StatusOK {
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
return fmt.Errorf("pocketbase: deleteWhere %s id=%s: status %d: %s", collection, id, resp.StatusCode, b)
|
||||
}
|
||||
resp.Body.Close()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1,476 +0,0 @@
|
||||
// Package writer handles persistence of scraped chapters and metadata.
|
||||
//
|
||||
// Directory layout:
|
||||
//
|
||||
// static/books/
|
||||
// ├── {book-slug}/
|
||||
// │ ├── metadata.yaml
|
||||
// │ ├── vol-0/ (no volume grouping)
|
||||
// │ │ ├── 1-50/
|
||||
// │ │ │ ├── chapter-1.md
|
||||
// │ │ │ └── …
|
||||
// │ │ └── 51-100/
|
||||
// │ │ └── …
|
||||
// │ └── vol-1/
|
||||
// │ └── …
|
||||
package writer
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/libnovel/scraper/internal/scraper"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
const chaptersPerFolder = 50
|
||||
|
||||
// Writer persists scraped content under a configurable root directory.
|
||||
type Writer struct {
|
||||
root string // e.g. "./static/books"
|
||||
}
|
||||
|
||||
// New creates a Writer that stores files under root.
|
||||
func New(root string) *Writer {
|
||||
return &Writer{root: root}
|
||||
}
|
||||
|
||||
// ─── Metadata ─────────────────────────────────────────────────────────────────
|
||||
|
||||
// WriteMetadata serialises meta to static/books/{slug}/metadata.yaml.
|
||||
// It creates the directory if it does not exist and overwrites any existing file.
|
||||
func (w *Writer) WriteMetadata(meta scraper.BookMeta) error {
|
||||
dir := w.bookDir(meta.Slug)
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
return fmt.Errorf("writer: mkdir %s: %w", dir, err)
|
||||
}
|
||||
|
||||
path := filepath.Join(dir, "metadata.yaml")
|
||||
f, err := os.Create(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("writer: create metadata %s: %w", path, err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
enc := yaml.NewEncoder(f)
|
||||
enc.SetIndent(2)
|
||||
if err := enc.Encode(meta); err != nil {
|
||||
return fmt.Errorf("writer: encode metadata: %w", err)
|
||||
}
|
||||
return enc.Close()
|
||||
}
|
||||
|
||||
// ReadMetadata reads the metadata.yaml for slug if it exists.
|
||||
// Returns (zero-value, false, nil) when the file does not exist.
|
||||
func (w *Writer) ReadMetadata(slug string) (scraper.BookMeta, bool, error) {
|
||||
path := filepath.Join(w.bookDir(slug), "metadata.yaml")
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return scraper.BookMeta{}, false, nil
|
||||
}
|
||||
return scraper.BookMeta{}, false, fmt.Errorf("writer: read metadata %s: %w", path, err)
|
||||
}
|
||||
|
||||
var meta scraper.BookMeta
|
||||
if err := yaml.Unmarshal(data, &meta); err != nil {
|
||||
return scraper.BookMeta{}, true, fmt.Errorf("writer: unmarshal metadata %s: %w", path, err)
|
||||
}
|
||||
return meta, true, nil
|
||||
}
|
||||
|
||||
// MetadataMtime returns the modification time (Unix seconds) of the
|
||||
// metadata.yaml file for slug, or 0 if the file cannot be stat'd.
|
||||
func (w *Writer) MetadataMtime(slug string) int64 {
|
||||
path := filepath.Join(w.bookDir(slug), "metadata.yaml")
|
||||
fi, err := os.Stat(path)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
return fi.ModTime().Unix()
|
||||
}
|
||||
|
||||
// ─── Chapters ─────────────────────────────────────────────────────────────────
|
||||
|
||||
// ChapterExists returns true if the markdown file for ref already exists on disk.
|
||||
func (w *Writer) ChapterExists(slug string, ref scraper.ChapterRef) bool {
|
||||
_, err := os.Stat(w.chapterPath(slug, ref))
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// WriteChapter writes chapter.Text to the appropriate markdown file.
|
||||
// The parent directories are created on demand.
|
||||
func (w *Writer) WriteChapter(slug string, chapter scraper.Chapter) error {
|
||||
path := w.chapterPath(slug, chapter.Ref)
|
||||
dir := filepath.Dir(path)
|
||||
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
return fmt.Errorf("writer: mkdir %s: %w", dir, err)
|
||||
}
|
||||
|
||||
// Build the markdown document.
|
||||
var sb strings.Builder
|
||||
sb.WriteString("# ")
|
||||
sb.WriteString(chapter.Ref.Title)
|
||||
sb.WriteString("\n\n")
|
||||
sb.WriteString(chapter.Text)
|
||||
sb.WriteString("\n")
|
||||
|
||||
if err := os.WriteFile(path, []byte(sb.String()), 0o644); err != nil {
|
||||
return fmt.Errorf("writer: write chapter %s: %w", path, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ─── Catalogue helpers ────────────────────────────────────────────────────────
|
||||
|
||||
// ListBooks returns metadata for every book that has a metadata.yaml under root.
|
||||
// Books with unreadable metadata files are silently skipped.
|
||||
func (w *Writer) ListBooks() ([]scraper.BookMeta, error) {
|
||||
entries, err := os.ReadDir(w.root)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, fmt.Errorf("writer: list books: %w", err)
|
||||
}
|
||||
var books []scraper.BookMeta
|
||||
for _, e := range entries {
|
||||
if !e.IsDir() {
|
||||
continue
|
||||
}
|
||||
meta, ok, _ := w.ReadMetadata(e.Name())
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
books = append(books, meta)
|
||||
}
|
||||
sort.Slice(books, func(i, j int) bool {
|
||||
return books[i].Title < books[j].Title
|
||||
})
|
||||
return books, nil
|
||||
}
|
||||
|
||||
// LocalSlugs returns the set of book slugs that have a metadata.yaml on disk.
|
||||
// It is cheaper than ListBooks because it only checks for file existence rather
|
||||
// than fully parsing every YAML file.
|
||||
func (w *Writer) LocalSlugs() map[string]bool {
|
||||
entries, err := os.ReadDir(w.root)
|
||||
if err != nil {
|
||||
return map[string]bool{}
|
||||
}
|
||||
slugs := make(map[string]bool, len(entries))
|
||||
for _, e := range entries {
|
||||
if !e.IsDir() {
|
||||
continue
|
||||
}
|
||||
metaPath := filepath.Join(w.root, e.Name(), "metadata.yaml")
|
||||
if _, err := os.Stat(metaPath); err == nil {
|
||||
slugs[e.Name()] = true
|
||||
}
|
||||
}
|
||||
return slugs
|
||||
}
|
||||
|
||||
// ChapterInfo is a lightweight chapter descriptor derived from on-disk files.
|
||||
type ChapterInfo struct {
|
||||
Number int
|
||||
Title string // chapter name, cleaned of number prefix and trailing date
|
||||
Date string // relative date scraped alongside the title, e.g. "1 year ago"
|
||||
}
|
||||
|
||||
// ListChapters returns all chapters on disk for slug, sorted by number.
|
||||
func (w *Writer) ListChapters(slug string) ([]ChapterInfo, error) {
|
||||
bookDir := w.bookDir(slug)
|
||||
var chapters []ChapterInfo
|
||||
|
||||
// Walk vol-*/range-*/ directories.
|
||||
volDirs, err := filepath.Glob(filepath.Join(bookDir, "vol-*"))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("writer: list chapters glob: %w", err)
|
||||
}
|
||||
for _, vd := range volDirs {
|
||||
rangeDirs, _ := filepath.Glob(filepath.Join(vd, "*-*"))
|
||||
for _, rd := range rangeDirs {
|
||||
files, _ := filepath.Glob(filepath.Join(rd, "chapter-*.md"))
|
||||
for _, f := range files {
|
||||
base := filepath.Base(f) // chapter-N.md
|
||||
numStr := strings.TrimSuffix(strings.TrimPrefix(base, "chapter-"), ".md")
|
||||
n, err := strconv.Atoi(numStr)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
title, date := chapterTitle(f, n)
|
||||
chapters = append(chapters, ChapterInfo{Number: n, Title: title, Date: date})
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Slice(chapters, func(i, j int) bool {
|
||||
return chapters[i].Number < chapters[j].Number
|
||||
})
|
||||
return chapters, nil
|
||||
}
|
||||
|
||||
// CountChapters returns the number of chapter markdown files on disk for slug.
|
||||
// It is cheaper than ListChapters because it does not read file contents.
|
||||
func (w *Writer) CountChapters(slug string) int {
|
||||
bookDir := w.bookDir(slug)
|
||||
volDirs, err := filepath.Glob(filepath.Join(bookDir, "vol-*"))
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
count := 0
|
||||
for _, vd := range volDirs {
|
||||
rangeDirs, _ := filepath.Glob(filepath.Join(vd, "*-*"))
|
||||
for _, rd := range rangeDirs {
|
||||
files, _ := filepath.Glob(filepath.Join(rd, "chapter-*.md"))
|
||||
count += len(files)
|
||||
}
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
// chapterTitle reads the first non-empty line of a markdown file and strips
|
||||
// the leading "# " heading marker. Falls back to "Chapter N".
|
||||
func chapterTitle(path string, n int) (title, date string) {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return fmt.Sprintf("Chapter %d", n), ""
|
||||
}
|
||||
for _, line := range strings.SplitN(string(data), "\n", 10) {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
line = strings.TrimPrefix(line, "# ")
|
||||
return SplitChapterTitle(line)
|
||||
}
|
||||
return fmt.Sprintf("Chapter %d", n), ""
|
||||
}
|
||||
|
||||
// SplitChapterTitle separates the human-readable chapter name from the
|
||||
// trailing relative-date string that novelfire.net appends to the heading.
|
||||
// Examples of raw heading text (after stripping "# "):
|
||||
//
|
||||
// "1 Chapter 1 - 1: The Academy's Weakest1 year ago"
|
||||
// "2 Chapter 2 - Enter the Storm3 months ago"
|
||||
//
|
||||
// The pattern is: optional leading number+whitespace, then the real title,
|
||||
// then a date that matches /\d+\s+(second|minute|hour|day|week|month|year)s?\s+ago$/
|
||||
func SplitChapterTitle(raw string) (title, date string) {
|
||||
// Strip a leading chapter-number index that novelfire sometimes prepends.
|
||||
// It looks like "1 " or "12 " at the very start.
|
||||
raw = strings.TrimSpace(raw)
|
||||
if idx := strings.IndexFunc(raw, func(r rune) bool { return r == ' ' || r == '\t' }); idx > 0 {
|
||||
prefix := raw[:idx]
|
||||
allDigit := true
|
||||
for _, c := range prefix {
|
||||
if c < '0' || c > '9' {
|
||||
allDigit = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if allDigit {
|
||||
raw = strings.TrimSpace(raw[idx:])
|
||||
}
|
||||
}
|
||||
|
||||
// Strip "Chapter N - N: " prefix (novelfire double-number format).
|
||||
// Also handles "Chapter N: " (single number) and "Chapter N - Title" without colon.
|
||||
chNumRe := regexp.MustCompile(`(?i)^chapter\s+\d+(?:\s*-\s*\d+)?\s*:\s*`)
|
||||
raw = strings.TrimSpace(chNumRe.ReplaceAllString(raw, ""))
|
||||
|
||||
// Match a trailing relative date: "<n> <unit>[s] ago"
|
||||
dateRe := regexp.MustCompile(`\s*(\d+\s+(?:second|minute|hour|day|week|month|year)s?\s+ago)\s*$`)
|
||||
if m := dateRe.FindStringSubmatchIndex(raw); m != nil {
|
||||
return strings.TrimSpace(raw[:m[0]]), strings.TrimSpace(raw[m[2]:m[3]])
|
||||
}
|
||||
return raw, ""
|
||||
}
|
||||
|
||||
// ReadChapter returns the raw markdown content for chapter number n of slug.
|
||||
func (w *Writer) ReadChapter(slug string, n int) (string, error) {
|
||||
// Reconstruct path using the same bucketing formula as chapterPath.
|
||||
ref := scraper.ChapterRef{Number: n, Volume: 0}
|
||||
path := w.chapterPath(slug, ref)
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("writer: read chapter %d: %w", n, err)
|
||||
}
|
||||
return string(data), nil
|
||||
}
|
||||
|
||||
// ─── Ranking ─────────────────────────────────────────────────────────────────
|
||||
|
||||
// RankingItem represents a single entry in the ranking.
|
||||
type RankingItem struct {
|
||||
Rank int `yaml:"rank" json:"rank"`
|
||||
Slug string `yaml:"slug" json:"slug"`
|
||||
Title string `yaml:"title" json:"title"`
|
||||
Author string `yaml:"author,omitempty" json:"author,omitempty"`
|
||||
Cover string `yaml:"cover,omitempty" json:"cover,omitempty"`
|
||||
Status string `yaml:"status,omitempty" json:"status,omitempty"`
|
||||
Genres []string `yaml:"genres,omitempty" json:"genres,omitempty"`
|
||||
SourceURL string `yaml:"source_url,omitempty" json:"source_url,omitempty"`
|
||||
}
|
||||
|
||||
// WriteRanking saves the ranking items as JSON to static/books/ranking.json.
|
||||
// This replaces the old markdown table format with a structured format that
|
||||
// is faster to read back (no custom parsing) and safe for titles containing "|".
|
||||
func (w *Writer) WriteRanking(items []RankingItem) error {
|
||||
path := filepath.Clean(w.rankingPath())
|
||||
dir := filepath.Dir(path)
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
return fmt.Errorf("writer: mkdir %s: %w", dir, err)
|
||||
}
|
||||
|
||||
data, err := json.MarshalIndent(items, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("writer: marshal ranking: %w", err)
|
||||
}
|
||||
if err := os.WriteFile(path, data, 0o644); err != nil {
|
||||
return fmt.Errorf("writer: write ranking %s: %w", path, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ReadRankingItems parses ranking.json into a slice of RankingItem.
|
||||
// Returns nil slice (not an error) when the file does not exist yet.
|
||||
func (w *Writer) ReadRankingItems() ([]RankingItem, error) {
|
||||
data, err := os.ReadFile(w.rankingPath())
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, fmt.Errorf("writer: read ranking: %w", err)
|
||||
}
|
||||
var items []RankingItem
|
||||
if err := json.Unmarshal(data, &items); err != nil {
|
||||
return nil, fmt.Errorf("writer: parse ranking json: %w", err)
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
// RankingFileInfo returns os.FileInfo for the ranking.json file, if it exists.
|
||||
func (w *Writer) RankingFileInfo() (os.FileInfo, error) {
|
||||
return os.Stat(w.rankingPath())
|
||||
}
|
||||
|
||||
func (w *Writer) rankingPath() string {
|
||||
return filepath.Join(w.root, "ranking.json")
|
||||
}
|
||||
|
||||
// ─── Ranking page HTML cache ──────────────────────────────────────────────────
|
||||
|
||||
// rankingCacheDir returns the directory that stores per-page HTML caches.
|
||||
func (w *Writer) rankingCacheDir() string {
|
||||
return filepath.Join(w.root, "_ranking_cache")
|
||||
}
|
||||
|
||||
// rankingPageCachePath returns the path for a cached ranking page HTML file.
|
||||
func (w *Writer) rankingPageCachePath(page int) string {
|
||||
return filepath.Join(w.rankingCacheDir(), fmt.Sprintf("page-%d.html", page))
|
||||
}
|
||||
|
||||
// WriteRankingPageCache persists raw HTML for the given ranking page number.
|
||||
func (w *Writer) WriteRankingPageCache(page int, html string) error {
|
||||
dir := w.rankingCacheDir()
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
return fmt.Errorf("writer: mkdir ranking cache %s: %w", dir, err)
|
||||
}
|
||||
path := w.rankingPageCachePath(page)
|
||||
if err := os.WriteFile(path, []byte(html), 0o644); err != nil {
|
||||
return fmt.Errorf("writer: write ranking page cache %s: %w", path, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ReadRankingPageCache reads the cached HTML for the given ranking page.
|
||||
// Returns ("", nil) when no cache file exists yet.
|
||||
func (w *Writer) ReadRankingPageCache(page int) (string, error) {
|
||||
data, err := os.ReadFile(w.rankingPageCachePath(page))
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return "", nil
|
||||
}
|
||||
return "", fmt.Errorf("writer: read ranking page cache page %d: %w", page, err)
|
||||
}
|
||||
return string(data), nil
|
||||
}
|
||||
|
||||
// RankingPageCacheInfo returns os.FileInfo for a cached ranking page file.
|
||||
// Returns (nil, nil) when the file does not exist.
|
||||
func (w *Writer) RankingPageCacheInfo(page int) (os.FileInfo, error) {
|
||||
info, err := os.Stat(w.rankingPageCachePath(page))
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
return info, nil
|
||||
}
|
||||
|
||||
// bookDir returns the root directory for a book slug.
|
||||
func (w *Writer) bookDir(slug string) string {
|
||||
return filepath.Join(w.root, slug)
|
||||
}
|
||||
|
||||
// AudioDir returns the directory used to cache generated MP3 files for a book.
|
||||
func (w *Writer) AudioDir(slug string) string {
|
||||
return filepath.Join(w.bookDir(slug), "audio")
|
||||
}
|
||||
|
||||
// AudioPath returns the full path for a cached chapter audio file.
|
||||
// The filename is keyed by chapter number, voice, and speed so that different
|
||||
// settings never collide. Speed is formatted to one decimal place (e.g. "1.0").
|
||||
func (w *Writer) AudioPath(slug string, n int, voice string, speed float64) string {
|
||||
safeVoice := sanitiseVoice(voice)
|
||||
filename := fmt.Sprintf("ch%d-%s-%.1f.mp3", n, safeVoice, speed)
|
||||
return filepath.Join(w.AudioDir(slug), filename)
|
||||
}
|
||||
|
||||
// AudioPartPath returns the path for an individual audio chunk generated during
|
||||
// chunked TTS. Part files are named ch{n}-{voice}-{speed}.part{p}.mp3 and are
|
||||
// deleted after they have been merged into the final AudioPath file.
|
||||
func (w *Writer) AudioPartPath(slug string, n int, voice string, speed float64, part int) string {
|
||||
safeVoice := sanitiseVoice(voice)
|
||||
filename := fmt.Sprintf("ch%d-%s-%.1f.part%d.mp3", n, safeVoice, speed, part)
|
||||
return filepath.Join(w.AudioDir(slug), filename)
|
||||
}
|
||||
|
||||
// sanitiseVoice converts a voice name into a string that is safe to embed in a
|
||||
// filename (only a-z, A-Z, 0-9, '_', '-' are kept; everything else becomes '_').
|
||||
func sanitiseVoice(voice string) string {
|
||||
return strings.Map(func(r rune) rune {
|
||||
if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '_' || r == '-' {
|
||||
return r
|
||||
}
|
||||
return '_'
|
||||
}, voice)
|
||||
}
|
||||
|
||||
// chapterPath computes the full file path for a chapter.
|
||||
//
|
||||
// vol-{volume}/{folderRange}/chapter-{number}.md
|
||||
//
|
||||
// Example: vol-0/1-50/chapter-1.md, vol-0/51-100/chapter-51.md
|
||||
func (w *Writer) chapterPath(slug string, ref scraper.ChapterRef) string {
|
||||
vol := ref.Volume // 0 == no volume grouping
|
||||
volDir := fmt.Sprintf("vol-%d", vol)
|
||||
|
||||
// Folder group: chapters 1-50 → "1-50", 51-100 → "51-100", …
|
||||
lo := ((ref.Number-1)/chaptersPerFolder)*chaptersPerFolder + 1
|
||||
hi := lo + chaptersPerFolder - 1
|
||||
rangeDir := fmt.Sprintf("%d-%d", lo, hi)
|
||||
|
||||
filename := fmt.Sprintf("chapter-%d.md", ref.Number)
|
||||
|
||||
return filepath.Join(w.bookDir(slug), volDir, rangeDir, filename)
|
||||
}
|
||||
Reference in New Issue
Block a user