libnovel/scraper/cmd/scraper/main.go

// Command scraper is the entrypoint for the libnovel scraper service.
//
// Usage (CLI one-shot):
//
//	scraper run [--url <book-url>]
//
// Usage (HTTP server):
//
//	scraper serve
//
// Environment variables:
//
//	SCRAPER_WORKERS             Chapter goroutine count        (default: NumCPU)
//	SCRAPER_HTTP_ADDR           HTTP listen address            (default: :8080)
//	KOKORO_URL                  Kokoro-FastAPI base URL        (default: "")
//	KOKORO_VOICE                Default TTS voice              (default: af_bella)
//	POCKETBASE_URL              PocketBase API base URL        (default: http://localhost:8090)
//	POCKETBASE_ADMIN_EMAIL      PocketBase admin email         (default: admin@libnovel.local)
//	POCKETBASE_ADMIN_PASSWORD   PocketBase admin password      (default: changeme123)
//	MINIO_ENDPOINT              MinIO endpoint host:port       (default: localhost:9000)
//	MINIO_ACCESS_KEY            MinIO access key               (default: admin)
//	MINIO_SECRET_KEY            MinIO secret key               (default: changeme123)
//	MINIO_USE_SSL               Use TLS for MinIO              (default: false)
//	MINIO_BUCKET_CHAPTERS       Chapter objects bucket         (default: libnovel-chapters)
//	MINIO_BUCKET_AUDIO          Audio objects bucket           (default: libnovel-audio)
//	LOG_LEVEL                   debug | info | warn | error    (default: info)
package main

import (
	"context"
	"fmt"
	"log/slog"
	"os"
	"os/exec"
	"os/signal"
	"runtime"
	"strconv"
	"strings"
	"syscall"
	"time"

	"github.com/libnovel/scraper/internal/browser"
	"github.com/libnovel/scraper/internal/novelfire"
	"github.com/libnovel/scraper/internal/orchestrator"
	"github.com/libnovel/scraper/internal/scraper/htmlutil"
	"github.com/libnovel/scraper/internal/server"
	"github.com/libnovel/scraper/internal/storage"
)

func main() {
	logLevel := slog.LevelInfo
	if v := os.Getenv("LOG_LEVEL"); v != "" {
		if err := logLevel.UnmarshalText([]byte(v)); err != nil {
			fmt.Fprintf(os.Stderr, "invalid LOG_LEVEL %q, using info\n", v)
		}
	}
	log := slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{
		Level: logLevel,
	}))

	if err := run(log); err != nil {
		log.Error("fatal", "err", err)
		os.Exit(1)
	}
}

func run(log *slog.Logger) error {
	args := os.Args[1:]
	if len(args) == 0 {
		printUsage()
		return nil
	}

	cmd := strings.ToLower(args[0])

	// All scraping uses direct HTTP — novelfire.net pages are server-rendered
	// and do not require a headless browser. A direct HTTP client is faster,
	// more reliable, and has no Browserless dependency.
	directCfg := browser.Config{MaxConcurrent: 5}
	if s := os.Getenv("SCRAPER_TIMEOUT"); s != "" {
		if n, err := strconv.Atoi(s); err == nil && n > 0 {
			directCfg.Timeout = time.Duration(n) * time.Second
		}
	}
	directClient := browser.NewDirectHTTPClient(directCfg)

	// ── Storage backends ────────────────────────────────────────────────────
	minioCfg := storage.MinioConfig{
		Endpoint:       envOr("MINIO_ENDPOINT", "localhost:9000"),
		PublicEndpoint: envOr("MINIO_PUBLIC_ENDPOINT", ""),
		AccessKey:      envOr("MINIO_ACCESS_KEY", "admin"),
		SecretKey:      envOr("MINIO_SECRET_KEY", "changeme123"),
		UseSSL:         strings.ToLower(os.Getenv("MINIO_USE_SSL")) == "true",
		PublicUseSSL:   strings.ToLower(os.Getenv("MINIO_PUBLIC_USE_SSL")) != "false",
		BucketChapters: envOr("MINIO_BUCKET_CHAPTERS", "libnovel-chapters"),
		BucketAudio:    envOr("MINIO_BUCKET_AUDIO", "libnovel-audio"),
		BucketBrowse:   envOr("MINIO_BUCKET_BROWSE", "libnovel-browse"),
	}
	pbCfg := storage.PocketBaseConfig{
		BaseURL:       envOr("POCKETBASE_URL", "http://localhost:8090"),
		AdminEmail:    envOr("POCKETBASE_ADMIN_EMAIL", "admin@libnovel.local"),
		AdminPassword: envOr("POCKETBASE_ADMIN_PASSWORD", "changeme123"),
	}

	ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
	defer stop()

	store, err := storage.NewHybridStore(ctx, pbCfg, minioCfg, log)
	if err != nil {
		return fmt.Errorf("storage init failed: %w", err)
	}

	nf := novelfire.New(directClient, log, directClient, directClient, store)

	workers := 0
	if s := os.Getenv("SCRAPER_WORKERS"); s != "" {
		n, err := strconv.Atoi(s)
		if err == nil && n > 0 {
			workers = n
		}
	}
	if workers == 0 {
		workers = runtime.NumCPU()
	}

	oCfg := orchestrator.Config{
		Workers: workers,
	}

	switch cmd {
	case "run":
		// Optional --url flag.
		if len(args) >= 3 && args[1] == "--url" {
			oCfg.SingleBookURL = args[2]
		}
		log.Info("starting one-shot scrape",
			"strategy", "direct",
			"workers", workers,
			"single_book", oCfg.SingleBookURL,
			"pocketbase_url", pbCfg.BaseURL,
			"pocketbase_email", pbCfg.AdminEmail,
		)
		o := orchestrator.New(oCfg, nf, log, store)
		return o.Run(ctx)

	case "refresh":
		// refresh <slug> - re-scrape a book from its saved source_url
		if len(args) < 2 {
			return fmt.Errorf("refresh command requires a book slug argument")
		}
		slug := args[1]
		meta, ok, err := store.ReadMetadata(ctx, slug)
		if err != nil {
			return fmt.Errorf("failed to read metadata for %s: %w", slug, err)
		}
		if !ok {
			return fmt.Errorf("book %q not found in store", slug)
		}
		if meta.SourceURL == "" {
			return fmt.Errorf("book %q has no source_url in metadata", slug)
		}
		oCfg.SingleBookURL = meta.SourceURL
		log.Info("refreshing book from source_url",
			"slug", slug,
			"source_url", meta.SourceURL,
			"pocketbase_url", pbCfg.BaseURL,
			"pocketbase_email", pbCfg.AdminEmail,
		)
		o := orchestrator.New(oCfg, nf, log, store)
		return o.Run(ctx)

	case "serve":
		addr := envOr("SCRAPER_HTTP_ADDR", ":8080")
		kokoroURL := envOr("KOKORO_URL", "https://kokoro.kalekber.cc")
		kokoroVoice := envOr("KOKORO_VOICE", "af_bella")
		log.Info("starting HTTP server",
			"addr", addr,
			"strategy", "direct",
			"workers", workers,
			"kokoro_url", kokoroURL,
			"kokoro_voice", kokoroVoice,
			"pocketbase_url", pbCfg.BaseURL,
			"pocketbase_email", pbCfg.AdminEmail,
		)
		srv := server.New(addr, oCfg, nf, log, store, kokoroURL, kokoroVoice)
		return srv.ListenAndServe(ctx)

	case "save-browse":
		return runSaveBrowse(ctx, args[1:], store, log)

	default:
		return fmt.Errorf("unknown command %q; use 'run', 'refresh', 'serve', or 'save-browse'", cmd)
	}
}

// runSaveBrowse implements the `save-browse` subcommand.
// It iterates over browse pages on novelfire.net, captures each using
// SingleFile CLI (connected to the existing Browserless instance), and
// stores the resulting self-contained HTML in the MinIO browse bucket.
// After storing each page it parses the HTML, upserts ranking records in
// PocketBase, and fires background goroutines to download cover images.
//
// Flags (all optional):
//
//	--genre   <value>   genre slug  (default: all)
//	--sort    <value>   sort order  (default: popular)
//	--status  <value>   status      (default: all)
//	--type    <value>   novel type  (default: all-novel)
//	--max-pages <n>     max pages   (default: 5)
func runSaveBrowse(ctx context.Context, args []string, store storage.Store, log *slog.Logger) error {
	// Parse flags manually to avoid importing flag package.
	genre := "all"
	sortBy := "popular"
	status := "all"
	novelType := "all-novel"
	maxPages := 5

	for i := 0; i < len(args); i++ {
		switch args[i] {
		case "--genre":
			if i+1 < len(args) {
				genre = args[i+1]
				i++
			}
		case "--sort":
			if i+1 < len(args) {
				sortBy = args[i+1]
				i++
			}
		case "--status":
			if i+1 < len(args) {
				status = args[i+1]
				i++
			}
		case "--type":
			if i+1 < len(args) {
				novelType = args[i+1]
				i++
			}
		case "--max-pages":
			if i+1 < len(args) {
				if n, err := strconv.Atoi(args[i+1]); err == nil && n > 0 {
					maxPages = n
				}
				i++
			}
		}
	}

	singleFilePath := envOr("SINGLEFILE_PATH", "single-file")
	browserlessURL := envOr("BROWSERLESS_URL", "http://localhost:3030")
	// SingleFile expects a WebSocket CDP endpoint.
	// Browserless exposes /chromium at the WS root.
	wsEndpoint := strings.Replace(browserlessURL, "http://", "ws://", 1)
	wsEndpoint = strings.Replace(wsEndpoint, "https://", "wss://", 1)

	log.Info("save-browse: starting",
		"genre", genre, "sort", sortBy, "status", status,
		"type", novelType, "max_pages", maxPages,
		"singlefile", singleFilePath,
		"browserless_ws", wsEndpoint,
	)

	tmpDir, err := os.MkdirTemp("", "libnovel-browse-*")
	if err != nil {
		return fmt.Errorf("save-browse: create temp dir: %w", err)
	}
	defer os.RemoveAll(tmpDir)

	const novelFireBase = "https://novelfire.net"
	const novelFireDomain = "novelfire.net"

	for page := 1; page <= maxPages; page++ {
		select {
		case <-ctx.Done():
			return ctx.Err()
		default:
		}

		pageURL := fmt.Sprintf("%s/genre-%s/sort-%s/status-%s/%s?page=%d",
			novelFireBase, genre, sortBy, status, novelType, page)

		// Use the new domain-based key layout: {domain}/html/page-{n}.html
		key := store.BrowseHTMLKey(novelFireDomain, page)

		outFile := fmt.Sprintf("%s/page-%d.html", tmpDir, page)

		log.Info("save-browse: capturing page", "page", page, "url", pageURL)

		//nolint:gosec // singleFilePath and pageURL are config/URL values, not user input.
		cmd := exec.CommandContext(ctx, singleFilePath,
			pageURL,
			"--browser-server="+wsEndpoint,
			"--output="+outFile,
		)
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr

		if runErr := cmd.Run(); runErr != nil {
			log.Warn("save-browse: SingleFile failed, skipping page",
				"page", page, "err", runErr)
			continue
		}

		htmlBytes, readErr := os.ReadFile(outFile)
		if readErr != nil {
			log.Warn("save-browse: failed to read output file",
				"page", page, "file", outFile, "err", readErr)
			continue
		}

		if putErr := store.SaveBrowsePage(ctx, key, string(htmlBytes)); putErr != nil {
			log.Warn("save-browse: failed to store snapshot in MinIO",
				"page", page, "key", key, "err", putErr)
			continue
		}

		log.Info("save-browse: snapshot stored", "page", page, "key", key,
			"bytes", len(htmlBytes))

		// Parse the stored HTML and populate the ranking collection.
		novels := parseSaveBrowseListings(htmlBytes, novelFireBase)
		for i, novel := range novels {
			rank := i + 1
			coverKey := store.BrowseCoverKey(novelFireDomain, novel.slug)

			item := storage.RankingItem{
				Rank:      rank,
				Slug:      novel.slug,
				Title:     novel.title,
				Cover:     coverKey,
				SourceURL: novel.url,
			}
			if werr := store.WriteRankingItem(ctx, item); werr != nil {
				log.Warn("save-browse: WriteRankingItem failed",
					"slug", novel.slug, "err", werr)
			}

			// Download cover image in the background (best-effort).
			if novel.coverURL != "" {
				go storage.DownloadAndStoreCover(store, log, coverKey, novel.coverURL)
			}
		}
		if len(novels) > 0 {
			log.Info("save-browse: ranking populated", "page", page, "count", len(novels))
		}
	}

	log.Info("save-browse: done")
	return nil
}

// novelListingCLI is a minimal novel listing used within the CLI command.
type novelListingCLI struct {
	slug     string
	title    string
	url      string
	coverURL string
}

// parseSaveBrowseListings extracts novel listings from raw HTML bytes.
// It reuses the same parsing logic as the server's parseBrowsePage but
// operates on []byte to avoid importing the server package.
func parseSaveBrowseListings(htmlBytes []byte, novelFireBase string) []novelListingCLI {
	type listing = novelListingCLI

	// Minimal tokeniser-based walk to find <li class="novel-item"> blocks.
	// We use the golang.org/x/net/html parser via a local import.
	// Because main.go already imports golang.org/x/net/html indirectly through
	// the server package build, we do a simple line-scan here instead to keep
	// the dependency surface small.
	//
	// Strategy: scan for href="/book/{slug}", img data-src/src, h4.novel-title text.
	var novels []listing

	lines := strings.Split(string(htmlBytes), "\n")
	var cur listing
	inNovelItem := false

	for _, line := range lines {
		trimmed := strings.TrimSpace(line)

		// Detect start of a novel-item list element.
		if strings.Contains(trimmed, `class="novel-item"`) || strings.Contains(trimmed, "novel-item") && strings.HasPrefix(trimmed, "<li") {
			inNovelItem = true
			cur = listing{}
		}

		if !inNovelItem {
			continue
		}

		// Detect end of list element.
		if trimmed == "</li>" && cur.slug != "" {
			novels = append(novels, cur)
			inNovelItem = false
			cur = listing{}
			continue
		}

		// Extract slug from href="/book/{slug}".
		if cur.slug == "" {
			if idx := strings.Index(trimmed, `href="/book/`); idx >= 0 {
				rest := trimmed[idx+len(`href="/book/`):]
				if end := strings.IndexAny(rest, `"/ `); end > 0 {
					cur.slug = rest[:end]
					cur.url = novelFireBase + "/book/" + cur.slug
				} else if end := strings.Index(rest, `"`); end > 0 {
					cur.slug = strings.TrimSuffix(rest[:end], "/")
					cur.url = novelFireBase + "/book/" + cur.slug
				}
			}
		}

		// Extract cover URL from data-src or src on img tags.
		if cur.coverURL == "" && strings.Contains(trimmed, "<img") {
			if src := extractAttr(trimmed, "data-src"); src != "" {
				cur.coverURL = htmlutil.ResolveURL(novelFireBase, src)
			} else if src := extractAttr(trimmed, "src"); src != "" && !strings.Contains(src, "data:") {
				cur.coverURL = htmlutil.ResolveURL(novelFireBase, src)
			}
		}

		// Extract title from novel-title element.
		if cur.title == "" && strings.Contains(trimmed, "novel-title") {
			// Try to grab inner text: <h4 class="novel-title">Title Here</h4>
			if start := strings.Index(trimmed, ">"); start >= 0 {
				rest := trimmed[start+1:]
				if end := strings.Index(rest, "<"); end > 0 {
					title := strings.TrimSpace(rest[:end])
					if title != "" {
						cur.title = title
					}
				}
			}
		}
	}

	// Flush any open item that wasn't closed by </li> (e.g. last item in file).
	if inNovelItem && cur.slug != "" {
		novels = append(novels, cur)
	}

	return novels
}

// extractAttr extracts an HTML attribute value from a raw tag string.
// e.g. extractAttr(`<img data-src="foo.jpg">`, "data-src") → "foo.jpg"
func extractAttr(tag, attr string) string {
	needle := attr + `="`
	idx := strings.Index(tag, needle)
	if idx < 0 {
		return ""
	}
	rest := tag[idx+len(needle):]
	end := strings.Index(rest, `"`)
	if end < 0 {
		return ""
	}
	return rest[:end]
}

func envOr(key, fallback string) string {
	if v := os.Getenv(key); v != "" {
		return v
	}
	return fallback
}

func printUsage() {
	fmt.Fprintf(os.Stderr, `libnovel scraper

Commands:
  run [--url <book-url>]   One-shot: scrape full catalogue, or a single book
  refresh <slug>            Re-scrape a book from its saved source_url
  serve                    Start HTTP server (POST /scrape, POST /scrape/book)
  save-browse              Capture browse pages via SingleFile → MinIO
    --genre   <slug>         genre filter  (default: all)
    --sort    <value>        sort order    (default: popular)
    --status  <value>        status filter (default: all)
    --type    <value>        novel type    (default: all-novel)
    --max-pages <n>          pages to capture (default: 5)

Environment variables:
  SCRAPER_WORKERS             Chapter goroutines        (default: NumCPU = %d)
  SCRAPER_HTTP_ADDR           HTTP listen address       (default: :8080)
  SCRAPER_TIMEOUT             HTTP request timeout sec  (default: 90)
  KOKORO_URL                  Kokoro-FastAPI base URL   (default: "", TTS disabled)
  KOKORO_VOICE                Default TTS voice         (default: af_bella)
  POCKETBASE_URL              PocketBase base URL       (default: http://localhost:8090)
  POCKETBASE_ADMIN_EMAIL      PocketBase admin email    (default: admin@libnovel.local)
  POCKETBASE_ADMIN_PASSWORD   PocketBase admin password (default: changeme123)
  MINIO_ENDPOINT              MinIO endpoint host:port  (default: localhost:9000)
  MINIO_ACCESS_KEY            MinIO access key          (default: admin)
  MINIO_SECRET_KEY            MinIO secret key          (default: changeme123)
  MINIO_USE_SSL               MinIO TLS                 (default: false)
  MINIO_BUCKET_CHAPTERS       Chapter objects bucket    (default: libnovel-chapters)
  MINIO_BUCKET_AUDIO          Audio objects bucket      (default: libnovel-audio)
  MINIO_BUCKET_BROWSE         Browse snapshots bucket   (default: libnovel-browse)
  BROWSERLESS_URL             Browserless WS endpoint   (default: http://localhost:3030)
  SINGLEFILE_PATH             Path to single-file CLI   (default: single-file)
  LOG_LEVEL                   debug|info|warn|error     (default: info)
`, runtime.NumCPU())
}