fix(scraper): replace browserless with direct HTTP and fix presign 404 race

- Fix audio presign 404: MinIO upload is now synchronous before response is sent,
  eliminating the race where presign was called before the file landed in MinIO
- Replace all Browserless usage with direct HTTP client across catalogue, metadata,
  ranking, and browse — novelfire.net pages are server-rendered and don't need a
  headless browser; direct is faster and more reliable
- Harden handleBrowse with 3-attempt retry loop, proper backoff, and full
  browser-like headers to reduce 502s from novelfire.net bot detection
- Remove Browserless env vars (BROWSERLESS_URL/TOKEN/STRATEGY) from main.go;
  add SCRAPER_TIMEOUT as a single timeout knob
- Clean up now-dead rejectResourceTypes var and Browserless-specific WaitFor/
  RejectResourceTypes/GotoOptions fields from scraper calls
This commit is contained in:
Admin
2026-03-04 17:32:18 +05:00
parent 0402c408e4
commit cf0c0dfaaf
3 changed files with 83 additions and 121 deletions

View File

@@ -78,26 +78,16 @@ func run(log *slog.Logger) error {
cmd := strings.ToLower(args[0])
browserCfg := browser.Config{
BaseURL: envOr("BROWSERLESS_URL", "http://localhost:3030"),
Token: envOr("BROWSERLESS_TOKEN", ""),
}
browserCfg.MaxConcurrent = 5
if s := os.Getenv("BROWSERLESS_MAX_CONCURRENT"); s != "" {
// All scraping uses direct HTTP — novelfire.net pages are server-rendered
// and do not require a headless browser. A direct HTTP client is faster,
// more reliable, and has no Browserless dependency.
directCfg := browser.Config{MaxConcurrent: 5}
if s := os.Getenv("SCRAPER_TIMEOUT"); s != "" {
if n, err := strconv.Atoi(s); err == nil && n > 0 {
browserCfg.MaxConcurrent = n
directCfg.Timeout = time.Duration(n) * time.Second
}
}
if s := os.Getenv("BROWSERLESS_TIMEOUT"); s != "" {
if n, err := strconv.Atoi(s); err == nil && n > 0 {
browserCfg.Timeout = time.Duration(n) * time.Second
}
}
strategy := browser.Strategy(strings.ToLower(envOr("BROWSERLESS_STRATEGY", string(browser.StrategyDirect))))
bc := newBrowserClient(strategy, browserCfg)
// Chapter list and chapter text are server-rendered on novelfire.net — direct HTTP for both.
directClient := browser.NewDirectHTTPClient(browserCfg)
directClient := browser.NewDirectHTTPClient(directCfg)
// ── Storage backends ────────────────────────────────────────────────────
minioCfg := storage.MinioConfig{
@@ -125,7 +115,7 @@ func run(log *slog.Logger) error {
return fmt.Errorf("storage init failed: %w", err)
}
nf := novelfire.New(bc, log, directClient, directClient, store)
nf := novelfire.New(directClient, log, directClient, directClient, store)
workers := 0
if s := os.Getenv("SCRAPER_WORKERS"); s != "" {
@@ -149,9 +139,8 @@ func run(log *slog.Logger) error {
oCfg.SingleBookURL = args[2]
}
log.Info("starting one-shot scrape",
"strategy", strategy,
"strategy", "direct",
"workers", workers,
"max_concurrent", browserCfg.MaxConcurrent,
"single_book", oCfg.SingleBookURL,
"pocketbase_url", pbCfg.BaseURL,
"pocketbase_email", pbCfg.AdminEmail,
@@ -191,9 +180,8 @@ func run(log *slog.Logger) error {
kokoroVoice := envOr("KOKORO_VOICE", "af_bella")
log.Info("starting HTTP server",
"addr", addr,
"strategy", strategy,
"strategy", "direct",
"workers", workers,
"max_concurrent", browserCfg.MaxConcurrent,
"kokoro_url", kokoroURL,
"kokoro_voice", kokoroVoice,
"pocketbase_url", pbCfg.BaseURL,
@@ -533,19 +521,6 @@ func downloadAndStoreCoverCLI(store storage.Store, log *slog.Logger, key, imageU
log.Debug("save-browse: cover stored", "key", key, "bytes", len(data))
}
func newBrowserClient(strategy browser.Strategy, cfg browser.Config) browser.BrowserClient {
switch strategy {
case browser.StrategyScrape:
return browser.NewScrapeClient(cfg)
case browser.StrategyCDP:
return browser.NewCDPClient(cfg)
case browser.StrategyDirect:
return browser.NewDirectHTTPClient(cfg)
default:
return browser.NewContentClient(cfg)
}
}
func envOr(key, fallback string) string {
if v := os.Getenv(key); v != "" {
return v