fix(scraper): replace browserless with direct HTTP and fix presign 404 race
- Fix audio presign 404: MinIO upload is now synchronous before response is sent, eliminating the race where presign was called before the file landed in MinIO - Replace all Browserless usage with direct HTTP client across catalogue, metadata, ranking, and browse — novelfire.net pages are server-rendered and don't need a headless browser; direct is faster and more reliable - Harden handleBrowse with 3-attempt retry loop, proper backoff, and full browser-like headers to reduce 502s from novelfire.net bot detection - Remove Browserless env vars (BROWSERLESS_URL/TOKEN/STRATEGY) from main.go; add SCRAPER_TIMEOUT as a single timeout knob - Clean up now-dead rejectResourceTypes var and Browserless-specific WaitFor/ RejectResourceTypes/GotoOptions fields from scraper calls
This commit is contained in:
@@ -78,26 +78,16 @@ func run(log *slog.Logger) error {
|
||||
|
||||
cmd := strings.ToLower(args[0])
|
||||
|
||||
browserCfg := browser.Config{
|
||||
BaseURL: envOr("BROWSERLESS_URL", "http://localhost:3030"),
|
||||
Token: envOr("BROWSERLESS_TOKEN", ""),
|
||||
}
|
||||
browserCfg.MaxConcurrent = 5
|
||||
if s := os.Getenv("BROWSERLESS_MAX_CONCURRENT"); s != "" {
|
||||
// All scraping uses direct HTTP — novelfire.net pages are server-rendered
|
||||
// and do not require a headless browser. A direct HTTP client is faster,
|
||||
// more reliable, and has no Browserless dependency.
|
||||
directCfg := browser.Config{MaxConcurrent: 5}
|
||||
if s := os.Getenv("SCRAPER_TIMEOUT"); s != "" {
|
||||
if n, err := strconv.Atoi(s); err == nil && n > 0 {
|
||||
browserCfg.MaxConcurrent = n
|
||||
directCfg.Timeout = time.Duration(n) * time.Second
|
||||
}
|
||||
}
|
||||
if s := os.Getenv("BROWSERLESS_TIMEOUT"); s != "" {
|
||||
if n, err := strconv.Atoi(s); err == nil && n > 0 {
|
||||
browserCfg.Timeout = time.Duration(n) * time.Second
|
||||
}
|
||||
}
|
||||
|
||||
strategy := browser.Strategy(strings.ToLower(envOr("BROWSERLESS_STRATEGY", string(browser.StrategyDirect))))
|
||||
bc := newBrowserClient(strategy, browserCfg)
|
||||
// Chapter list and chapter text are server-rendered on novelfire.net — direct HTTP for both.
|
||||
directClient := browser.NewDirectHTTPClient(browserCfg)
|
||||
directClient := browser.NewDirectHTTPClient(directCfg)
|
||||
|
||||
// ── Storage backends ────────────────────────────────────────────────────
|
||||
minioCfg := storage.MinioConfig{
|
||||
@@ -125,7 +115,7 @@ func run(log *slog.Logger) error {
|
||||
return fmt.Errorf("storage init failed: %w", err)
|
||||
}
|
||||
|
||||
nf := novelfire.New(bc, log, directClient, directClient, store)
|
||||
nf := novelfire.New(directClient, log, directClient, directClient, store)
|
||||
|
||||
workers := 0
|
||||
if s := os.Getenv("SCRAPER_WORKERS"); s != "" {
|
||||
@@ -149,9 +139,8 @@ func run(log *slog.Logger) error {
|
||||
oCfg.SingleBookURL = args[2]
|
||||
}
|
||||
log.Info("starting one-shot scrape",
|
||||
"strategy", strategy,
|
||||
"strategy", "direct",
|
||||
"workers", workers,
|
||||
"max_concurrent", browserCfg.MaxConcurrent,
|
||||
"single_book", oCfg.SingleBookURL,
|
||||
"pocketbase_url", pbCfg.BaseURL,
|
||||
"pocketbase_email", pbCfg.AdminEmail,
|
||||
@@ -191,9 +180,8 @@ func run(log *slog.Logger) error {
|
||||
kokoroVoice := envOr("KOKORO_VOICE", "af_bella")
|
||||
log.Info("starting HTTP server",
|
||||
"addr", addr,
|
||||
"strategy", strategy,
|
||||
"strategy", "direct",
|
||||
"workers", workers,
|
||||
"max_concurrent", browserCfg.MaxConcurrent,
|
||||
"kokoro_url", kokoroURL,
|
||||
"kokoro_voice", kokoroVoice,
|
||||
"pocketbase_url", pbCfg.BaseURL,
|
||||
@@ -533,19 +521,6 @@ func downloadAndStoreCoverCLI(store storage.Store, log *slog.Logger, key, imageU
|
||||
log.Debug("save-browse: cover stored", "key", key, "bytes", len(data))
|
||||
}
|
||||
|
||||
func newBrowserClient(strategy browser.Strategy, cfg browser.Config) browser.BrowserClient {
|
||||
switch strategy {
|
||||
case browser.StrategyScrape:
|
||||
return browser.NewScrapeClient(cfg)
|
||||
case browser.StrategyCDP:
|
||||
return browser.NewCDPClient(cfg)
|
||||
case browser.StrategyDirect:
|
||||
return browser.NewDirectHTTPClient(cfg)
|
||||
default:
|
||||
return browser.NewContentClient(cfg)
|
||||
}
|
||||
}
|
||||
|
||||
func envOr(key, fallback string) string {
|
||||
if v := os.Getenv(key); v != "" {
|
||||
return v
|
||||
|
||||
Reference in New Issue
Block a user