kamil/libnovel
1
0
Fork 0
Code Issues Pull Requests 3 Actions Packages Projects Releases 162 Wiki Activity

feat: add exponential backoff, some UI elements to see the resut of a scrape

Browse Source
This commit is contained in:
Admin
2026-02-26 18:51:32 +05:00
parent d68ea71239
commit e6e6f7dc4d
12 changed files with 462 additions and 153 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
scraper/cmd/scraper/main.go
View File

@@ -10,12 +10,14 @@
//
// Environment variables:
//
// BROWSERLESS_URL Browserless base URL (default: http://localhost:3000)
// BROWSERLESS_TOKEN Browserless API token (default: "")
// BROWSERLESS_STRATEGY content | scrape | cdp (default: content)
// SCRAPER_WORKERS Chapter goroutine count (default: NumCPU)
// SCRAPER_STATIC_ROOT Output directory (default: ./static/books)
// SCRAPER_HTTP_ADDR HTTP listen address (default: :8080)
// BROWSERLESS_URL Browserless base URL (default: http://localhost:3000)
// BROWSERLESS_TOKEN Browserless API token (default: "")
// BROWSERLESS_STRATEGY content | scrape | cdp (default: content)
// BROWSERLESS_MAX_CONCURRENT Max simultaneous browser sessions (default: 5)
// SCRAPER_WORKERS Chapter goroutine count (default: NumCPU)
// SCRAPER_STATIC_ROOT Output directory (default: ./static/books)
// SCRAPER_HTTP_ADDR HTTP listen address (default: :8080)
// LOG_LEVEL debug | info | warn | error (default: info)
package main
import (
@@ -36,8 +38,14 @@ import (
)
func main() {
logLevel := slog.LevelInfo
if v := os.Getenv("LOG_LEVEL"); v != "" {
if err := logLevel.UnmarshalText([]byte(v)); err != nil {
fmt.Fprintf(os.Stderr, "invalid LOG_LEVEL %q, using info\n", v)
}
}
log := slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{
Level: slog.LevelInfo,
Level: logLevel,
}))
if err := run(log); err != nil {
@@ -59,6 +67,12 @@ func run(log *slog.Logger) error {
BaseURL: envOr("BROWSERLESS_URL", "http://localhost:3000"),
Token: envOr("BROWSERLESS_TOKEN", ""),
}
browserCfg.MaxConcurrent = 5
if s := os.Getenv("BROWSERLESS_MAX_CONCURRENT"); s != "" {
if n, err := strconv.Atoi(s); err == nil && n > 0 {
browserCfg.MaxConcurrent = n
}
}
strategy := browser.Strategy(strings.ToLower(envOr("BROWSERLESS_STRATEGY", string(browser.StrategyContent))))
bc := newBrowserClient(strategy, browserCfg)
@@ -93,6 +107,7 @@ func run(log *slog.Logger) error {
log.Info("starting one-shot scrape",
"strategy", strategy,
"workers", workers,
"max_concurrent", browserCfg.MaxConcurrent,
"static_root", oCfg.StaticRoot,
"single_book", oCfg.SingleBookURL,
)
@@ -105,6 +120,7 @@ func run(log *slog.Logger) error {
"addr", addr,
"strategy", strategy,
"workers", workers,
"max_concurrent", browserCfg.MaxConcurrent,
)
srv := server.New(addr, oCfg, nf, log)
return srv.ListenAndServe(ctx)
@@ -140,11 +156,13 @@ Commands:
serve Start HTTP server (POST /scrape, POST /scrape/book)
Environment variables:
BROWSERLESS_URL Browserless base URL (default: http://localhost:3000)
BROWSERLESS_TOKEN API token (default: "")
BROWSERLESS_STRATEGY content | scrape | cdp (default: content)
SCRAPER_WORKERS Chapter goroutines (default: NumCPU = %d)
SCRAPER_STATIC_ROOT Output directory (default: ./static/books)
SCRAPER_HTTP_ADDR HTTP listen address (default: :8080)
BROWSERLESS_URL Browserless base URL (default: http://localhost:3000)
BROWSERLESS_TOKEN API token (default: "")
BROWSERLESS_STRATEGY content | scrape | cdp (default: content)
BROWSERLESS_MAX_CONCURRENT Max simultaneous sessions (default: 5)
SCRAPER_WORKERS Chapter goroutines (default: NumCPU = %d)
SCRAPER_STATIC_ROOT Output directory (default: ./static/books)
SCRAPER_HTTP_ADDR HTTP listen address (default: :8080)
LOG_LEVEL debug|info|warn|error (default: info)
`, runtime.NumCPU())
}