diff --git a/scraper/cmd/scraper/main.go b/scraper/cmd/scraper/main.go index 090e22f..d4d8e36 100644 --- a/scraper/cmd/scraper/main.go +++ b/scraper/cmd/scraper/main.go @@ -12,6 +12,10 @@ // // SCRAPER_WORKERS Chapter goroutine count (default: NumCPU) // SCRAPER_HTTP_ADDR HTTP listen address (default: :8080) +// SCRAPER_PROXY Outbound proxy for all scraper requests, e.g. +// http://user:pass@proxy-host:3128 — use a +// residential proxy to bypass datacenter IP blocks. +// Falls back to HTTP_PROXY / HTTPS_PROXY if unset. // KOKORO_URL Kokoro-FastAPI base URL (default: "") // KOKORO_VOICE Default TTS voice (default: af_bella) // POCKETBASE_URL PocketBase API base URL (default: http://localhost:8090) @@ -493,6 +497,7 @@ Environment variables: SCRAPER_WORKERS Chapter goroutines (default: NumCPU = %d) SCRAPER_HTTP_ADDR HTTP listen address (default: :8080) SCRAPER_TIMEOUT HTTP request timeout sec (default: 90) + SCRAPER_PROXY Outbound proxy URL (default: "", falls back to HTTP_PROXY/HTTPS_PROXY) KOKORO_URL Kokoro-FastAPI base URL (default: "", TTS disabled) KOKORO_VOICE Default TTS voice (default: af_bella) POCKETBASE_URL PocketBase base URL (default: http://localhost:8090) diff --git a/scraper/internal/browser/http.go b/scraper/internal/browser/http.go index 5b9c374..df169da 100644 --- a/scraper/internal/browser/http.go +++ b/scraper/internal/browser/http.go @@ -1,10 +1,14 @@ package browser import ( + "compress/gzip" "context" "fmt" "io" "net/http" + "net/url" + "os" + "strings" "time" ) @@ -18,11 +22,41 @@ func NewDirectHTTPClient(cfg Config) BrowserClient { if cfg.Timeout == 0 { cfg.Timeout = 30 * time.Second } - return &httpClient{ - cfg: cfg, - http: &http.Client{Timeout: cfg.Timeout}, - sem: makeSem(cfg.MaxConcurrent), + + transport := http.DefaultTransport.(*http.Transport).Clone() + + // Wire in proxy from environment (HTTP_PROXY / HTTPS_PROXY / NO_PROXY). + // This lets operators route traffic through a residential proxy by simply + // setting HTTPS_PROXY=http://user:pass@proxy-host:port without any code + // changes — the standard approach for bypassing datacenter IP blocks. + if proxyURL := proxyFromEnv(); proxyURL != nil { + transport.Proxy = http.ProxyURL(proxyURL) + } else { + transport.Proxy = http.ProxyFromEnvironment } + + return &httpClient{ + cfg: cfg, + http: &http.Client{ + Timeout: cfg.Timeout, + Transport: transport, + }, + sem: makeSem(cfg.MaxConcurrent), + } +} + +// proxyFromEnv returns an explicit proxy URL if SCRAPER_PROXY is set, otherwise +// nil (and http.ProxyFromEnvironment handles the standard HTTP_PROXY / HTTPS_PROXY). +func proxyFromEnv() *url.URL { + raw := os.Getenv("SCRAPER_PROXY") + if raw == "" { + return nil + } + u, err := url.Parse(raw) + if err != nil || u.Host == "" { + return nil + } + return u } func (c *httpClient) Strategy() Strategy { return StrategyDirect } @@ -37,9 +71,25 @@ func (c *httpClient) GetContent(ctx context.Context, req ContentRequest) (string if err != nil { return "", fmt.Errorf("http: build request: %w", err) } - httpReq.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36") - httpReq.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8") - httpReq.Header.Set("Accept-Language", "en-US,en;q=0.5") + + // Mimic a real Chrome browser request to reduce bot-detection likelihood. + // These headers match what Chrome 124 sends for a top-level navigation. + httpReq.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36") + httpReq.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7") + httpReq.Header.Set("Accept-Language", "en-US,en;q=0.9") + httpReq.Header.Set("Accept-Encoding", "gzip, deflate, br") + httpReq.Header.Set("Connection", "keep-alive") + httpReq.Header.Set("Upgrade-Insecure-Requests", "1") + httpReq.Header.Set("Sec-Fetch-Dest", "document") + httpReq.Header.Set("Sec-Fetch-Mode", "navigate") + httpReq.Header.Set("Sec-Fetch-Site", "none") + httpReq.Header.Set("Sec-Fetch-User", "?1") + httpReq.Header.Set("Cache-Control", "max-age=0") + + // Set Referer for subsequent page requests (anything that is not the root). + if parsed, pErr := url.Parse(req.URL); pErr == nil && parsed.Path != "" && parsed.Path != "/" { + httpReq.Header.Set("Referer", parsed.Scheme+"://"+parsed.Host+"/") + } resp, err := c.http.Do(httpReq) if err != nil { @@ -52,7 +102,22 @@ func (c *httpClient) GetContent(ctx context.Context, req ContentRequest) (string return "", fmt.Errorf("http: unexpected status %d: %s", resp.StatusCode, b) } - raw, err := io.ReadAll(resp.Body) + // Decompress gzip/br responses when the server honours Accept-Encoding. + // net/http decompresses gzip automatically only when it sets the header + // itself; since we set Accept-Encoding explicitly we must do it ourselves. + body := resp.Body + if strings.EqualFold(resp.Header.Get("Content-Encoding"), "gzip") { + gr, gzErr := gzip.NewReader(resp.Body) + if gzErr != nil { + return "", fmt.Errorf("http: gzip reader: %w", gzErr) + } + defer gr.Close() + body = gr + } + // br (Brotli) decompression requires an external package; skip for now — + // the server will fall back to gzip or plain text for unknown encodings. + + raw, err := io.ReadAll(body) if err != nil { return "", fmt.Errorf("http: read body: %w", err) }