package browser import ( "compress/gzip" "context" "fmt" "io" "net/http" "net/url" "os" "strings" "time" "github.com/andybalholm/brotli" ) type httpClient struct { cfg Config http *http.Client sem chan struct{} } func NewDirectHTTPClient(cfg Config) BrowserClient { if cfg.Timeout == 0 { cfg.Timeout = 30 * time.Second } transport := http.DefaultTransport.(*http.Transport).Clone() // Wire in proxy from environment (HTTP_PROXY / HTTPS_PROXY / NO_PROXY). // This lets operators route traffic through a residential proxy by simply // setting HTTPS_PROXY=http://user:pass@proxy-host:port without any code // changes — the standard approach for bypassing datacenter IP blocks. if proxyURL := proxyFromEnv(); proxyURL != nil { transport.Proxy = http.ProxyURL(proxyURL) } else { transport.Proxy = http.ProxyFromEnvironment } return &httpClient{ cfg: cfg, http: &http.Client{ Timeout: cfg.Timeout, Transport: transport, }, sem: makeSem(cfg.MaxConcurrent), } } // proxyFromEnv returns an explicit proxy URL if SCRAPER_PROXY is set, otherwise // nil (and http.ProxyFromEnvironment handles the standard HTTP_PROXY / HTTPS_PROXY). func proxyFromEnv() *url.URL { raw := os.Getenv("SCRAPER_PROXY") if raw == "" { return nil } u, err := url.Parse(raw) if err != nil || u.Host == "" { return nil } return u } func (c *httpClient) Strategy() Strategy { return StrategyDirect } func (c *httpClient) GetContent(ctx context.Context, req ContentRequest) (string, error) { if err := acquire(ctx, c.sem); err != nil { return "", fmt.Errorf("http: semaphore: %w", err) } defer release(c.sem) httpReq, err := http.NewRequestWithContext(ctx, http.MethodGet, req.URL, nil) if err != nil { return "", fmt.Errorf("http: build request: %w", err) } // Mimic a real Chrome browser request to reduce bot-detection likelihood. // These headers match what Chrome 124 sends for a top-level navigation. httpReq.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36") httpReq.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7") httpReq.Header.Set("Accept-Language", "en-US,en;q=0.9") httpReq.Header.Set("Accept-Encoding", "gzip, deflate, br") httpReq.Header.Set("Connection", "keep-alive") httpReq.Header.Set("Upgrade-Insecure-Requests", "1") httpReq.Header.Set("Sec-Fetch-Dest", "document") httpReq.Header.Set("Sec-Fetch-Mode", "navigate") httpReq.Header.Set("Sec-Fetch-Site", "none") httpReq.Header.Set("Sec-Fetch-User", "?1") httpReq.Header.Set("Cache-Control", "max-age=0") // Set Referer for subsequent page requests (anything that is not the root). if parsed, pErr := url.Parse(req.URL); pErr == nil && parsed.Path != "" && parsed.Path != "/" { httpReq.Header.Set("Referer", parsed.Scheme+"://"+parsed.Host+"/") } resp, err := c.http.Do(httpReq) if err != nil { return "", fmt.Errorf("http: do request: %w", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { b, _ := io.ReadAll(resp.Body) return "", fmt.Errorf("http: unexpected status %d: %s", resp.StatusCode, b) } // Decompress gzip/br responses when the server honours Accept-Encoding. // net/http decompresses gzip automatically only when it sets the header // itself; since we set Accept-Encoding explicitly we must do it ourselves. body := resp.Body switch strings.ToLower(resp.Header.Get("Content-Encoding")) { case "gzip": gr, gzErr := gzip.NewReader(resp.Body) if gzErr != nil { return "", fmt.Errorf("http: gzip reader: %w", gzErr) } defer gr.Close() body = gr case "br": body = io.NopCloser(brotli.NewReader(resp.Body)) } raw, err := io.ReadAll(body) if err != nil { return "", fmt.Errorf("http: read body: %w", err) } return string(raw), nil } func (c *httpClient) ScrapePage(_ context.Context, _ ScrapeRequest) (ScrapeResponse, error) { return ScrapeResponse{}, fmt.Errorf("http client does not support ScrapePage; use browserless") } func (c *httpClient) CDPSession(_ context.Context, _ string, _ CDPSessionFunc) error { return fmt.Errorf("http client does not support CDP; use browserless") }