libnovel/scraper/internal/browser/http.go

package browser

import (
	"compress/gzip"
	"context"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"os"
	"strings"
	"time"

	"github.com/andybalholm/brotli"
)

type httpClient struct {
	cfg  Config
	http *http.Client
	sem  chan struct{}
}

func NewDirectHTTPClient(cfg Config) BrowserClient {
	if cfg.Timeout == 0 {
		cfg.Timeout = 30 * time.Second
	}

	transport := http.DefaultTransport.(*http.Transport).Clone()

	// Wire in proxy from environment (HTTP_PROXY / HTTPS_PROXY / NO_PROXY).
	// This lets operators route traffic through a residential proxy by simply
	// setting HTTPS_PROXY=http://user:pass@proxy-host:port without any code
	// changes — the standard approach for bypassing datacenter IP blocks.
	if proxyURL := proxyFromEnv(); proxyURL != nil {
		transport.Proxy = http.ProxyURL(proxyURL)
	} else {
		transport.Proxy = http.ProxyFromEnvironment
	}

	return &httpClient{
		cfg: cfg,
		http: &http.Client{
			Timeout:   cfg.Timeout,
			Transport: transport,
		},
		sem: makeSem(cfg.MaxConcurrent),
	}
}

// proxyFromEnv returns an explicit proxy URL if SCRAPER_PROXY is set, otherwise
// nil (and http.ProxyFromEnvironment handles the standard HTTP_PROXY / HTTPS_PROXY).
func proxyFromEnv() *url.URL {
	raw := os.Getenv("SCRAPER_PROXY")
	if raw == "" {
		return nil
	}
	u, err := url.Parse(raw)
	if err != nil || u.Host == "" {
		return nil
	}
	return u
}

func (c *httpClient) Strategy() Strategy { return StrategyDirect }

func (c *httpClient) GetContent(ctx context.Context, req ContentRequest) (string, error) {
	if err := acquire(ctx, c.sem); err != nil {
		return "", fmt.Errorf("http: semaphore: %w", err)
	}
	defer release(c.sem)

	httpReq, err := http.NewRequestWithContext(ctx, http.MethodGet, req.URL, nil)
	if err != nil {
		return "", fmt.Errorf("http: build request: %w", err)
	}

	// Mimic a real Chrome browser request to reduce bot-detection likelihood.
	// These headers match what Chrome 124 sends for a top-level navigation.
	httpReq.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36")
	httpReq.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7")
	httpReq.Header.Set("Accept-Language", "en-US,en;q=0.9")
	httpReq.Header.Set("Accept-Encoding", "gzip, deflate, br")
	httpReq.Header.Set("Connection", "keep-alive")
	httpReq.Header.Set("Upgrade-Insecure-Requests", "1")
	httpReq.Header.Set("Sec-Fetch-Dest", "document")
	httpReq.Header.Set("Sec-Fetch-Mode", "navigate")
	httpReq.Header.Set("Sec-Fetch-Site", "none")
	httpReq.Header.Set("Sec-Fetch-User", "?1")
	httpReq.Header.Set("Cache-Control", "max-age=0")

	// Set Referer for subsequent page requests (anything that is not the root).
	if parsed, pErr := url.Parse(req.URL); pErr == nil && parsed.Path != "" && parsed.Path != "/" {
		httpReq.Header.Set("Referer", parsed.Scheme+"://"+parsed.Host+"/")
	}

	resp, err := c.http.Do(httpReq)
	if err != nil {
		return "", fmt.Errorf("http: do request: %w", err)
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		b, _ := io.ReadAll(resp.Body)
		return "", fmt.Errorf("http: unexpected status %d: %s", resp.StatusCode, b)
	}

	// Decompress gzip/br responses when the server honours Accept-Encoding.
	// net/http decompresses gzip automatically only when it sets the header
	// itself; since we set Accept-Encoding explicitly we must do it ourselves.
	body := resp.Body
	switch strings.ToLower(resp.Header.Get("Content-Encoding")) {
	case "gzip":
		gr, gzErr := gzip.NewReader(resp.Body)
		if gzErr != nil {
			return "", fmt.Errorf("http: gzip reader: %w", gzErr)
		}
		defer gr.Close()
		body = gr
	case "br":
		body = io.NopCloser(brotli.NewReader(resp.Body))
	}

	raw, err := io.ReadAll(body)
	if err != nil {
		return "", fmt.Errorf("http: read body: %w", err)
	}
	return string(raw), nil
}

func (c *httpClient) ScrapePage(_ context.Context, _ ScrapeRequest) (ScrapeResponse, error) {
	return ScrapeResponse{}, fmt.Errorf("http client does not support ScrapePage; use browserless")
}

func (c *httpClient) CDPSession(_ context.Context, _ string, _ CDPSessionFunc) error {
	return fmt.Errorf("http client does not support CDP; use browserless")
}