feat: initial commit

scraper/internal/browser/content_scrape.go

@@ -0,0 +1,146 @@
+package browser
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"time"
+)
+
+// Config holds the connection parameters for a Browserless instance.
+type Config struct {
+	// BaseURL is the HTTP base URL, e.g. "http://localhost:3000".
+	BaseURL string
+	// Token is the optional API token (BROWSERLESS_TOKEN env var).
+	Token string
+	// Timeout is the per-request HTTP timeout; defaults to 60 s.
+	Timeout time.Duration
+}
+
+// contentClient implements BrowserClient using the /content endpoint.
+type contentClient struct {
+	cfg  Config
+	http *http.Client
+}
+
+// NewContentClient returns a BrowserClient that uses POST /content.
+func NewContentClient(cfg Config) BrowserClient {
+	if cfg.Timeout == 0 {
+		cfg.Timeout = 60 * time.Second
+	}
+	return &contentClient{
+		cfg:  cfg,
+		http: &http.Client{Timeout: cfg.Timeout},
+	}
+}
+
+func (c *contentClient) Strategy() Strategy { return StrategyContent }
+
+func (c *contentClient) GetContent(ctx context.Context, req ContentRequest) (string, error) {
+	body, err := json.Marshal(req)
+	if err != nil {
+		return "", fmt.Errorf("content: marshal request: %w", err)
+	}
+
+	url := c.cfg.BaseURL + "/content"
+	if c.cfg.Token != "" {
+		url += "?token=" + c.cfg.Token
+	}
+
+	httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body))
+	if err != nil {
+		return "", fmt.Errorf("content: build request: %w", err)
+	}
+	httpReq.Header.Set("Content-Type", "application/json")
+
+	resp, err := c.http.Do(httpReq)
+	if err != nil {
+		return "", fmt.Errorf("content: do request: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		b, _ := io.ReadAll(resp.Body)
+		return "", fmt.Errorf("content: unexpected status %d: %s", resp.StatusCode, b)
+	}
+
+	raw, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return "", fmt.Errorf("content: read body: %w", err)
+	}
+	return string(raw), nil
+}
+
+func (c *contentClient) ScrapePage(_ context.Context, _ ScrapeRequest) (ScrapeResponse, error) {
+	return ScrapeResponse{}, fmt.Errorf("content client does not support /scrape; use NewScrapeClient")
+}
+
+func (c *contentClient) CDPSession(_ context.Context, _ string, _ CDPSessionFunc) error {
+	return fmt.Errorf("content client does not support CDP; use NewCDPClient")
+}
+
+// ─── /scrape client ───────────────────────────────────────────────────────────
+
+type scrapeClient struct {
+	cfg  Config
+	http *http.Client
+}
+
+// NewScrapeClient returns a BrowserClient that uses POST /scrape.
+func NewScrapeClient(cfg Config) BrowserClient {
+	if cfg.Timeout == 0 {
+		cfg.Timeout = 60 * time.Second
+	}
+	return &scrapeClient{
+		cfg:  cfg,
+		http: &http.Client{Timeout: cfg.Timeout},
+	}
+}
+
+func (c *scrapeClient) Strategy() Strategy { return StrategyScrape }
+
+func (c *scrapeClient) GetContent(_ context.Context, _ ContentRequest) (string, error) {
+	return "", fmt.Errorf("scrape client does not support /content; use NewContentClient")
+}
+
+func (c *scrapeClient) ScrapePage(ctx context.Context, req ScrapeRequest) (ScrapeResponse, error) {
+	body, err := json.Marshal(req)
+	if err != nil {
+		return ScrapeResponse{}, fmt.Errorf("scrape: marshal request: %w", err)
+	}
+
+	url := c.cfg.BaseURL + "/scrape"
+	if c.cfg.Token != "" {
+		url += "?token=" + c.cfg.Token
+	}
+
+	httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body))
+	if err != nil {
+		return ScrapeResponse{}, fmt.Errorf("scrape: build request: %w", err)
+	}
+	httpReq.Header.Set("Content-Type", "application/json")
+
+	resp, err := c.http.Do(httpReq)
+	if err != nil {
+		return ScrapeResponse{}, fmt.Errorf("scrape: do request: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		b, _ := io.ReadAll(resp.Body)
+		return ScrapeResponse{}, fmt.Errorf("scrape: unexpected status %d: %s", resp.StatusCode, b)
+	}
+
+	var result ScrapeResponse
+	if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
+		return ScrapeResponse{}, fmt.Errorf("scrape: decode response: %w", err)
+	}
+	return result, nil
+}
+
+func (c *scrapeClient) CDPSession(_ context.Context, _ string, _ CDPSessionFunc) error {
+	return fmt.Errorf("scrape client does not support CDP; use NewCDPClient")
+}