feat: initial commit

scraper/internal/browser/cdp.go

@@ -0,0 +1,131 @@
+package browser
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"strings"
+	"sync/atomic"
+	"time"
+
+	"github.com/gorilla/websocket"
+)
+
+// cdpClient implements BrowserClient using the CDP WebSocket endpoint.
+type cdpClient struct {
+	cfg Config
+}
+
+// NewCDPClient returns a BrowserClient that uses CDP WebSocket sessions.
+func NewCDPClient(cfg Config) BrowserClient {
+	if cfg.Timeout == 0 {
+		cfg.Timeout = 60 * time.Second
+	}
+	return &cdpClient{cfg: cfg}
+}
+
+func (c *cdpClient) Strategy() Strategy { return StrategyCDP }
+
+func (c *cdpClient) GetContent(_ context.Context, _ ContentRequest) (string, error) {
+	return "", fmt.Errorf("CDP client does not support /content; use NewContentClient")
+}
+
+func (c *cdpClient) ScrapePage(_ context.Context, _ ScrapeRequest) (ScrapeResponse, error) {
+	return ScrapeResponse{}, fmt.Errorf("CDP client does not support /scrape; use NewScrapeClient")
+}
+
+// CDPSession opens a WebSocket to the Browserless /devtools/browser endpoint,
+// navigates to pageURL, and invokes fn with a live CDPConn.
+func (c *cdpClient) CDPSession(ctx context.Context, pageURL string, fn CDPSessionFunc) error {
+	// Build WebSocket URL: ws://host:port/devtools/browser?token=...&url=...
+	wsURL := strings.Replace(c.cfg.BaseURL, "http://", "ws://", 1)
+	wsURL = strings.Replace(wsURL, "https://", "wss://", 1)
+	wsURL += "/devtools/browser"
+	sep := "?"
+	if c.cfg.Token != "" {
+		wsURL += sep + "token=" + c.cfg.Token
+		sep = "&"
+	}
+	wsURL += sep + "url=" + pageURL
+
+	dialer := websocket.Dialer{
+		HandshakeTimeout: 15 * time.Second,
+		Proxy:            http.ProxyFromEnvironment,
+	}
+
+	conn, _, err := dialer.DialContext(ctx, wsURL, nil)
+	if err != nil {
+		return fmt.Errorf("cdp: dial %s: %w", wsURL, err)
+	}
+
+	cdp := &cdpConn{ws: conn}
+	defer cdp.Close()
+
+	return fn(ctx, cdp)
+}
+
+// ─── cdpConn ─────────────────────────────────────────────────────────────────
+
+type cdpConn struct {
+	ws      *websocket.Conn
+	counter atomic.Int64
+}
+
+type cdpRequest struct {
+	ID     int64          `json:"id"`
+	Method string         `json:"method"`
+	Params map[string]any `json:"params,omitempty"`
+}
+
+type cdpResponse struct {
+	ID     int64          `json:"id"`
+	Result map[string]any `json:"result,omitempty"`
+	Error  *struct {
+		Code    int    `json:"code"`
+		Message string `json:"message"`
+	} `json:"error,omitempty"`
+}
+
+func (c *cdpConn) Send(ctx context.Context, method string, params map[string]any) (map[string]any, error) {
+	id := c.counter.Add(1)
+
+	req := cdpRequest{ID: id, Method: method, Params: params}
+	data, err := json.Marshal(req)
+	if err != nil {
+		return nil, fmt.Errorf("cdp send: marshal: %w", err)
+	}
+
+	if dl, ok := ctx.Deadline(); ok {
+		_ = c.ws.SetWriteDeadline(dl)
+	}
+	if err := c.ws.WriteMessage(websocket.TextMessage, data); err != nil {
+		return nil, fmt.Errorf("cdp send: write: %w", err)
+	}
+
+	// Read messages until we find the response matching our id.
+	for {
+		if dl, ok := ctx.Deadline(); ok {
+			_ = c.ws.SetReadDeadline(dl)
+		}
+		_, msg, err := c.ws.ReadMessage()
+		if err != nil {
+			return nil, fmt.Errorf("cdp send: read: %w", err)
+		}
+		var resp cdpResponse
+		if err := json.Unmarshal(msg, &resp); err != nil {
+			continue // skip non-JSON frames (events etc.)
+		}
+		if resp.ID != id {
+			continue // event or different command reply
+		}
+		if resp.Error != nil {
+			return nil, fmt.Errorf("cdp error %d: %s", resp.Error.Code, resp.Error.Message)
+		}
+		return resp.Result, nil
+	}
+}
+
+func (c *cdpConn) Close() error {
+	return c.ws.Close()
+}