feat: initial commit
This commit is contained in:
Admin
131
scraper/internal/browser/cdp.go
Normal file
131
scraper/internal/browser/cdp.go
Normal file
@@ -0,0 +1,131 @@
|
||||
package browser
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/gorilla/websocket"
|
||||
)
|
||||
|
||||
// cdpClient implements BrowserClient using the CDP WebSocket endpoint.
|
||||
type cdpClient struct {
|
||||
cfg Config
|
||||
}
|
||||
|
||||
// NewCDPClient returns a BrowserClient that uses CDP WebSocket sessions.
|
||||
func NewCDPClient(cfg Config) BrowserClient {
|
||||
if cfg.Timeout == 0 {
|
||||
cfg.Timeout = 60 * time.Second
|
||||
}
|
||||
return &cdpClient{cfg: cfg}
|
||||
}
|
||||
|
||||
func (c *cdpClient) Strategy() Strategy { return StrategyCDP }
|
||||
|
||||
func (c *cdpClient) GetContent(_ context.Context, _ ContentRequest) (string, error) {
|
||||
return "", fmt.Errorf("CDP client does not support /content; use NewContentClient")
|
||||
}
|
||||
|
||||
func (c *cdpClient) ScrapePage(_ context.Context, _ ScrapeRequest) (ScrapeResponse, error) {
|
||||
return ScrapeResponse{}, fmt.Errorf("CDP client does not support /scrape; use NewScrapeClient")
|
||||
}
|
||||
|
||||
// CDPSession opens a WebSocket to the Browserless /devtools/browser endpoint,
|
||||
// navigates to pageURL, and invokes fn with a live CDPConn.
|
||||
func (c *cdpClient) CDPSession(ctx context.Context, pageURL string, fn CDPSessionFunc) error {
|
||||
// Build WebSocket URL: ws://host:port/devtools/browser?token=...&url=...
|
||||
wsURL := strings.Replace(c.cfg.BaseURL, "http://", "ws://", 1)
|
||||
wsURL = strings.Replace(wsURL, "https://", "wss://", 1)
|
||||
wsURL += "/devtools/browser"
|
||||
sep := "?"
|
||||
if c.cfg.Token != "" {
|
||||
wsURL += sep + "token=" + c.cfg.Token
|
||||
sep = "&"
|
||||
}
|
||||
wsURL += sep + "url=" + pageURL
|
||||
|
||||
dialer := websocket.Dialer{
|
||||
HandshakeTimeout: 15 * time.Second,
|
||||
Proxy: http.ProxyFromEnvironment,
|
||||
}
|
||||
|
||||
conn, _, err := dialer.DialContext(ctx, wsURL, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cdp: dial %s: %w", wsURL, err)
|
||||
}
|
||||
|
||||
cdp := &cdpConn{ws: conn}
|
||||
defer cdp.Close()
|
||||
|
||||
return fn(ctx, cdp)
|
||||
}
|
||||
|
||||
// ─── cdpConn ─────────────────────────────────────────────────────────────────
|
||||
|
||||
type cdpConn struct {
|
||||
ws *websocket.Conn
|
||||
counter atomic.Int64
|
||||
}
|
||||
|
||||
type cdpRequest struct {
|
||||
ID int64 `json:"id"`
|
||||
Method string `json:"method"`
|
||||
Params map[string]any `json:"params,omitempty"`
|
||||
}
|
||||
|
||||
type cdpResponse struct {
|
||||
ID int64 `json:"id"`
|
||||
Result map[string]any `json:"result,omitempty"`
|
||||
Error *struct {
|
||||
Code int `json:"code"`
|
||||
Message string `json:"message"`
|
||||
} `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
func (c *cdpConn) Send(ctx context.Context, method string, params map[string]any) (map[string]any, error) {
|
||||
id := c.counter.Add(1)
|
||||
|
||||
req := cdpRequest{ID: id, Method: method, Params: params}
|
||||
data, err := json.Marshal(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cdp send: marshal: %w", err)
|
||||
}
|
||||
|
||||
if dl, ok := ctx.Deadline(); ok {
|
||||
_ = c.ws.SetWriteDeadline(dl)
|
||||
}
|
||||
if err := c.ws.WriteMessage(websocket.TextMessage, data); err != nil {
|
||||
return nil, fmt.Errorf("cdp send: write: %w", err)
|
||||
}
|
||||
|
||||
// Read messages until we find the response matching our id.
|
||||
for {
|
||||
if dl, ok := ctx.Deadline(); ok {
|
||||
_ = c.ws.SetReadDeadline(dl)
|
||||
}
|
||||
_, msg, err := c.ws.ReadMessage()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cdp send: read: %w", err)
|
||||
}
|
||||
var resp cdpResponse
|
||||
if err := json.Unmarshal(msg, &resp); err != nil {
|
||||
continue // skip non-JSON frames (events etc.)
|
||||
}
|
||||
if resp.ID != id {
|
||||
continue // event or different command reply
|
||||
}
|
||||
if resp.Error != nil {
|
||||
return nil, fmt.Errorf("cdp error %d: %s", resp.Error.Code, resp.Error.Message)
|
||||
}
|
||||
return resp.Result, nil
|
||||
}
|
||||
}
|
||||
|
||||
func (c *cdpConn) Close() error {
|
||||
return c.ws.Close()
|
||||
}
|
||||
146
scraper/internal/browser/content_scrape.go
Normal file
146
scraper/internal/browser/content_scrape.go
Normal file
@@ -0,0 +1,146 @@
|
||||
package browser
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Config holds the connection parameters for a Browserless instance.
|
||||
type Config struct {
|
||||
// BaseURL is the HTTP base URL, e.g. "http://localhost:3000".
|
||||
BaseURL string
|
||||
// Token is the optional API token (BROWSERLESS_TOKEN env var).
|
||||
Token string
|
||||
// Timeout is the per-request HTTP timeout; defaults to 60 s.
|
||||
Timeout time.Duration
|
||||
}
|
||||
|
||||
// contentClient implements BrowserClient using the /content endpoint.
|
||||
type contentClient struct {
|
||||
cfg Config
|
||||
http *http.Client
|
||||
}
|
||||
|
||||
// NewContentClient returns a BrowserClient that uses POST /content.
|
||||
func NewContentClient(cfg Config) BrowserClient {
|
||||
if cfg.Timeout == 0 {
|
||||
cfg.Timeout = 60 * time.Second
|
||||
}
|
||||
return &contentClient{
|
||||
cfg: cfg,
|
||||
http: &http.Client{Timeout: cfg.Timeout},
|
||||
}
|
||||
}
|
||||
|
||||
func (c *contentClient) Strategy() Strategy { return StrategyContent }
|
||||
|
||||
func (c *contentClient) GetContent(ctx context.Context, req ContentRequest) (string, error) {
|
||||
body, err := json.Marshal(req)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("content: marshal request: %w", err)
|
||||
}
|
||||
|
||||
url := c.cfg.BaseURL + "/content"
|
||||
if c.cfg.Token != "" {
|
||||
url += "?token=" + c.cfg.Token
|
||||
}
|
||||
|
||||
httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("content: build request: %w", err)
|
||||
}
|
||||
httpReq.Header.Set("Content-Type", "application/json")
|
||||
|
||||
resp, err := c.http.Do(httpReq)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("content: do request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
return "", fmt.Errorf("content: unexpected status %d: %s", resp.StatusCode, b)
|
||||
}
|
||||
|
||||
raw, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("content: read body: %w", err)
|
||||
}
|
||||
return string(raw), nil
|
||||
}
|
||||
|
||||
func (c *contentClient) ScrapePage(_ context.Context, _ ScrapeRequest) (ScrapeResponse, error) {
|
||||
return ScrapeResponse{}, fmt.Errorf("content client does not support /scrape; use NewScrapeClient")
|
||||
}
|
||||
|
||||
func (c *contentClient) CDPSession(_ context.Context, _ string, _ CDPSessionFunc) error {
|
||||
return fmt.Errorf("content client does not support CDP; use NewCDPClient")
|
||||
}
|
||||
|
||||
// ─── /scrape client ───────────────────────────────────────────────────────────
|
||||
|
||||
type scrapeClient struct {
|
||||
cfg Config
|
||||
http *http.Client
|
||||
}
|
||||
|
||||
// NewScrapeClient returns a BrowserClient that uses POST /scrape.
|
||||
func NewScrapeClient(cfg Config) BrowserClient {
|
||||
if cfg.Timeout == 0 {
|
||||
cfg.Timeout = 60 * time.Second
|
||||
}
|
||||
return &scrapeClient{
|
||||
cfg: cfg,
|
||||
http: &http.Client{Timeout: cfg.Timeout},
|
||||
}
|
||||
}
|
||||
|
||||
func (c *scrapeClient) Strategy() Strategy { return StrategyScrape }
|
||||
|
||||
func (c *scrapeClient) GetContent(_ context.Context, _ ContentRequest) (string, error) {
|
||||
return "", fmt.Errorf("scrape client does not support /content; use NewContentClient")
|
||||
}
|
||||
|
||||
func (c *scrapeClient) ScrapePage(ctx context.Context, req ScrapeRequest) (ScrapeResponse, error) {
|
||||
body, err := json.Marshal(req)
|
||||
if err != nil {
|
||||
return ScrapeResponse{}, fmt.Errorf("scrape: marshal request: %w", err)
|
||||
}
|
||||
|
||||
url := c.cfg.BaseURL + "/scrape"
|
||||
if c.cfg.Token != "" {
|
||||
url += "?token=" + c.cfg.Token
|
||||
}
|
||||
|
||||
httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return ScrapeResponse{}, fmt.Errorf("scrape: build request: %w", err)
|
||||
}
|
||||
httpReq.Header.Set("Content-Type", "application/json")
|
||||
|
||||
resp, err := c.http.Do(httpReq)
|
||||
if err != nil {
|
||||
return ScrapeResponse{}, fmt.Errorf("scrape: do request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
return ScrapeResponse{}, fmt.Errorf("scrape: unexpected status %d: %s", resp.StatusCode, b)
|
||||
}
|
||||
|
||||
var result ScrapeResponse
|
||||
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
||||
return ScrapeResponse{}, fmt.Errorf("scrape: decode response: %w", err)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (c *scrapeClient) CDPSession(_ context.Context, _ string, _ CDPSessionFunc) error {
|
||||
return fmt.Errorf("scrape client does not support CDP; use NewCDPClient")
|
||||
}
|
||||
101
scraper/internal/browser/interface.go
Normal file
101
scraper/internal/browser/interface.go
Normal file
@@ -0,0 +1,101 @@
|
||||
// Package browser defines the BrowserClient interface and helper types for
|
||||
// communicating with a Browserless instance.
|
||||
package browser
|
||||
|
||||
import "context"
|
||||
|
||||
// Strategy selects which Browserless API endpoint / protocol to use.
|
||||
type Strategy string
|
||||
|
||||
const (
|
||||
// StrategyContent uses the POST /content endpoint, which returns the final
|
||||
// rendered HTML of the page. Fastest; suitable for most JS-rendered sites.
|
||||
StrategyContent Strategy = "content"
|
||||
|
||||
// StrategyScrape uses the POST /scrape endpoint, which accepts a list of
|
||||
// CSS selectors and returns structured JSON. Good when you know exactly
|
||||
// which elements you need.
|
||||
StrategyScrape Strategy = "scrape"
|
||||
|
||||
// StrategyCDP uses the WebSocket /devtools/browser endpoint (Chrome
|
||||
// DevTools Protocol). Most powerful; required for complex interactions
|
||||
// (clicking, scrolling, waiting for network idle, etc.).
|
||||
StrategyCDP Strategy = "cdp"
|
||||
)
|
||||
|
||||
// ContentRequest is the body sent to POST /content.
|
||||
type ContentRequest struct {
|
||||
URL string `json:"url"`
|
||||
WaitFor string `json:"waitForSelector,omitempty"`
|
||||
WaitForTimeout int `json:"waitForTimeout,omitempty"` // ms
|
||||
RejectResources bool `json:"rejectResources,omitempty"`
|
||||
}
|
||||
|
||||
// ScrapeElement is one element descriptor inside a ScrapeRequest.
|
||||
type ScrapeElement struct {
|
||||
Selector string `json:"selector"`
|
||||
Timeout int `json:"timeout,omitempty"` // ms
|
||||
}
|
||||
|
||||
// ScrapeRequest is the body sent to POST /scrape.
|
||||
type ScrapeRequest struct {
|
||||
URL string `json:"url"`
|
||||
Elements []ScrapeElement `json:"elements"`
|
||||
WaitFor string `json:"waitForSelector,omitempty"`
|
||||
}
|
||||
|
||||
// ScrapeResult is one entry in the response from POST /scrape.
|
||||
type ScrapeResult struct {
|
||||
Selector string `json:"selector"`
|
||||
Results []ScrapeElement `json:"results"`
|
||||
}
|
||||
|
||||
// ScrapeAttribute holds a single attribute value from a scraped element.
|
||||
type ScrapeAttribute struct {
|
||||
Name string `json:"name"`
|
||||
Value string `json:"value"`
|
||||
}
|
||||
|
||||
// ScrapedElement is one item inside ScrapeResult.Results.
|
||||
type ScrapedElement struct {
|
||||
Text string `json:"text"`
|
||||
Attributes []ScrapeAttribute `json:"attributes"`
|
||||
}
|
||||
|
||||
// ScrapeResponse is the top-level response from POST /scrape.
|
||||
type ScrapeResponse struct {
|
||||
Data []ScrapeResult `json:"data"`
|
||||
}
|
||||
|
||||
// BrowserClient is an abstraction over the three Browserless API strategies.
|
||||
// Callers choose the strategy best suited to the target site; the interface
|
||||
// signature is identical regardless of strategy.
|
||||
type BrowserClient interface {
|
||||
// Strategy returns the strategy this client uses.
|
||||
Strategy() Strategy
|
||||
|
||||
// GetContent fetches the fully-rendered HTML of url using the /content
|
||||
// endpoint. Only meaningful when Strategy() == StrategyContent.
|
||||
GetContent(ctx context.Context, req ContentRequest) (string, error)
|
||||
|
||||
// ScrapePage calls the /scrape endpoint and returns structured data.
|
||||
// Only meaningful when Strategy() == StrategyScrape.
|
||||
ScrapePage(ctx context.Context, req ScrapeRequest) (ScrapeResponse, error)
|
||||
|
||||
// CDPSession opens a CDP WebSocket session and calls fn with the raw
|
||||
// WebSocket connection. Only meaningful when Strategy() == StrategyCDP.
|
||||
// The session is closed when fn returns.
|
||||
CDPSession(ctx context.Context, pageURL string, fn CDPSessionFunc) error
|
||||
}
|
||||
|
||||
// CDPSessionFunc is the callback invoked inside a CDP session.
|
||||
// conn is a live *websocket.Conn connected to a Browserless CDP endpoint.
|
||||
type CDPSessionFunc func(ctx context.Context, conn CDPConn) error
|
||||
|
||||
// CDPConn is the minimal interface the orchestrator needs over a CDP WebSocket.
|
||||
type CDPConn interface {
|
||||
// Send sends a raw CDP command (JSON-encoded) and returns the response.
|
||||
Send(ctx context.Context, method string, params map[string]any) (map[string]any, error)
|
||||
// Close closes the underlying connection.
|
||||
Close() error
|
||||
}
|
||||
Reference in New Issue
Block a user