feat: add exponential backoff, some UI elements to see the resut of a scrape

scraper/internal/browser/cdp.go

@@ -15,6 +15,7 @@ import (
 // cdpClient implements BrowserClient using the CDP WebSocket endpoint.
 type cdpClient struct {
 	cfg Config
+	sem chan struct{}
 }
 
 // NewCDPClient returns a BrowserClient that uses CDP WebSocket sessions.
@@ -22,7 +23,7 @@ func NewCDPClient(cfg Config) BrowserClient {
 	if cfg.Timeout == 0 {
 		cfg.Timeout = 60 * time.Second
 	}
-	return &cdpClient{cfg: cfg}
+	return &cdpClient{cfg: cfg, sem: makeSem(cfg.MaxConcurrent)}
 }
 
 func (c *cdpClient) Strategy() Strategy { return StrategyCDP }
@@ -38,6 +39,11 @@ func (c *cdpClient) ScrapePage(_ context.Context, _ ScrapeRequest) (ScrapeRespon
 // CDPSession opens a WebSocket to the Browserless /devtools/browser endpoint,
 // navigates to pageURL, and invokes fn with a live CDPConn.
 func (c *cdpClient) CDPSession(ctx context.Context, pageURL string, fn CDPSessionFunc) error {
+	if err := acquire(ctx, c.sem); err != nil {
+		return fmt.Errorf("cdp: semaphore: %w", err)
+	}
+	defer release(c.sem)
+
 	// Build WebSocket URL: ws://host:port/devtools/browser?token=...&url=...
 	wsURL := strings.Replace(c.cfg.BaseURL, "http://", "ws://", 1)
 	wsURL = strings.Replace(wsURL, "https://", "wss://", 1)