kamil/libnovel
1
0
Fork 0
Code Issues Pull Requests 3 Actions Packages Projects Releases 162 Wiki Activity

feat: add exponential backoff, some UI elements to see the resut of a scrape

Browse Source
This commit is contained in:
Admin
2026-02-26 18:51:32 +05:00
parent d68ea71239
commit e6e6f7dc4d
12 changed files with 462 additions and 153 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
scraper/internal/browser/interface.go
View File

@@ -23,12 +23,18 @@ const (
StrategyCDP Strategy = "cdp"
)
// WaitForSelector describes the waitForSelector option sent to Browserless.
type WaitForSelector struct {
Selector string `json:"selector"`
Timeout int `json:"timeout,omitempty"` // ms
}
// ContentRequest is the body sent to POST /content.
type ContentRequest struct {
URL string `json:"url"`
WaitFor string `json:"waitForSelector,omitempty"`
WaitForTimeout int `json:"waitForTimeout,omitempty"` // ms
RejectResources bool `json:"rejectResources,omitempty"`
URL string `json:"url"`
WaitFor *WaitForSelector `json:"waitForSelector,omitempty"`
WaitForTimeout int `json:"waitForTimeout,omitempty"` // ms
RejectResourceTypes []string `json:"rejectResourceTypes,omitempty"` // e.g. ["image","stylesheet"]
}
// ScrapeElement is one element descriptor inside a ScrapeRequest.
@@ -39,9 +45,9 @@ type ScrapeElement struct {
// ScrapeRequest is the body sent to POST /scrape.
type ScrapeRequest struct {
URL string `json:"url"`
Elements []ScrapeElement `json:"elements"`
WaitFor string `json:"waitForSelector,omitempty"`
URL string `json:"url"`
Elements []ScrapeElement `json:"elements"`
WaitFor *WaitForSelector `json:"waitForSelector,omitempty"`
}
// ScrapeResult is one entry in the response from POST /scrape.