feat(scrape-range): add chapter range scraping for admin users
Adds FromChapter/ToChapter fields to orchestrator.Config and skips out-of-range chapters in processBook. Exposes POST /scrape/book/range Go endpoint and a matching UI proxy at /api/scrape/range. The book detail page now shows admins a range input (from/to chapter) and a per-chapter 'scrape from here up' button.
This commit is contained in:
@@ -45,6 +45,14 @@ type Config struct {
|
||||
// that one book instead of walking the full catalogue.
|
||||
SingleBookURL string
|
||||
|
||||
// FromChapter, when > 0, skips chapters with number < FromChapter.
|
||||
// Only effective in single-book mode.
|
||||
FromChapter int
|
||||
|
||||
// ToChapter, when > 0, skips chapters with number > ToChapter.
|
||||
// Only effective in single-book mode. 0 means "no upper limit".
|
||||
ToChapter int
|
||||
|
||||
// OnProgress is called periodically with the current progress counters.
|
||||
// It is always called on completion (success or failure). May be nil.
|
||||
OnProgress func(p Progress)
|
||||
@@ -204,6 +212,15 @@ func (o *Orchestrator) Run(ctx context.Context) error {
|
||||
|
||||
// Enqueue chapter jobs.
|
||||
for _, ref := range refs {
|
||||
// Apply chapter range filter (only in single-book mode when set).
|
||||
if o.cfg.FromChapter > 0 && ref.Number < o.cfg.FromChapter {
|
||||
chaptersSkipped.Add(1)
|
||||
continue
|
||||
}
|
||||
if o.cfg.ToChapter > 0 && ref.Number > o.cfg.ToChapter {
|
||||
chaptersSkipped.Add(1)
|
||||
continue
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
|
||||
@@ -33,6 +33,29 @@ func (s *Server) handleScrapeBook(w http.ResponseWriter, r *http.Request) {
|
||||
s.runAsync(w, cfg)
|
||||
}
|
||||
|
||||
// handleScrapeBookRange handles POST /api/scrape/book/range.
|
||||
// Body: {"url": "...", "from": N, "to": M}
|
||||
// Scrapes only chapters in the range [from, to] (inclusive).
|
||||
// from=0 means "start from chapter 1"; to=0 means "no upper limit".
|
||||
func (s *Server) handleScrapeBookRange(w http.ResponseWriter, r *http.Request) {
|
||||
var body struct {
|
||||
URL string `json:"url"`
|
||||
From int `json:"from"`
|
||||
To int `json:"to"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil || body.URL == "" {
|
||||
http.Error(w, `{"error":"request body must be JSON with \"url\" field"}`, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
cfg := s.oCfg
|
||||
cfg.SingleBookURL = body.URL
|
||||
cfg.FromChapter = body.From
|
||||
cfg.ToChapter = body.To
|
||||
|
||||
s.runAsync(w, cfg)
|
||||
}
|
||||
|
||||
// runAsync launches an orchestrator in the background and returns 202 Accepted.
|
||||
// Only one scrape job runs at a time; concurrent requests receive 409 Conflict.
|
||||
func (s *Server) runAsync(w http.ResponseWriter, cfg orchestrator.Config) {
|
||||
|
||||
Reference in New Issue
Block a user