feat(storage): add scraping_tasks collection and Store interface methods
Adds a scraping_tasks PocketBase collection with fields for kind, status, progress counters, timestamps, and error info. Exposes CreateScrapeTask, UpdateScrapeTask, and ListScrapeTasks on the Store interface with implementations in HybridStore and PocketBaseStore.
This commit is contained in:
@@ -2,14 +2,18 @@
|
||||
//
|
||||
// Collections expected in PocketBase:
|
||||
//
|
||||
// books — slug(text,unique), title, author, cover, status, genres(json),
|
||||
// summary, total_chapters(number), source_url, ranking(number), updated(date)
|
||||
// chapters_idx — slug(text), number(number), title, date_label, updated(date)
|
||||
// ranking — rank(number), slug(text,unique), title, author, cover, status,
|
||||
// genres(json), source_url, updated(date)
|
||||
// progress — session_id(text), slug(text), chapter(number), updated(date)
|
||||
// audio_cache — cache_key(text,unique), filename(text), updated(date)
|
||||
// app_users — username(text,unique), password_hash(text), role(text), created(date)
|
||||
// books — slug(text,unique), title, author, cover, status, genres(json),
|
||||
// summary, total_chapters(number), source_url, ranking(number), updated(date)
|
||||
// chapters_idx — slug(text), number(number), title, date_label, updated(date)
|
||||
// ranking — rank(number), slug(text,unique), title, author, cover, status,
|
||||
// genres(json), source_url, updated(date)
|
||||
// progress — session_id(text), slug(text), chapter(number), updated(date)
|
||||
// audio_cache — cache_key(text,unique), filename(text), updated(date)
|
||||
// app_users — username(text,unique), password_hash(text), role(text), created(date)
|
||||
// scraping_tasks — id(auto), kind(text), target_url(text), status(text),
|
||||
// books_found(number), chapters_scraped(number),
|
||||
// chapters_skipped(number), errors(number),
|
||||
// started(date), finished(date), error_message(text)
|
||||
package storage
|
||||
|
||||
import (
|
||||
@@ -345,6 +349,22 @@ func (s *PocketBaseStore) EnsureCollections(ctx context.Context) error {
|
||||
{"name": "created", "type": "date"},
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "scraping_tasks",
|
||||
"type": "base",
|
||||
"fields": []map[string]interface{}{
|
||||
{"name": "kind", "type": "text", "required": true}, // "catalogue" | "book"
|
||||
{"name": "target_url", "type": "text"}, // set for single-book scrapes
|
||||
{"name": "status", "type": "text", "required": true}, // "running" | "done" | "failed" | "cancelled"
|
||||
{"name": "books_found", "type": "number"},
|
||||
{"name": "chapters_scraped", "type": "number"},
|
||||
{"name": "chapters_skipped", "type": "number"},
|
||||
{"name": "errors", "type": "number"},
|
||||
{"name": "started", "type": "date"},
|
||||
{"name": "finished", "type": "date"},
|
||||
{"name": "error_message", "type": "text"},
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, col := range collections {
|
||||
name, _ := col["name"].(string)
|
||||
@@ -590,6 +610,58 @@ func (s *PocketBaseStore) GetAudioCache(ctx context.Context, cacheKey string) (s
|
||||
return filename, filename != "", nil
|
||||
}
|
||||
|
||||
// ─── Scraping tasks ───────────────────────────────────────────────────────────
|
||||
|
||||
// CreateScrapingTask inserts a new scraping_tasks record with status="running"
|
||||
// and returns the newly created record's ID.
|
||||
func (s *PocketBaseStore) CreateScrapingTask(ctx context.Context, kind, targetURL string) (string, error) {
|
||||
data := map[string]interface{}{
|
||||
"kind": kind,
|
||||
"target_url": targetURL,
|
||||
"status": "running",
|
||||
"books_found": 0,
|
||||
"chapters_scraped": 0,
|
||||
"chapters_skipped": 0,
|
||||
"errors": 0,
|
||||
"started": time.Now().UTC().Format(time.RFC3339),
|
||||
}
|
||||
resp, err := s.pb.do(ctx, http.MethodPost, "/api/collections/scraping_tasks/records", data)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated {
|
||||
return "", fmt.Errorf("pocketbase: CreateScrapingTask: status %d: %s", resp.StatusCode, b)
|
||||
}
|
||||
var rec map[string]interface{}
|
||||
if err := json.Unmarshal(b, &rec); err != nil {
|
||||
return "", fmt.Errorf("pocketbase: CreateScrapingTask: decode: %w", err)
|
||||
}
|
||||
id, _ := rec["id"].(string)
|
||||
return id, nil
|
||||
}
|
||||
|
||||
// UpdateScrapingTask patches counters on an existing scraping_tasks record.
|
||||
func (s *PocketBaseStore) UpdateScrapingTask(ctx context.Context, id string, data map[string]interface{}) error {
|
||||
resp, err := s.pb.do(ctx, http.MethodPatch,
|
||||
fmt.Sprintf("/api/collections/scraping_tasks/records/%s", id), data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
return fmt.Errorf("pocketbase: UpdateScrapingTask id=%s: status %d: %s", id, resp.StatusCode, b)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ListScrapingTasks returns all scraping_tasks sorted by started descending.
|
||||
func (s *PocketBaseStore) ListScrapingTasks(ctx context.Context) ([]map[string]interface{}, error) {
|
||||
return s.pb.listAll(ctx, "scraping_tasks", "", "-started")
|
||||
}
|
||||
|
||||
// ─── helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
// pbEsc escapes a string for use in a PocketBase filter expression.
|
||||
|
||||
Reference in New Issue
Block a user