kamil/libnovel
1
0
Fork 0
Code Issues Pull Requests 3 Actions Packages Projects Releases 162 Wiki Activity

feat(e2e): use direct HTTP for chapter scraping, cap TTS at 200 chars

Browse Source 
- e2e fixture: replace single contentClient with directClient (plain HTTP)
  for chapter/metadata/ranking + contentClient (Browserless) for urlClient
  only — matches production wiring and is significantly faster
- server: add max_chars field to audio request body; truncates stripped text
  to N runes before sending to Kokoro (used by e2e for quick TTS tests)
- fix: move RankingItem to scraper package to break novelfire→storage import
  cycle; storage.RankingItem is now a type alias for backward compat
- fix: update stale New() call in novelfire integration test (missing args)
- fix: replace removed blob-ranking methods in storage integration test with
  current per-item API (UpsertRankingItem/ListRankingItems/RankingLastUpdated)
- justfile: add test-e2e and e2e tasks
This commit is contained in:
Admin
2026-03-03 20:02:15 +05:00
parent b8d4d94b18
commit af3c487afb
8 changed files with 105 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files

104
scraper/internal/storage/integration_test.go
View File

@@ -463,79 +463,61 @@ func TestPocketBaseStore_Ranking(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
const testData = `[{"rank":1,"slug":"test-book","title":"Test Book"}]`
t.Run("SetRanking", func(t *testing.T) {
if err := store.SetRanking(ctx, testData); err != nil {
t.Fatalf("SetRanking: %v", err)
}
t.Log("SetRanking succeeded")
})
t.Run("GetRanking", func(t *testing.T) {
data, updated, err := store.GetRanking(ctx)
if err != nil {
t.Fatalf("GetRanking: %v", err)
}
if data == "" {
t.Error("GetRanking returned empty data")
}
if updated.IsZero() {
t.Error("GetRanking returned zero updated time")
}
t.Logf("data: %s", data)
t.Logf("updated: %s", updated)
})
t.Run("RankingModTime", func(t *testing.T) {
fi, err := store.RankingModTime(ctx)
if err != nil {
t.Fatalf("RankingModTime: %v", err)
}
if fi == nil {
t.Fatal("RankingModTime returned nil FileInfo")
}
if fi.ModTime().IsZero() {
t.Error("RankingModTime.ModTime() is zero")
}
t.Logf("ranking modtime: %s", fi.ModTime())
})
}
// TestPocketBaseStore_RankingPageHTML tests SetRankingPageHTML → GetRankingPageHTML.
func TestPocketBaseStore_RankingPageHTML(t *testing.T) {
store := newTestPocketBaseStore(t)
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
const page = 9999 // unlikely to collide with real data
const html = `<html><body>integration test page 9999</body></html>`
slug1 := testSlug(t) + "-rank1"
slug2 := testSlug(t) + "-rank2"
t.Cleanup(func() {
cleanCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
_ = store.pb.deleteWhere(cleanCtx, "ranking_html", fmt.Sprintf(`page=%d`, page))
})
t.Run("SetRankingPageHTML", func(t *testing.T) {
if err := store.SetRankingPageHTML(ctx, page, html); err != nil {
t.Fatalf("SetRankingPageHTML: %v", err)
for _, sl := range []string{slug1, slug2} {
_ = store.pb.deleteWhere(cleanCtx, "ranking", fmt.Sprintf(`slug="%s"`, sl))
}
})
t.Run("GetRankingPageHTML", func(t *testing.T) {
got, updated, err := store.GetRankingPageHTML(ctx, page)
items := []RankingItem{
{Rank: 1, Slug: slug1, Title: "Test Book One", SourceURL: "https://example.com/1"},
{Rank: 2, Slug: slug2, Title: "Test Book Two", SourceURL: "https://example.com/2"},
}
t.Run("WriteRankingItem", func(t *testing.T) {
for _, item := range items {
if err := store.UpsertRankingItem(ctx, item); err != nil {
t.Fatalf("UpsertRankingItem(%q): %v", item.Slug, err)
}
}
t.Log("UpsertRankingItem succeeded")
})
t.Run("ReadRankingItems", func(t *testing.T) {
got, err := store.ListRankingItems(ctx)
if err != nil {
t.Fatalf("GetRankingPageHTML: %v", err)
t.Fatalf("ListRankingItems: %v", err)
}
if got != html {
t.Errorf("html mismatch:\ngot: %q\nwant: %q", got, html)
found := 0
for _, g := range got {
if g.Slug == slug1 || g.Slug == slug2 {
found++
}
}
if found != 2 {
t.Errorf("ListRankingItems: found %d of 2 test items in %d total", found, len(got))
}
t.Logf("ListRankingItems returned %d total items, %d test items", len(got), found)
})
t.Run("RankingFreshEnough", func(t *testing.T) {
updated, err := store.RankingLastUpdated(ctx)
if err != nil {
t.Fatalf("RankingLastUpdated: %v", err)
}
if updated.IsZero() {
t.Error("updated time is zero")
t.Error("RankingLastUpdated returned zero time immediately after write")
}
t.Logf("retrieved HTML (%d bytes), updated=%s", len(got), updated)
fresh := time.Since(updated) < 24*time.Hour
if !fresh {
t.Errorf("RankingLastUpdated = %s; want within 24h", updated)
}
t.Logf("RankingLastUpdated = %s (fresh=%v)", updated, fresh)
})
}