feat: initial commit
This commit is contained in:
Admin
216
scraper/internal/scraper/htmlutil/htmlutil.go
Normal file
216
scraper/internal/scraper/htmlutil/htmlutil.go
Normal file
@@ -0,0 +1,216 @@
|
||||
// Package htmlutil provides helper functions for parsing HTML with
|
||||
// golang.org/x/net/html and extracting values by Selector descriptors.
|
||||
package htmlutil
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/libnovel/scraper/internal/scraper"
|
||||
"golang.org/x/net/html"
|
||||
)
|
||||
|
||||
// ParseHTML parses raw HTML and returns the root node.
|
||||
func ParseHTML(raw string) (*html.Node, error) {
|
||||
return html.Parse(strings.NewReader(raw))
|
||||
}
|
||||
|
||||
// selectorMatches reports whether node n matches sel.
|
||||
func selectorMatches(n *html.Node, sel scraper.Selector) bool {
|
||||
if n.Type != html.ElementNode {
|
||||
return false
|
||||
}
|
||||
if sel.Tag != "" && n.Data != sel.Tag {
|
||||
return false
|
||||
}
|
||||
if sel.ID != "" {
|
||||
for _, a := range n.Attr {
|
||||
if a.Key == "id" && a.Val == sel.ID {
|
||||
goto checkClass
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
checkClass:
|
||||
if sel.Class != "" {
|
||||
for _, a := range n.Attr {
|
||||
if a.Key == "class" {
|
||||
for _, cls := range strings.Fields(a.Val) {
|
||||
if cls == sel.Class {
|
||||
goto matched
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
matched:
|
||||
return true
|
||||
}
|
||||
|
||||
// attrVal returns the value of attribute key from node n.
|
||||
func attrVal(n *html.Node, key string) string {
|
||||
for _, a := range n.Attr {
|
||||
if a.Key == key {
|
||||
return a.Val
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// textContent returns the concatenated text content of all descendant text nodes.
|
||||
func textContent(n *html.Node) string {
|
||||
var sb strings.Builder
|
||||
var walk func(*html.Node)
|
||||
walk = func(cur *html.Node) {
|
||||
if cur.Type == html.TextNode {
|
||||
sb.WriteString(cur.Data)
|
||||
}
|
||||
for c := cur.FirstChild; c != nil; c = c.NextSibling {
|
||||
walk(c)
|
||||
}
|
||||
}
|
||||
walk(n)
|
||||
return strings.TrimSpace(sb.String())
|
||||
}
|
||||
|
||||
// FindFirst returns the first node matching sel within root.
|
||||
func FindFirst(root *html.Node, sel scraper.Selector) *html.Node {
|
||||
var found *html.Node
|
||||
var walk func(*html.Node) bool
|
||||
walk = func(n *html.Node) bool {
|
||||
if selectorMatches(n, sel) {
|
||||
found = n
|
||||
return true
|
||||
}
|
||||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||
if walk(c) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
walk(root)
|
||||
return found
|
||||
}
|
||||
|
||||
// FindAll returns all nodes matching sel within root.
|
||||
func FindAll(root *html.Node, sel scraper.Selector) []*html.Node {
|
||||
var results []*html.Node
|
||||
var walk func(*html.Node)
|
||||
walk = func(n *html.Node) {
|
||||
if selectorMatches(n, sel) {
|
||||
results = append(results, n)
|
||||
}
|
||||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||
walk(c)
|
||||
}
|
||||
}
|
||||
walk(root)
|
||||
return results
|
||||
}
|
||||
|
||||
// ExtractText extracts a string value from node n using sel.
|
||||
// If sel.Attr is set the attribute value is returned; otherwise the inner text.
|
||||
func ExtractText(n *html.Node, sel scraper.Selector) string {
|
||||
if sel.Attr != "" {
|
||||
return attrVal(n, sel.Attr)
|
||||
}
|
||||
return textContent(n)
|
||||
}
|
||||
|
||||
// ExtractFirst locates the first match in root and returns its text/attr value.
|
||||
func ExtractFirst(root *html.Node, sel scraper.Selector) string {
|
||||
n := FindFirst(root, sel)
|
||||
if n == nil {
|
||||
return ""
|
||||
}
|
||||
return ExtractText(n, sel)
|
||||
}
|
||||
|
||||
// ExtractAll locates all matches in root and returns their text/attr values.
|
||||
func ExtractAll(root *html.Node, sel scraper.Selector) []string {
|
||||
nodes := FindAll(root, sel)
|
||||
out := make([]string, 0, len(nodes))
|
||||
for _, n := range nodes {
|
||||
if v := ExtractText(n, sel); v != "" {
|
||||
out = append(out, v)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// InnerHTML returns the serialized inner HTML of node n.
|
||||
func InnerHTML(n *html.Node) string {
|
||||
var sb strings.Builder
|
||||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||
_ = html.Render(&sb, c)
|
||||
}
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
// NodeToMarkdown converts the children of an HTML node to a plain-text/Markdown
|
||||
// representation suitable for chapter storage. Block elements become newlines;
|
||||
// inline elements are inlined.
|
||||
func NodeToMarkdown(n *html.Node) string {
|
||||
var sb strings.Builder
|
||||
nodeToMD(n, &sb)
|
||||
return strings.TrimSpace(sb.String())
|
||||
}
|
||||
|
||||
var blockElements = map[string]bool{
|
||||
"p": true, "div": true, "br": true, "h1": true, "h2": true,
|
||||
"h3": true, "h4": true, "h5": true, "h6": true, "li": true,
|
||||
"blockquote": true, "pre": true, "hr": true,
|
||||
}
|
||||
|
||||
func nodeToMD(n *html.Node, sb *strings.Builder) {
|
||||
switch n.Type {
|
||||
case html.TextNode:
|
||||
sb.WriteString(n.Data)
|
||||
case html.ElementNode:
|
||||
tag := n.Data
|
||||
switch tag {
|
||||
case "br":
|
||||
sb.WriteString("\n")
|
||||
case "hr":
|
||||
sb.WriteString("\n---\n")
|
||||
case "h1", "h2", "h3", "h4", "h5", "h6":
|
||||
level := int(tag[1] - '0')
|
||||
sb.WriteString("\n" + strings.Repeat("#", level) + " ")
|
||||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||
nodeToMD(c, sb)
|
||||
}
|
||||
sb.WriteString("\n\n")
|
||||
return
|
||||
case "p", "div", "blockquote":
|
||||
sb.WriteString("\n")
|
||||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||
nodeToMD(c, sb)
|
||||
}
|
||||
sb.WriteString("\n")
|
||||
return
|
||||
case "em", "i":
|
||||
sb.WriteString("*")
|
||||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||
nodeToMD(c, sb)
|
||||
}
|
||||
sb.WriteString("*")
|
||||
return
|
||||
case "strong", "b":
|
||||
sb.WriteString("**")
|
||||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||
nodeToMD(c, sb)
|
||||
}
|
||||
sb.WriteString("**")
|
||||
return
|
||||
case "script", "style", "noscript":
|
||||
return // drop
|
||||
}
|
||||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||
nodeToMD(c, sb)
|
||||
}
|
||||
if blockElements[tag] {
|
||||
sb.WriteString("\n")
|
||||
}
|
||||
}
|
||||
}
|
||||
149
scraper/internal/scraper/interfaces.go
Normal file
149
scraper/internal/scraper/interfaces.go
Normal file
@@ -0,0 +1,149 @@
|
||||
// Package scraper defines the core interfaces and domain types for the libnovel
|
||||
// scraping system. Each novel source implements these interfaces; the orchestrator
|
||||
// wires them together without knowing anything about the concrete provider.
|
||||
package scraper
|
||||
|
||||
import "context"
|
||||
|
||||
// ─── Domain types ────────────────────────────────────────────────────────────
|
||||
|
||||
// BookMeta carries all bibliographic information about a novel.
|
||||
type BookMeta struct {
|
||||
// Slug is a URL-safe identifier derived from the book title, e.g. "a-dragon-against-the-whole-world".
|
||||
Slug string `yaml:"slug"`
|
||||
// Title is the human-readable novel title.
|
||||
Title string `yaml:"title"`
|
||||
// Author of the novel.
|
||||
Author string `yaml:"author"`
|
||||
// Cover is an absolute URL to the cover image.
|
||||
Cover string `yaml:"cover,omitempty"`
|
||||
// Status is e.g. "Ongoing", "Completed".
|
||||
Status string `yaml:"status,omitempty"`
|
||||
// Genres is a list of genre tags.
|
||||
Genres []string `yaml:"genres,omitempty"`
|
||||
// Summary is the full description/synopsis text.
|
||||
Summary string `yaml:"summary,omitempty"`
|
||||
// TotalChapters is the total number of chapters known at scrape time.
|
||||
TotalChapters int `yaml:"total_chapters,omitempty"`
|
||||
// SourceURL is the canonical URL of the book's landing page.
|
||||
SourceURL string `yaml:"source_url"`
|
||||
}
|
||||
|
||||
// CatalogueEntry is a lightweight reference returned by CatalogueProvider.
|
||||
type CatalogueEntry struct {
|
||||
// Title is the novel title as shown in the catalogue listing.
|
||||
Title string
|
||||
// URL is the canonical landing-page URL of the novel.
|
||||
URL string
|
||||
}
|
||||
|
||||
// ChapterRef is a reference to a single chapter returned by ChapterListProvider.
|
||||
type ChapterRef struct {
|
||||
// Number is the 1-based chapter index within the book.
|
||||
Number int
|
||||
// Title is the chapter display title.
|
||||
Title string
|
||||
// URL is the full URL of the chapter page.
|
||||
URL string
|
||||
// Volume is an optional volume number (0 means no volume grouping).
|
||||
Volume int
|
||||
}
|
||||
|
||||
// Chapter contains the fully-extracted text of a single chapter.
|
||||
type Chapter struct {
|
||||
Ref ChapterRef
|
||||
// Text is the plain / lightly-formatted chapter body (Markdown).
|
||||
Text string
|
||||
}
|
||||
|
||||
// ─── Scraping selector descriptors ───────────────────────────────────────────
|
||||
|
||||
// Selector describes how to locate an element in an HTML document.
|
||||
// Exactly one of Tag, Class, or ID should be non-empty; when multiple are set
|
||||
// they are combined (AND semantics).
|
||||
type Selector struct {
|
||||
// Tag is the HTML element name, e.g. "div", "p", "h1".
|
||||
Tag string
|
||||
// Class is one CSS class name (without the leading dot).
|
||||
Class string
|
||||
// ID is the element id attribute (without the leading #).
|
||||
ID string
|
||||
// Attr is an optional attribute name whose value should be extracted
|
||||
// instead of the text content (e.g. "href", "src").
|
||||
Attr string
|
||||
// Multiple indicates that all matching elements should be collected,
|
||||
// not just the first one.
|
||||
Multiple bool
|
||||
}
|
||||
|
||||
// ─── Provider interfaces ──────────────────────────────────────────────────────
|
||||
|
||||
// CatalogueProvider can enumerate every novel available on a source site.
|
||||
// It handles pagination transparently and streams CatalogueEntry values.
|
||||
type CatalogueProvider interface {
|
||||
// CatalogueURL returns the root URL of the catalogue listing.
|
||||
CatalogueURL() string
|
||||
|
||||
// EntriesSelector returns the selector that matches each novel card / row
|
||||
// in the catalogue listing page.
|
||||
EntriesSelector() Selector
|
||||
|
||||
// NextPageSelector returns the selector for the "next page" link.
|
||||
// If the current page has no next page the implementation must return
|
||||
// ("", nil) from ScrapeNextPage.
|
||||
NextPageSelector() Selector
|
||||
|
||||
// ScrapeCatalogue pages through the entire catalogue, sending
|
||||
// CatalogueEntry values to the returned channel. The channel is closed
|
||||
// when all pages have been scraped or ctx is cancelled.
|
||||
// Errors are surfaced via the error channel; a non-nil error does not
|
||||
// necessarily terminate scraping.
|
||||
ScrapeCatalogue(ctx context.Context) (<-chan CatalogueEntry, <-chan error)
|
||||
}
|
||||
|
||||
// MetadataProvider can extract structured book metadata from a novel's landing page.
|
||||
type MetadataProvider interface {
|
||||
// MetadataSelectors returns a map of field name → Selector used to
|
||||
// locate each metadata element on the book page.
|
||||
// Required keys: "title", "author".
|
||||
// Optional keys: "cover", "status", "genres", "summary", "total_chapters".
|
||||
MetadataSelectors() map[string]Selector
|
||||
|
||||
// ScrapeMetadata fetches and parses the metadata for the book at bookURL.
|
||||
ScrapeMetadata(ctx context.Context, bookURL string) (BookMeta, error)
|
||||
}
|
||||
|
||||
// ChapterListProvider can enumerate all chapters of a book from the chapter-list page.
|
||||
type ChapterListProvider interface {
|
||||
// ChaptersURL derives the chapter-list URL from a book landing-page URL.
|
||||
ChaptersURL(bookURL string) string
|
||||
|
||||
// ChapterEntrySelector returns the selector that matches each chapter row
|
||||
// in the chapter list page.
|
||||
ChapterEntrySelector() Selector
|
||||
|
||||
// ScrapeChapterList returns all chapter references for a book, ordered
|
||||
// by chapter number ascending.
|
||||
ScrapeChapterList(ctx context.Context, bookURL string) ([]ChapterRef, error)
|
||||
}
|
||||
|
||||
// ChapterTextProvider can extract the readable text from a single chapter page.
|
||||
type ChapterTextProvider interface {
|
||||
// ChapterTextSelector returns the selector that wraps the chapter body.
|
||||
ChapterTextSelector() Selector
|
||||
|
||||
// ScrapeChapterText fetches chapterURL and returns the chapter text as Markdown.
|
||||
ScrapeChapterText(ctx context.Context, ref ChapterRef) (Chapter, error)
|
||||
}
|
||||
|
||||
// NovelScraper is the full interface that a concrete novel source must implement.
|
||||
// It composes all four provider interfaces.
|
||||
type NovelScraper interface {
|
||||
CatalogueProvider
|
||||
MetadataProvider
|
||||
ChapterListProvider
|
||||
ChapterTextProvider
|
||||
|
||||
// SourceName returns the human-readable name of this scraper, e.g. "novelfire.net".
|
||||
SourceName() string
|
||||
}
|
||||
Reference in New Issue
Block a user