tnypxl/rollup
1
0
Fork 0
mirror of https://github.com/tnypxl/rollup.git synced 2025-12-15 23:13:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor: Modify scraper to capture only the main content

Browse Source
This commit is contained in:
Arik Jones (aider)
2024-09-14 15:20:15 -05:00
parent bfd70fd786
commit d0ee666b07
2 changed files with 15 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
internal/scraper/scraper.go
View File

@@ -100,11 +100,20 @@ func FetchWebpageContent(urlStr string) (string, error) {
return "", fmt.Errorf("error waiting for body: %v", err)
}
log.Println("Getting page content")
content, err := page.Content()
log.Println("Getting main content")
content, err := page.InnerHTML("main")
if err != nil {
log.Printf("Error getting page content: %v\n", err)
return "", fmt.Errorf("could not get page content: %v", err)
log.Printf("Error getting main content: %v\n", err)
return "", fmt.Errorf("could not get main content: %v", err)
}
if content == "" {
log.Println("Main content is empty, falling back to body content")
content, err = page.InnerHTML("body")
if err != nil {
log.Printf("Error getting body content: %v\n", err)
return "", fmt.Errorf("could not get body content: %v", err)
}
}
log.Printf("Successfully fetched webpage content (length: %d)\n", len(content))