tnypxl/rollup
1
0
Fork 0
mirror of https://github.com/tnypxl/rollup.git synced 2025-12-15 06:53:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor: Modify scraper to capture only the main content

Browse Source
This commit is contained in:
Arik Jones (aider)
2024-09-14 15:20:15 -05:00
parent bfd70fd786
commit d0ee666b07
2 changed files with 15 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
cmd/web.go
View File

@@ -127,17 +127,8 @@ func extractAndConvertContent(urlStr string) (string, error) {
return "", fmt.Errorf("error fetching webpage content: %v", err)
}
if scraperConfig.CSSLocator != "" {
content, err = scraper.ExtractContentWithCSS(content, scraperConfig.CSSLocator)
if err != nil {
return "", fmt.Errorf("error extracting content with CSS selector: %v", err)
}
} else if xpathSelector != "" {
content, err = scraper.ExtractContentWithXPath(content, xpathSelector)
if err != nil {
return "", fmt.Errorf("error extracting content with XPath selector: %v", err)
}
}
// The content is already extracted using the main element,
// so we don't need to use ExtractContentWithCSS or ExtractContentWithXPath here
// Create a new converter
converter := md.NewConverter("", true, nil)