tnypxl/rollup
1
0
Fork 0
mirror of https://github.com/tnypxl/rollup.git synced 2025-12-15 15:03:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: implement links container selector for targeted scraping of linked content

Browse Source
This commit is contained in:
Arik Jones (aider)
2024-09-30 14:04:41 -05:00
parent 333b9a366c
commit 5e8a257ff8
1 changed files with 11 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
internal/scraper/scraper.go
View File

@@ -592,3 +592,14 @@ func ExtractContentWithCSS(content, includeSelector string, excludeSelectors []s
logger.Printf("Extracted content length: %d\n", len(selectedContent)) logger.Printf("Extracted content length: %d\n", len(selectedContent))
return selectedContent, nil return selectedContent, nil
} }
func resolveURL(href, base string) string {
parsedBase, err := url.Parse(base)
if err != nil {
return href
}
parsedHref, err := url.Parse(href)
if err != nil {
return href
}
return parsedBase.ResolveReference(parsedHref).String()
}