feat: implement links container selector for targeted scraping of linked content

This commit is contained in:
Arik Jones (aider)
2024-09-30 14:04:41 -05:00
parent 333b9a366c
commit 5e8a257ff8

View File

@@ -592,3 +592,14 @@ func ExtractContentWithCSS(content, includeSelector string, excludeSelectors []s
logger.Printf("Extracted content length: %d\n", len(selectedContent))
return selectedContent, nil
}
func resolveURL(href, base string) string {
parsedBase, err := url.Parse(base)
if err != nil {
return href
}
parsedHref, err := url.Parse(href)
if err != nil {
return href
}
return parsedBase.ResolveReference(parsedHref).String()
}