From eaa7135eab5ec9e62e0e235dc1a7407aefce738e Mon Sep 17 00:00:00 2001 From: "Arik Jones (aider)" Date: Sat, 14 Sep 2024 17:05:05 -0500 Subject: [PATCH] feat: Improve content extraction with fallback to body --- internal/scraper/scraper.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/internal/scraper/scraper.go b/internal/scraper/scraper.go index 85ef93b..14e3819 100644 --- a/internal/scraper/scraper.go +++ b/internal/scraper/scraper.go @@ -253,7 +253,11 @@ func ExtractContentWithCSS(content, includeSelector string, excludeSelectors []s selection := doc.Find(includeSelector) if selection.Length() == 0 { - return "", fmt.Errorf("no content found with CSS selector: %s", includeSelector) + log.Printf("Warning: No content found with CSS selector: %s. Falling back to body content.\n", includeSelector) + selection = doc.Find("body") + if selection.Length() == 0 { + return "", fmt.Errorf("no content found in body") + } } for _, excludeSelector := range excludeSelectors {