From 2ab0d74279ab497cea2a38d713a3a44320e66f51 Mon Sep 17 00:00:00 2001 From: Arik Jones Date: Sat, 14 Sep 2024 19:42:38 -0500 Subject: [PATCH] fix: Update scraper to handle empty URLs --- internal/scraper/scraper.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/scraper/scraper.go b/internal/scraper/scraper.go index 14e3819..592fc0b 100644 --- a/internal/scraper/scraper.go +++ b/internal/scraper/scraper.go @@ -212,7 +212,7 @@ func scrollPage(page playwright.Page) error { // ExtractLinks extracts all links from the given URL func ExtractLinks(urlStr string) ([]string, error) { log.Printf("Extracting links from URL: %s\n", urlStr) - + page, err := browser.NewPage() if err != nil { return nil, fmt.Errorf("could not create page: %v", err) @@ -245,7 +245,7 @@ func ExtractLinks(urlStr string) ([]string, error) { // ExtractContentWithCSS extracts content from HTML using a CSS selector func ExtractContentWithCSS(content, includeSelector string, excludeSelectors []string) (string, error) { log.Printf("Extracting content with CSS selector: %s\n", includeSelector) - + doc, err := goquery.NewDocumentFromReader(strings.NewReader(content)) if err != nil { return "", fmt.Errorf("error parsing HTML: %v", err)