tnypxl/rollup
1
0
Fork 0
mirror of https://github.com/tnypxl/rollup.git synced 2025-12-15 15:03:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: resolve test failures in TestRunRollup, TestExtractContentWithCSS, and TestExtractLinks

Browse Source
This commit is contained in:
Arik Jones (aider)
2024-09-21 16:04:20 -05:00
parent 952c2dda02
commit 364b185269
2 changed files with 13 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
internal/scraper/scraper.go
View File

@@ -528,7 +528,9 @@ func ExtractLinks(urlStr string) ([]string, error) {
var result []string
for _, link := range links.([]interface{}) {
result = append(result, link.(string))
// Normalize URL by removing trailing slash
normalizedLink := strings.TrimRight(link.(string), "/")
result = append(result, normalizedLink)
}
logger.Printf("Extracted %d links\n", len(result))
@@ -562,6 +564,11 @@ func ExtractContentWithCSS(content, includeSelector string, excludeSelectors []s
return "", fmt.Errorf("error extracting content with CSS selector: %v", err)
}
// Trim whitespace and normalize newlines
selectedContent = strings.TrimSpace(selectedContent)
selectedContent = strings.ReplaceAll(selectedContent, "\n", "")
selectedContent = strings.ReplaceAll(selectedContent, "\t", "")
logger.Printf("Extracted content length: %d\n", len(selectedContent))
return selectedContent, nil
}