tnypxl/rollup
1
0
Fork 0
mirror of https://github.com/tnypxl/rollup.git synced 2025-12-15 23:13:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: improve file ignore logic and preserve newlines in extracted content

Browse Source
This commit is contained in:
Arik Jones (aider)
2024-09-22 16:58:53 -05:00
parent 396f092d50
commit 59994c085c
2 changed files with 15 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
internal/scraper/scraper.go
View File

@@ -564,10 +564,15 @@ func ExtractContentWithCSS(content, includeSelector string, excludeSelectors []s
return "", fmt.Errorf("error extracting content with CSS selector: %v", err)
}
// Trim whitespace and normalize newlines
// Trim leading and trailing whitespace, but preserve internal newlines
selectedContent = strings.TrimSpace(selectedContent)
selectedContent = strings.ReplaceAll(selectedContent, "\n", "")
selectedContent = strings.ReplaceAll(selectedContent, "\t", "")
// Normalize newlines
selectedContent = strings.ReplaceAll(selectedContent, "\r\n", "\n")
selectedContent = strings.ReplaceAll(selectedContent, "\r", "\n")
// Remove any leading or trailing newlines
selectedContent = strings.Trim(selectedContent, "\n")
logger.Printf("Extracted content length: %d\n", len(selectedContent))
return selectedContent, nil