tnypxl/rollup
1
0
Fork 0
mirror of https://github.com/tnypxl/rollup.git synced 2025-12-15 15:03:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: Move HTML to Markdown conversion to scraper.go

Browse Source
This commit is contained in:
Arik Jones (aider)
2024-09-14 20:55:35 -05:00
parent 939cffb55e
commit c1755836b5
2 changed files with 13 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
cmd/web.go
View File

@@ -174,13 +174,9 @@ func extractAndConvertContent(urlStr string) (string, error) {
}
}
// Create a new converter
converter := md.NewConverter("", true, nil)
// Convert HTML to Markdown
markdown, err := converter.ConvertString(content)
markdown, err := scraper.ProcessHTMLContent(content, scraper.Config{})
if err != nil {
return "", fmt.Errorf("error converting HTML to Markdown: %v", err)
return "", fmt.Errorf("error processing HTML content: %v", err)
}
parsedURL, err := url.Parse(urlStr)

20
internal/scraper/scraper.go
View File

@@ -11,7 +11,7 @@ import (
"github.com/PuerkitoBio/goquery"
"github.com/playwright-community/playwright-go"
"github.com/russross/blackfriday/v2"
md "github.com/JohannesKaufmann/html-to-markdown"
)
var logger *log.Logger
@@ -231,16 +231,18 @@ func ProcessHTMLContent(htmlContent string, config Config) (string, error) {
return "", fmt.Errorf("error extracting content: %v", err)
}
markdown := convertToMarkdown(content)
log.Printf("Converted HTML to Markdown (length: %d)\n", len(markdown))
return markdown, nil
// Create a new converter
converter := md.NewConverter("", true, nil)
// Convert HTML to Markdown
markdown, err := converter.ConvertString(content)
if err != nil {
log.Printf("Error converting HTML to Markdown: %v\n", err)
return "", fmt.Errorf("error converting HTML to Markdown: %v", err)
}
func convertToMarkdown(html string) string {
// Use a simple HTML-to-Markdown conversion
markdown := blackfriday.Run([]byte(html),
blackfriday.WithExtensions(blackfriday.CommonExtensions|blackfriday.HardLineBreak))
return string(markdown)
log.Printf("Converted HTML to Markdown (length: %d)\n", len(markdown))
return markdown, nil
}
func scrollPage(page playwright.Page) error {