tnypxl/rollup
1
0
Fork 0
mirror of https://github.com/tnypxl/rollup.git synced 2025-12-15 15:03:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: resolve build errors in cmd/web.go

Browse Source
This commit is contained in:
Arik Jones (aider)
2024-09-14 14:43:18 -05:00
parent 0494d9433f
commit d74213e4ff
1 changed files with 15 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
cmd/web.go
View File

@@ -4,10 +4,8 @@ import (
"fmt" "fmt"
"net/url" "net/url"
"os" "os"
"strings"
"time"
"github.com/JohannesKaufmann/html-to-markdown" md "github.com/JohannesKaufmann/html-to-markdown"
"github.com/spf13/cobra" "github.com/spf13/cobra"
"github.com/tnypxl/rollup/internal/config" "github.com/tnypxl/rollup/internal/config"
"github.com/tnypxl/rollup/internal/scraper" "github.com/tnypxl/rollup/internal/scraper"
@@ -223,22 +221,26 @@ func runWeb(cmd *cobra.Command, args []string) error {
return nil return nil
} }
func generateDefaultFilename(urls []string) string {
// Simple implementation for now
return "rollup-web-content.md"
}
func scrapeRecursively(url string, depth int) (string, error) {
// Simple implementation for now
return extractAndConvertContent(url)
}
func extractAndConvertContent(urlStr string) (string, error) { func extractAndConvertContent(urlStr string) (string, error) {
content, err := scraper.FetchWebpageContent(urlStr) content, err := scraper.FetchWebpageContent(urlStr)
if err != nil { if err != nil {
return "", fmt.Errorf("error fetching webpage content: %v", err) return "", fmt.Errorf("error fetching webpage content: %v", err)
} }
if cssSelector != "" { if cssSelector != "" || xpathSelector != "" {
content, err = scraper.ExtractContentWithCSS(content, cssSelector) // TODO: Implement content extraction with CSS or XPath selector
if err != nil { // For now, we'll just use the full content
return "", fmt.Errorf("error extracting content with CSS selector: %v", err) fmt.Println("Warning: CSS and XPath selectors are not yet implemented")
}
} else if xpathSelector != "" {
content, err = scraper.ExtractContentWithXPath(content, xpathSelector)
if err != nil {
return "", fmt.Errorf("error extracting content with XPath selector: %v", err)
}
} }
// Create a new converter // Create a new converter