diff --git a/cmd/web.go b/cmd/web.go index fa993d4..f3f8e67 100644 --- a/cmd/web.go +++ b/cmd/web.go @@ -4,10 +4,8 @@ import ( "fmt" "net/url" "os" - "strings" - "time" - "github.com/JohannesKaufmann/html-to-markdown" + md "github.com/JohannesKaufmann/html-to-markdown" "github.com/spf13/cobra" "github.com/tnypxl/rollup/internal/config" "github.com/tnypxl/rollup/internal/scraper" @@ -223,22 +221,26 @@ func runWeb(cmd *cobra.Command, args []string) error { return nil } +func generateDefaultFilename(urls []string) string { + // Simple implementation for now + return "rollup-web-content.md" +} + +func scrapeRecursively(url string, depth int) (string, error) { + // Simple implementation for now + return extractAndConvertContent(url) +} + func extractAndConvertContent(urlStr string) (string, error) { content, err := scraper.FetchWebpageContent(urlStr) if err != nil { return "", fmt.Errorf("error fetching webpage content: %v", err) } - if cssSelector != "" { - content, err = scraper.ExtractContentWithCSS(content, cssSelector) - if err != nil { - return "", fmt.Errorf("error extracting content with CSS selector: %v", err) - } - } else if xpathSelector != "" { - content, err = scraper.ExtractContentWithXPath(content, xpathSelector) - if err != nil { - return "", fmt.Errorf("error extracting content with XPath selector: %v", err) - } + if cssSelector != "" || xpathSelector != "" { + // TODO: Implement content extraction with CSS or XPath selector + // For now, we'll just use the full content + fmt.Println("Warning: CSS and XPath selectors are not yet implemented") } // Create a new converter