tnypxl/rollup
1
0
Fork 0
mirror of https://github.com/tnypxl/rollup.git synced 2025-12-15 23:13:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

remove old commands

Browse Source
This commit is contained in:
Arik Jones
2024-11-23 13:49:01 -06:00
parent 318951063a
commit eaaa6449b4
2 changed files with 0 additions and 332 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
cmd/root.go
View File

@@ -1,35 +0,0 @@
package cmd
import (
"github.com/spf13/cobra"
config "github.com/tnypxl/rollup/internal/config"
)
var (
configFile string
verbose bool
)
var rootCmd = &cobra.Command{
Use: "rollup",
Short: "Rollup is a tool for combining and processing files",
Long: `Rollup is a versatile tool that can combine and process files in various ways.
Use subcommands to perform specific operations.`,
}
func Execute(conf *config.Config) error {
if conf == nil {
conf = &config.Config{} // Use an empty config if none is provided
}
cfg = conf // Set the cfg variable in cmd/files.go
return rootCmd.Execute()
}
func init() {
rootCmd.PersistentFlags().StringVarP(&configFile, "config", "f", "", "Path to the config file (default: rollup.yml in the current directory)")
rootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "Enable verbose logging")
rootCmd.AddCommand(filesCmd)
rootCmd.AddCommand(webCmd)
rootCmd.AddCommand(generateCmd)
}

297
cmd/web.go
View File

@@ -1,297 +0,0 @@
package cmd
import (
"fmt"
"io"
"log"
"net/url"
"os"
"regexp"
"strings"
"time"
"github.com/spf13/cobra"
"github.com/tnypxl/rollup/internal/config"
"github.com/tnypxl/rollup/internal/scraper"
)
var (
urls []string
outputType string
includeSelector string
excludeSelectors []string
)
var scraperConfig scraper.Config
var webCmd = &cobra.Command{
Use: "web",
Short: "Scrape main content from webpages and convert to Markdown",
Long: `Scrape the main content from one or more webpages, ignoring navigational elements, ads, and other UI aspects. Convert the content to a well-structured Markdown file.`,
RunE: runWeb,
}
func init() {
webCmd.Flags().StringSliceVarP(&urls, "urls", "u", []string{}, "URLs of the webpages to scrape (comma-separated)")
webCmd.Flags().StringVarP(&outputType, "output", "o", "single", "Output type: 'single' for one file, 'separate' for multiple files")
webCmd.Flags().StringVar(&includeSelector, "css", "", "CSS selector to extract specific content")
webCmd.Flags().StringSliceVar(&excludeSelectors, "exclude", []string{}, "CSS selectors to exclude from the extracted content (comma-separated)")
}
func runWeb(cmd *cobra.Command, args []string) error {
scraper.SetupLogger(verbose)
logger := log.New(os.Stdout, "WEB: ", log.LstdFlags)
if !verbose {
logger.SetOutput(io.Discard)
}
logger.Printf("Starting web scraping process with verbose mode: %v", verbose)
scraperConfig.Verbose = verbose
var siteConfigs []scraper.SiteConfig
if len(cfg.Sites) > 0 {
logger.Printf("Using configuration from rollup.yml for %d sites", len(cfg.Sites))
siteConfigs = make([]scraper.SiteConfig, len(cfg.Sites))
for i, site := range cfg.Sites {
siteConfigs[i] = scraper.SiteConfig{
BaseURL: site.BaseURL,
CSSLocator: site.CSSLocator,
ExcludeSelectors: site.ExcludeSelectors,
AllowedPaths: site.AllowedPaths,
ExcludePaths: site.ExcludePaths,
OutputAlias: site.OutputAlias,
PathOverrides: convertPathOverrides(site.PathOverrides),
}
logger.Printf("Site %d configuration: BaseURL=%s, CSSLocator=%s, AllowedPaths=%v",
i+1, site.BaseURL, site.CSSLocator, site.AllowedPaths)
}
} else {
logger.Printf("No sites defined in rollup.yml, falling back to URL-based configuration")
siteConfigs = make([]scraper.SiteConfig, len(urls))
for i, u := range urls {
siteConfigs[i] = scraper.SiteConfig{
BaseURL: u,
CSSLocator: includeSelector,
ExcludeSelectors: excludeSelectors,
}
logger.Printf("URL %d configuration: BaseURL=%s, CSSLocator=%s",
i+1, u, includeSelector)
}
}
if len(siteConfigs) == 0 {
logger.Println("Error: No sites or URLs provided")
return fmt.Errorf("no sites or URLs provided. Use --urls flag with comma-separated URLs or set 'scrape.sites' in the rollup.yml file")
}
// Set default values for rate limiting
defaultRequestsPerSecond := 1.0
defaultBurstLimit := 3
// Use default values if not set in the configuration
requestsPerSecond := defaultRequestsPerSecond
if cfg.RequestsPerSecond != nil {
requestsPerSecond = *cfg.RequestsPerSecond
}
burstLimit := defaultBurstLimit
if cfg.BurstLimit != nil {
burstLimit = *cfg.BurstLimit
}
scraperConfig := scraper.Config{
Sites: siteConfigs,
OutputType: outputType,
Verbose: verbose,
Scrape: scraper.ScrapeConfig{
RequestsPerSecond: requestsPerSecond,
BurstLimit: burstLimit,
},
}
logger.Printf("Scraper configuration: OutputType=%s, RequestsPerSecond=%f, BurstLimit=%d",
outputType, requestsPerSecond, burstLimit)
logger.Println("Starting scraping process")
startTime := time.Now()
progressTicker := time.NewTicker(time.Second)
defer progressTicker.Stop()
done := make(chan bool)
messagePrinted := false
go func() {
for {
select {
case <-progressTicker.C:
if time.Since(startTime) > 5*time.Second && !messagePrinted {
fmt.Print("This is taking a while (hold tight) ")
messagePrinted = true
} else if messagePrinted {
fmt.Print(".")
}
case <-done:
return
}
}
}()
scrapedContent, err := scraper.ScrapeSites(scraperConfig)
done <- true
fmt.Println() // New line after progress indicator
if err != nil {
logger.Printf("Error occurred during scraping: %v", err)
return fmt.Errorf("error scraping content: %v", err)
}
logger.Printf("Scraping completed. Total content scraped: %d", len(scrapedContent))
if outputType == "single" {
logger.Println("Writing content to a single file")
return writeSingleFile(scrapedContent)
} else {
logger.Println("Writing content to multiple files")
return writeMultipleFiles(scrapedContent)
}
}
func writeSingleFile(content map[string]string) error {
outputFile := generateDefaultFilename()
file, err := os.Create(outputFile)
if err != nil {
return fmt.Errorf("error creating output file: %v", err)
}
defer file.Close()
for url, c := range content {
_, err = fmt.Fprintf(file, "# ::: Content from %s\n\n%s\n\n---\n\n", url, c)
if err != nil {
return fmt.Errorf("error writing content to file: %v", err)
}
}
fmt.Printf("Content has been extracted from %d URL(s) and saved to %s\n", len(content), outputFile)
return nil
}
func writeMultipleFiles(content map[string]string) error {
for url, c := range content {
filename, err := getFilenameFromContent(c, url)
if err != nil {
return fmt.Errorf("error generating filename for %s: %v", url, err)
}
file, err := os.Create(filename)
if err != nil {
return fmt.Errorf("error creating output file %s: %v", filename, err)
}
_, err = file.WriteString(fmt.Sprintf("# ::: Content from %s\n\n%s\n", url, c))
if err != nil {
file.Close()
return fmt.Errorf("error writing content to file %s: %v", filename, err)
}
file.Close()
fmt.Printf("Content from %s has been saved to %s\n", url, filename)
}
return nil
}
func generateDefaultFilename() string {
timestamp := time.Now().Format("20060102-150405")
return fmt.Sprintf("web-%s.rollup.md", timestamp)
}
func scrapeURL(urlStr string) (string, error) {
content, err := testExtractAndConvertContent(urlStr)
if err != nil {
return "", err
}
return content, nil
}
var (
testExtractAndConvertContent = extractAndConvertContent
)
func extractAndConvertContent(urlStr string) (string, error) {
content, err := scraper.FetchWebpageContent(urlStr)
if err != nil {
return "", fmt.Errorf("error fetching webpage content: %v", err)
}
if includeSelector != "" {
content, err = scraper.ExtractContentWithCSS(content, includeSelector, excludeSelectors)
if err != nil {
return "", fmt.Errorf("error extracting content with CSS: %v", err)
}
}
markdown, err := scraper.ProcessHTMLContent(content, scraper.Config{})
if err != nil {
return "", fmt.Errorf("error processing HTML content: %v", err)
}
parsedURL, err := url.Parse(urlStr)
if err != nil {
return "", fmt.Errorf("error parsing URL: %v", err)
}
header := fmt.Sprintf("# ::: Content from %s\n\n", parsedURL.String())
return header + markdown + "\n\n", nil
}
func getFilenameFromContent(content, urlStr string) (string, error) {
// Try to extract title from content
titleStart := strings.Index(content, "<title>")
titleEnd := strings.Index(content, "</title>")
if titleStart != -1 && titleEnd != -1 && titleEnd > titleStart {
title := strings.TrimSpace(content[titleStart+7 : titleEnd])
if title != "" {
return sanitizeFilename(title) + ".rollup.md", nil
}
}
// If no title found or title is empty, use the URL
parsedURL, err := url.Parse(urlStr)
if err != nil {
return "", fmt.Errorf("invalid URL: %v", err)
}
if parsedURL.Host == "" {
return "", fmt.Errorf("invalid URL: missing host")
}
filename := parsedURL.Host
if parsedURL.Path != "" && parsedURL.Path != "/" {
filename += strings.TrimSuffix(parsedURL.Path, "/")
}
return sanitizeFilename(filename) + ".rollup.md", nil
}
func sanitizeFilename(name string) string {
// Remove any character that isn't alphanumeric, dash, or underscore
reg := regexp.MustCompile("[^a-zA-Z0-9-_]+")
name = reg.ReplaceAllString(name, "_")
// Trim any leading or trailing underscores
name = strings.Trim(name, "_")
// If the name is empty after sanitization, use a default name
if name == "" {
name = "untitled"
}
return name
}
func convertPathOverrides(configOverrides []config.PathOverride) []scraper.PathOverride {
scraperOverrides := make([]scraper.PathOverride, len(configOverrides))
for i, override := range configOverrides {
scraperOverrides[i] = scraper.PathOverride{
Path: override.Path,
CSSLocator: override.CSSLocator,
ExcludeSelectors: override.ExcludeSelectors,
}
}
return scraperOverrides
}