feat: implement site-based scraping with path overrides

cmd/web.go

@@ -40,36 +40,48 @@ func init() {
 func runWeb(cmd *cobra.Command, args []string) error {
     scraperConfig.Verbose = verbose
 
-	// Use config if available, otherwise use command-line flags
+    var siteConfigs []scraper.SiteConfig
-	var urlConfigs []scraper.URLConfig
+    if len(cfg.Scrape.Sites) > 0 {
-	if len(urls) == 0 && len(cfg.Scrape.URLs) > 0 {
+        siteConfigs = make([]scraper.SiteConfig, len(cfg.Scrape.Sites))
-		urlConfigs = make([]scraper.URLConfig, len(cfg.Scrape.URLs))
+        for i, site := range cfg.Scrape.Sites {
-		for i, u := range cfg.Scrape.URLs {
+            siteConfigs[i] = scraper.SiteConfig{
-			urlConfigs[i] = scraper.URLConfig{
+                BaseURL:          site.BaseURL,
-				URL:              u.URL,
+                CSSLocator:       site.CSSLocator,
-				CSSLocator:       u.CSSLocator,
+                ExcludeSelectors: site.ExcludeSelectors,
-				ExcludeSelectors: u.ExcludeSelectors,
+                MaxDepth:         site.MaxDepth,
-				OutputAlias:      u.OutputAlias,
+                AllowedPaths:     site.AllowedPaths,
+                ExcludePaths:     site.ExcludePaths,
+                OutputAlias:      site.OutputAlias,
+                PathOverrides:    site.PathOverrides,
             }
         }
     } else {
-		urlConfigs = make([]scraper.URLConfig, len(urls))
+        // Fallback to URL-based configuration if no sites are defined
+        siteConfigs = make([]scraper.SiteConfig, len(urls))
         for i, u := range urls {
-			urlConfigs[i] = scraper.URLConfig{URL: u, CSSLocator: includeSelector}
+            siteConfigs[i] = scraper.SiteConfig{
+                BaseURL:          u,
+                CSSLocator:       includeSelector,
+                ExcludeSelectors: excludeSelectors,
+            }
         }
     }
 
-	if len(urlConfigs) == 0 {
+    if len(siteConfigs) == 0 {
-		return fmt.Errorf("no URLs provided. Use --urls flag with comma-separated URLs or set 'scrape.urls' in the rollup.yml file")
+        return fmt.Errorf("no sites or URLs provided. Use --urls flag with comma-separated URLs or set 'scrape.sites' in the rollup.yml file")
     }
 
     scraperConfig := scraper.Config{
-		URLs:       urlConfigs,
+        Sites:      siteConfigs,
-		OutputType: outputType,
+        OutputType: cfg.Scrape.OutputType,
         Verbose:    verbose,
+        Scrape: scraper.ScrapeConfig{
+            RequestsPerSecond: cfg.Scrape.RequestsPerSecond,
+            BurstLimit:        cfg.Scrape.BurstLimit,
+        },
     }
 
-	scrapedContent, err := scraper.ScrapeMultipleURLs(scraperConfig)
+    scrapedContent, err := scraper.ScrapeSites(scraperConfig)
     if err != nil {
         return fmt.Errorf("error scraping content: %v", err)
     }

internal/config/config.go

@@ -15,17 +15,27 @@ type Config struct {
 }
 
 type ScrapeConfig struct {
-	URLs              []URLConfig `yaml:"urls"`
+    Sites             []SiteConfig `yaml:"sites"`
     OutputType        string       `yaml:"output_type"`
     RequestsPerSecond float64      `yaml:"requests_per_second"`
     BurstLimit        int          `yaml:"burst_limit"`
 }
 
-type URLConfig struct {
+type SiteConfig struct {
-	URL              string   `yaml:"url"`
+    BaseURL          string            `yaml:"base_url"`
     CSSLocator       string            `yaml:"css_locator"`
     ExcludeSelectors []string          `yaml:"exclude_selectors"`
+    MaxDepth         int               `yaml:"max_depth"`
+    AllowedPaths     []string          `yaml:"allowed_paths"`
+    ExcludePaths     []string          `yaml:"exclude_paths"`
     OutputAlias      string            `yaml:"output_alias"`
+    PathOverrides    []PathOverride    `yaml:"path_overrides"`
+}
+
+type PathOverride struct {
+    Path             string   `yaml:"path"`
+    CSSLocator       string   `yaml:"css_locator"`
+    ExcludeSelectors []string `yaml:"exclude_selectors"`
 }
 
 func Load(configPath string) (*Config, error) {

internal/scraper/scraper.go

@@ -38,52 +38,22 @@ type ScrapeConfig struct {
 	BurstLimit        int
 }
 
-// ScrapeMultipleURLs scrapes multiple URLs concurrently
+func ScrapeSites(config Config) (map[string]string, error) {
-func ScrapeMultipleURLs(config Config) (map[string]string, error) {
     results := make(chan struct {
         url     string
         content string
         err     error
-	}, len(config.URLs))
+    })
 
-	// Use default values if not specified in the config
+    limiter := rate.NewLimiter(rate.Limit(config.Scrape.RequestsPerSecond), config.Scrape.BurstLimit)
-	requestsPerSecond := 0.5 // Default to 1 request every 2 seconds
-	if config.Scrape.RequestsPerSecond > 0 {
-		requestsPerSecond = config.Scrape.RequestsPerSecond
-	}
-
-	burstLimit := 1 // Default to 1
-	if config.Scrape.BurstLimit > 0 {
-		burstLimit = config.Scrape.BurstLimit
-	}
-
-	// Create a rate limiter based on the configuration
-	limiter := rate.NewLimiter(rate.Limit(requestsPerSecond), burstLimit)
 
     var wg sync.WaitGroup
-	for _, urlConfig := range config.URLs {
+    for _, site := range config.Sites {
         wg.Add(1)
-		go func(cfg URLConfig) {
+        go func(site SiteConfig) {
             defer wg.Done()
-			
+            scrapeSite(site, config, results, limiter)
-			// Wait for rate limiter before making the request
+        }(site)
-			err := limiter.Wait(context.Background())
-			if err != nil {
-				results <- struct {
-					url     string
-					content string
-					err     error
-				}{cfg.URL, "", fmt.Errorf("rate limiter error: %v", err)}
-				return
-			}
-
-			content, err := scrapeURL(cfg)
-			results <- struct {
-				url     string
-				content string
-				err     error
-			}{cfg.URL, content, err}
-		}(urlConfig)
     }
 
     go func() {
@@ -103,14 +73,107 @@ func ScrapeMultipleURLs(config Config) (map[string]string, error) {
     return scrapedContent, nil
 }
 
-func scrapeURL(config URLConfig) (string, error) {
+func scrapeSite(site SiteConfig, config Config, results chan<- struct {
-	content, err := FetchWebpageContent(config.URL)
+    url     string
+    content string
+    err     error
+}, limiter *rate.Limiter) {
+    visited := make(map[string]bool)
+    queue := []string{site.BaseURL}
+
+    for len(queue) > 0 {
+        url := queue[0]
+        queue = queue[1:]
+
+        if visited[url] {
+            continue
+        }
+        visited[url] = true
+
+        if !isAllowedURL(url, site) {
+            continue
+        }
+
+        // Wait for rate limiter before making the request
+        err := limiter.Wait(context.Background())
+        if err != nil {
+            results <- struct {
+                url     string
+                content string
+                err     error
+            }{url, "", fmt.Errorf("rate limiter error: %v", err)}
+            continue
+        }
+
+        cssLocator, excludeSelectors := getOverrides(url, site)
+        content, err := scrapeURL(url, cssLocator, excludeSelectors)
+        results <- struct {
+            url     string
+            content string
+            err     error
+        }{url, content, err}
+
+        if len(visited) < site.MaxDepth {
+            links, _ := ExtractLinks(url)
+            for _, link := range links {
+                if !visited[link] && isAllowedURL(link, site) {
+                    queue = append(queue, link)
+                }
+            }
+        }
+    }
+}
+
+func isAllowedURL(url string, site SiteConfig) bool {
+    parsedURL, err := url.Parse(url)
+    if err != nil {
+        return false
+    }
+
+    baseURL, _ := url.Parse(site.BaseURL)
+    if parsedURL.Host != baseURL.Host {
+        return false
+    }
+
+    path := parsedURL.Path
+    for _, allowedPath := range site.AllowedPaths {
+        if strings.HasPrefix(path, allowedPath) {
+            for _, excludePath := range site.ExcludePaths {
+                if strings.HasPrefix(path, excludePath) {
+                    return false
+                }
+            }
+            return true
+        }
+    }
+
+    return false
+}
+
+func getOverrides(url string, site SiteConfig) (string, []string) {
+    parsedURL, _ := url.Parse(url)
+    path := parsedURL.Path
+
+    for _, override := range site.PathOverrides {
+        if strings.HasPrefix(path, override.Path) {
+            if override.CSSLocator != "" {
+                return override.CSSLocator, override.ExcludeSelectors
+            }
+            return site.CSSLocator, override.ExcludeSelectors
+        }
+    }
+
+    return site.CSSLocator, site.ExcludeSelectors
+}
+
+func scrapeURL(url, cssLocator string, excludeSelectors []string) (string, error) {
+    content, err := FetchWebpageContent(url)
     if err != nil {
         return "", err
     }
 
-	if config.CSSLocator != "" {
+    if cssLocator != "" {
-		content, err = ExtractContentWithCSS(content, config.CSSLocator, config.ExcludeSelectors)
+        content, err = ExtractContentWithCSS(content, cssLocator, excludeSelectors)
         if err != nil {
             return "", err
         }