tnypxl/rollup
1
0
Fork 0
mirror of https://github.com/tnypxl/rollup.git synced 2025-12-16 23:43:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: set default values for requests_per_second and burst_limit in configuration to prevent rate limiter errors

Browse Source
This commit is contained in:
Arik Jones (aider)
2024-09-30 14:19:00 -05:00
parent e3355269b8
commit 01465a08b7
3 changed files with 32 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
cmd/web.go
View File

@@ -36,6 +36,16 @@ func init() {
webCmd.Flags().StringSliceVar(&excludeSelectors, "exclude", []string{}, "CSS selectors to exclude from the extracted content (comma-separated)")
}
func validateScrapeConfig(scrapeConfig config.ScrapeConfig) error {
if scrapeConfig.RequestsPerSecond <= 0 {
return fmt.Errorf("requests_per_second must be greater than 0")
}
if scrapeConfig.BurstLimit <= 0 {
return fmt.Errorf("burst_limit must be greater than 0")
}
return nil
}
func runWeb(cmd *cobra.Command, args []string) error {
scraper.SetupLogger(verbose)
logger := log.New(os.Stdout, "WEB: ", log.LstdFlags)
@@ -97,6 +107,13 @@ func runWeb(cmd *cobra.Command, args []string) error {
logger.Printf("Scraper configuration: OutputType=%s, RequestsPerSecond=%f, BurstLimit=%d",
outputType, scraperConfig.Scrape.RequestsPerSecond, scraperConfig.Scrape.BurstLimit)
// Validate scrape configuration
err := validateScrapeConfig(cfg.Scrape)
if err != nil {
logger.Printf("Invalid scrape configuration: %v", err)
return err
}
// Start scraping using scraper.ScrapeSites
logger.Println("Starting scraping process")
scrapedContent, err := scraper.ScrapeSites(scraperConfig)