tnypxl/rollup
1
0
Fork 0
mirror of https://github.com/tnypxl/rollup.git synced 2025-12-16 23:43:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: set default values for requests_per_second and burst_limit in configuration to prevent rate limiter errors

Browse Source
This commit is contained in:
Arik Jones (aider)
2024-09-30 14:19:00 -05:00
parent e3355269b8
commit 01465a08b7
3 changed files with 32 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
internal/scraper/scraper.go
View File

@@ -68,6 +68,14 @@ func ScrapeSites(config Config) (map[string]string, error) {
err error
})
// Ensure RequestsPerSecond and BurstLimit are valid
if config.Scrape.RequestsPerSecond <= 0 {
config.Scrape.RequestsPerSecond = 1.0
}
if config.Scrape.BurstLimit <= 0 {
config.Scrape.BurstLimit = 5
}
limiter := rate.NewLimiter(rate.Limit(config.Scrape.RequestsPerSecond), config.Scrape.BurstLimit)
logger.Printf("Rate limiter configured with %f requests per second and burst limit of %d\n", config.Scrape.RequestsPerSecond, config.Scrape.BurstLimit)