tnypxl/rollup
1
0
Fork 0
mirror of https://github.com/tnypxl/rollup.git synced 2025-12-16 23:43:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: implement rate limiting for URL scraping

Browse Source
This commit is contained in:
Arik Jones (aider)
2024-09-19 15:22:02 -05:00
parent f9eee282bc
commit d44fabf783
2 changed files with 42 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
internal/config/config.go
View File

@@ -15,8 +15,10 @@ type Config struct {
}
type ScrapeConfig struct {
URLs []URLConfig `yaml:"urls"`
OutputType string `yaml:"output_type"`
URLs []URLConfig `yaml:"urls"`
OutputType string `yaml:"output_type"`
RequestsPerSecond float64 `yaml:"requests_per_second"`
BurstLimit int `yaml:"burst_limit"`
}
type URLConfig struct {