tnypxl/rollup
1
0
Fork 0
mirror of https://github.com/tnypxl/rollup.git synced 2025-12-15 23:13:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix logging and other issues from preventing scraping

Browse Source
This commit is contained in:
Arik Jones
2024-09-21 15:54:33 -05:00
parent 5482621d99
commit 73116e8d82
9 changed files with 975 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
internal/config/config.go
View File

@@ -15,15 +15,27 @@ type Config struct {
}
type ScrapeConfig struct {
URLs []URLConfig `yaml:"urls"`
OutputType string `yaml:"output_type"`
Sites []SiteConfig `yaml:"sites"`
OutputType string `yaml:"output_type"`
RequestsPerSecond float64 `yaml:"requests_per_second"`
BurstLimit int `yaml:"burst_limit"`
}
type URLConfig struct {
URL string `yaml:"url"`
CSSLocator string `yaml:"css_locator"`
ExcludeSelectors []string `yaml:"exclude_selectors"`
OutputAlias string `yaml:"output_alias"`
type SiteConfig struct {
BaseURL string `yaml:"base_url"`
CSSLocator string `yaml:"css_locator"`
ExcludeSelectors []string `yaml:"exclude_selectors"`
MaxDepth int `yaml:"max_depth"`
AllowedPaths []string `yaml:"allowed_paths"`
ExcludePaths []string `yaml:"exclude_paths"`
OutputAlias string `yaml:"output_alias"`
PathOverrides []PathOverride `yaml:"path_overrides"`
}
type PathOverride struct {
Path string `yaml:"path"`
CSSLocator string `yaml:"css_locator"`
ExcludeSelectors []string `yaml:"exclude_selectors"`
}
func Load(configPath string) (*Config, error) {