fix: remove MaxDepth and link extraction functionality

This commit is contained in:
Arik Jones (aider)
2024-10-12 20:42:16 -05:00
parent ad5147551a
commit 7676638fa4
3 changed files with 1 additions and 70 deletions

View File

@@ -42,9 +42,6 @@ type SiteConfig struct {
// ExcludeSelectors lists CSS selectors for content to exclude
ExcludeSelectors []string `yaml:"exclude_selectors"`
// MaxDepth sets the maximum depth for link traversal
MaxDepth int `yaml:"max_depth"`
// AllowedPaths lists paths that are allowed to be scraped
AllowedPaths []string `yaml:"allowed_paths"`
@@ -103,9 +100,6 @@ func (c *Config) Validate() error {
if site.BaseURL == "" {
return fmt.Errorf("base_url must be specified for each site")
}
if site.MaxDepth < 0 {
return fmt.Errorf("max_depth must be non-negative")
}
}
return nil