docs: update configuration section in README.md

This commit is contained in:
Arik Jones (aider)
2024-09-22 18:36:17 -05:00
parent d3ff7cb862
commit a57e4a648d

View File

@@ -76,14 +76,27 @@ ignore:
code_generated: code_generated:
- **/generated/** - **/generated/**
scrape: scrape:
urls: sites:
- url: https://example.com - base_url: https://example.com
css_locator: .content css_locator: .content
exclude_selectors: exclude_selectors:
- .ads - .ads
- .navigation - .navigation
max_depth: 2
allowed_paths:
- /blog
- /docs
exclude_paths:
- /admin
output_alias: example output_alias: example
path_overrides:
- path: /special-page
css_locator: .special-content
exclude_selectors:
- .special-ads
output_type: single output_type: single
requests_per_second: 1.0
burst_limit: 3
``` ```
## Examples ## Examples