flatten scrape config to 'sites:'

* flatten scrape config to 'sites:'. Update unit tests and readme.
* remove check for file_extensions configuration. 
* show progress indication after 5 seconds.
* add documentation to functions
* fix: remove MaxDepth and link extraction functionality
* fix: Remove MaxDepth references from cmd/web.go
This commit is contained in:
tnypxl
2024-10-14 16:09:58 -05:00
committed by GitHub
parent 333b9a366c
commit 02e39baf38
9 changed files with 317 additions and 283 deletions

View File

@@ -20,7 +20,7 @@ func main() {
var err error
if !isHelpCommand {
configPath := config.DefaultConfigPath()
configPath := "rollup.yml"
cfg, err = config.Load(configPath)
if err != nil {
log.Printf("Warning: Failed to load configuration: %v", err)