Merge branch 'claude/fix-bugs-and-gaps-01DvJSzruQh49DU6XK5AykQU'

# Conflicts:
#	internal/config/config.go
#	internal/config/config_test.go

README.md

@@ -1,225 +1,145 @@
 # Rollup
 
-Rollup aggregates the contents of text-based files and webpages into markdown files.
+Rollup aggregates the contents of text-based files and webpages into a markdown file.
 
 ## Features
 
-- **File aggregation**: Combine multiple source files into a single markdown document
+- File type filtering for targeted content aggregation
-- **File type filtering**: Include only specific file extensions
+- Ignore patterns for excluding specific files or directories
-- **Ignore patterns**: Exclude files/directories using glob patterns
+- Support for code-generated file detection and exclusion
-- **Code-generated file detection**: Mark auto-generated files as read-only in output
+- Advanced web scraping functionality with depth control
-- **Web scraping**: Scrape webpage content using Playwright browser automation
+- Verbose logging option for detailed operation insights
-- **HTML to Markdown conversion**: Automatically converts scraped HTML to clean markdown
+- Exclusionary CSS selectors for precise web content extraction
-- **CSS selectors**: Extract specific content sections or exclude unwanted elements
+- Support for multiple URLs in web scraping operations
-- **Path-based overrides**: Configure different selectors for specific URL paths
+- Configurable output format for web scraping (single file or separate files)
-- **Rate limiting**: Configurable requests per second and burst limits for web scraping
+- Flexible configuration file support (YAML)
-- **Output modes**: Single combined file or separate files per source
+- Automatic generation of default configuration file
-- **Verbose logging**: Detailed operation insights for debugging
+- Custom output file naming
-- **YAML configuration**: Flexible configuration file support
+- Rate limiting for web scraping to respect server resources
 
 ## Installation
 
-Ensure you have Go 1.21+ installed, then run:
+To install Rollup, make sure you have Go installed on your system, then run:
 
 ```bash
-go install github.com/tnypxl/rollup@latest
+go get github.com/tnypxl/rollup
-```
-
-Or build from source:
-
-```bash
-git clone https://github.com/tnypxl/rollup.git
-cd rollup
-go build -o rollup .
 ```
 
 ## Usage
 
+Basic usage:
+
 ```bash
 rollup [command] [flags]
 ```
 
 ### Commands
 
-| Command | Description |
+- `rollup files`: Rollup files into a single Markdown file
-|---------|-------------|
+- `rollup web`: Scrape main content from webpages and convert to Markdown
-| `files` | Aggregate local files into a single markdown file |
+- `rollup generate`: Generate a rollup.yml config file
-| `web` | Scrape webpages and convert to markdown |
-| `generate` | Generate a default rollup.yml config file |
 
 ### Flags for `files` command
 
-| Flag | Short | Default | Description |
+- `--path, -p`: Path to the project directory (default: current directory)
-|------|-------|---------|-------------|
+- `--types, -t`: Comma-separated list of file extensions to include (default: .go,.md,.txt)
-| `--path` | `-p` | `.` | Path to the project directory |
+- `--codegen, -g`: Comma-separated list of glob patterns for code-generated files
-| `--types` | `-t` | `go,md,txt` | Comma-separated list of file extensions (without dots) |
+- `--ignore, -i`: Comma-separated list of glob patterns for files to ignore
-| `--codegen` | `-g` | | Glob patterns for code-generated files |
-| `--ignore` | `-i` | | Glob patterns for files to ignore |
 
 ### Flags for `web` command
 
-| Flag | Short | Description |
+- `--urls, -u`: URLs of the webpages to scrape (comma-separated)
-|------|-------|-------------|
+- `--output, -o`: Output type: 'single' for one file, 'separate' for multiple files (default: single)
-| `--urls` | `-u` | URLs of webpages to scrape (comma-separated) |
+- `--depth, -d`: Depth of link traversal (default: 0, only scrape the given URLs)
-| `--output` | `-o` | Output type: `single` or `separate` |
+- `--css`: CSS selector to extract specific content
-| `--css` | | CSS selector to extract specific content |
+- `--exclude`: CSS selectors to exclude from the extracted content (comma-separated)
-| `--exclude` | | CSS selectors to exclude (comma-separated) |
 
 ### Global flags
 
-| Flag | Short | Description |
+- `--config, -f`: Path to the configuration file (default: rollup.yml in the current directory)
-|------|-------|-------------|
+- `--verbose, -v`: Enable verbose logging
-| `--config` | `-f` | Path to config file (default: `rollup.yml`) |
-| `--verbose` | `-v` | Enable verbose logging |
 
 ## Configuration
 
-Rollup reads from `rollup.yml` by default. Use `--config` to specify a different file.
+Rollup can be configured using a YAML file. By default, it looks for `rollup.yml` in the current directory. You can specify a different configuration file using the `--config` flag.
 
-### Configuration Options
+Example `rollup.yml`:
 
 ```yaml
-# File extensions to include (without leading dots)
 file_extensions:
   - go
   - md
-  - js
-
-# Glob patterns for paths to ignore
 ignore_paths:
   - node_modules/**
   - vendor/**
   - .git/**
-
-# Glob patterns for code-generated files (marked as read-only in output)
 code_generated_paths:
-  - "**/*.pb.go"
+  - **/generated/**
-  - "**/generated/**"
-
-# Web scraping site configurations
 sites:
   - base_url: https://example.com
-    css_locator: .main-content
+    css_locator: .content
     exclude_selectors:
       - .ads
       - .navigation
-      - footer
+    max_depth: 2
     allowed_paths:
-      - /docs
       - /blog
+      - /docs
     exclude_paths:
       - /admin
-    file_name_prefix: example-docs
+    output_alias: example
     path_overrides:
       - path: /special-page
         css_locator: .special-content
         exclude_selectors:
           - .special-ads
-
-# Output type for web scraping: 'single' or 'separate'
 output_type: single
-
-# Rate limiting for web requests
 requests_per_second: 1.0
 burst_limit: 3
 ```
 
-### Configuration Reference
-
-| Field | Type | Description |
-|-------|------|-------------|
-| `file_extensions` | list | File extensions to include in file rollup |
-| `ignore_paths` | list | Glob patterns for files/directories to skip |
-| `code_generated_paths` | list | Glob patterns for auto-generated files |
-| `sites` | list | Web scraping target configurations |
-| `output_type` | string | `single` (one file) or `separate` (multiple files) |
-| `requests_per_second` | float | Rate limit for web requests (default: 1.0) |
-| `burst_limit` | int | Maximum burst size for rate limiting (default: 3) |
-
-#### Site Configuration
-
-| Field | Type | Description |
-|-------|------|-------------|
-| `base_url` | string | Starting URL for scraping (required) |
-| `css_locator` | string | CSS selector for content extraction |
-| `exclude_selectors` | list | CSS selectors for content to exclude |
-| `allowed_paths` | list | URL paths allowed for scraping |
-| `exclude_paths` | list | URL paths to skip |
-| `file_name_prefix` | string | Prefix for output file names |
-| `path_overrides` | list | Path-specific selector overrides |
-
 ## Examples
 
-### File Aggregation
+1. Rollup files with default configuration:
 
-```bash
+   ```bash
-# Rollup files using config file
+   rollup files
-rollup files
+   ```
 
-# Specify file types and ignore patterns
+2. Web scraping with multiple URLs:
-rollup files --types=go,js,ts --ignore="vendor/**,*_test.go"
 
-# Rollup a specific directory
+   ```bash
-rollup files --path=/path/to/project
+   rollup web --urls=https://example.com,https://another-example.com
-```
+   ```
 
-### Web Scraping
+3. Generate a default configuration file:
 
-```bash
+   ```bash
-# Scrape URLs from command line
+   rollup generate
-rollup web --urls=https://example.com/docs
+   ```
 
-# Scrape multiple URLs
+4. Use a custom configuration file:
-rollup web --urls=https://example.com,https://another.com
 
-# Extract specific content with CSS selector
+   ```bash
-rollup web --urls=https://example.com --css=".article-content"
+   rollup files --config=my-config.yml
+   ```
 
-# Exclude elements from scraped content
+5. Web scraping with separate output files:
-rollup web --urls=https://example.com --css=".content" --exclude=".ads,.sidebar"
 
-# Output to separate files
+   ```bash
-rollup web --urls=https://example.com --output=separate
+   rollup web --urls=https://example.com,https://another-example.com --output=separate
-```
+   ```
 
-### Configuration Generation
+6. Rollup files with specific types and ignore patterns:
 
-```bash
+   ```bash
-# Generate rollup.yml based on files in current directory
+   rollup files --types=go,md --ignore=vendor/**,*_test.go
-rollup generate
+   ```
-```
 
-### Using Custom Config
+7. Web scraping with depth and CSS selector:
-
+   ```bash
-```bash
+   rollup web --urls=https://example.com --depth=2 --css=.main-content
-rollup files --config=my-config.yml
+   ```
-rollup web --config=my-config.yml
-```
-
-## Output
-
-### File Rollup Output
-
-The `files` command generates a markdown file named `<project-name>-<timestamp>.rollup.md` containing all matched files:
-
-```markdown
-# File: src/main.go
-
-​```go
-package main
-// ... file contents
-​```
-
-# File: docs/README.md (Code-generated, Read-only)
-
-​```md
-// ... file contents
-​```
-```
-
-### Web Rollup Output
-
-The `web` command generates markdown files from scraped content, with filenames based on the page title or URL.
 
 ## Contributing
 

cmd/web.go

@@ -29,8 +29,6 @@ var webCmd = &cobra.Command{
 	Short: "Scrape main content from webpages and convert to Markdown",
 	Long:  `Scrape the main content from one or more webpages, ignoring navigational elements, ads, and other UI aspects. Convert the content to a well-structured Markdown file.`,
 	PreRunE: func(cmd *cobra.Command, args []string) error {
-		// Setup logger before initializing Playwright
-		scraper.SetupLogger(verbose)
 		// Initialize Playwright for web scraping
 		if err := scraper.InitPlaywright(); err != nil {
 			return fmt.Errorf("failed to initialize Playwright: %w", err)
@@ -85,7 +83,6 @@ func runWeb(cmd *cobra.Command, args []string) error {
 				BaseURL:          u,
 				CSSLocator:       includeSelector,
 				ExcludeSelectors: excludeSelectors,
-				AllowedPaths:     []string{""},
 			}
 			logger.Printf("URL %d configuration: BaseURL=%s, CSSLocator=%s",
 				i+1, u, includeSelector)