mirror of
https://github.com/tnypxl/rollup.git
synced 2025-12-16 15:33:17 +00:00
remove old implementation of the web scraper
This commit is contained in:
Arik Jones
242
cmd/files.go
242
cmd/files.go
@@ -1,242 +0,0 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/tnypxl/rollup/internal/config"
|
||||
)
|
||||
|
||||
var cfg *config.Config
|
||||
|
||||
var (
|
||||
path string
|
||||
fileTypes string
|
||||
codeGenPatterns string
|
||||
ignorePatterns string
|
||||
)
|
||||
|
||||
var filesCmd = &cobra.Command{
|
||||
Use: "files",
|
||||
Short: "Rollup files into a single Markdown file",
|
||||
Long: `The files subcommand writes the contents of all files (with target custom file types provided)
|
||||
in a given project, current path or a custom path, to a single timestamped markdown file
|
||||
whose name is <project-directory-name>-rollup-<timestamp>.md.`,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
return runRollup(cfg)
|
||||
},
|
||||
}
|
||||
|
||||
func init() {
|
||||
filesCmd.Flags().StringVarP(&path, "path", "p", ".", "Path to the project directory")
|
||||
filesCmd.Flags().StringVarP(&fileTypes, "types", "t", ".go,.md,.txt", "Comma-separated list of file extensions to include")
|
||||
filesCmd.Flags().StringVarP(&codeGenPatterns, "codegen", "g", "", "Comma-separated list of glob patterns for code-generated files")
|
||||
filesCmd.Flags().StringVarP(&ignorePatterns, "ignore", "i", "", "Comma-separated list of glob patterns for files to ignore")
|
||||
}
|
||||
|
||||
func matchGlob(pattern, path string) bool {
|
||||
parts := strings.Split(pattern, "/")
|
||||
return matchGlobRecursive(parts, path)
|
||||
}
|
||||
|
||||
func matchGlobRecursive(patternParts []string, path string) bool {
|
||||
if len(patternParts) == 0 {
|
||||
return path == ""
|
||||
}
|
||||
|
||||
if patternParts[0] == "**" {
|
||||
for i := 0; i <= len(path); i++ {
|
||||
if matchGlobRecursive(patternParts[1:], path[i:]) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
i := strings.IndexByte(path, '/')
|
||||
if i < 0 {
|
||||
matched, _ := filepath.Match(patternParts[0], path)
|
||||
return matched && len(patternParts) == 1
|
||||
}
|
||||
|
||||
matched, _ := filepath.Match(patternParts[0], path[:i])
|
||||
return matched && matchGlobRecursive(patternParts[1:], path[i+1:])
|
||||
}
|
||||
|
||||
func isCodeGenerated(filePath string, patterns []string) bool {
|
||||
for _, pattern := range patterns {
|
||||
if strings.Contains(pattern, "**") {
|
||||
if matchGlob(pattern, filePath) {
|
||||
return true
|
||||
}
|
||||
} else {
|
||||
matched, err := filepath.Match(pattern, filepath.Base(filePath))
|
||||
if err == nil && matched {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isIgnored(filePath string, patterns []string) bool {
|
||||
for _, pattern := range patterns {
|
||||
if strings.Contains(pattern, "**") {
|
||||
if matchGlob(pattern, filePath) {
|
||||
return true
|
||||
}
|
||||
} else {
|
||||
// Check if the pattern matches the full path or any part of it
|
||||
if matched, _ := filepath.Match(pattern, filePath); matched {
|
||||
return true
|
||||
}
|
||||
pathParts := strings.Split(filePath, string(os.PathSeparator))
|
||||
for i := range pathParts {
|
||||
partialPath := filepath.Join(pathParts[:i+1]...)
|
||||
if matched, _ := filepath.Match(pattern, partialPath); matched {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func runRollup(cfg *config.Config) error {
|
||||
// Use config if available, otherwise use command-line flags
|
||||
var types []string
|
||||
var codeGenList, ignoreList []string
|
||||
if cfg != nil && len(cfg.FileExtensions) > 0 {
|
||||
types = cfg.FileExtensions
|
||||
} else {
|
||||
types = strings.Split(fileTypes, ",")
|
||||
}
|
||||
if cfg != nil && len(cfg.CodeGeneratedPaths) > 0 {
|
||||
codeGenList = cfg.CodeGeneratedPaths
|
||||
} else {
|
||||
codeGenList = strings.Split(codeGenPatterns, ",")
|
||||
}
|
||||
if cfg != nil && len(cfg.IgnorePaths) > 0 {
|
||||
ignoreList = cfg.IgnorePaths
|
||||
} else {
|
||||
ignoreList = strings.Split(ignorePatterns, ",")
|
||||
}
|
||||
|
||||
// Get the absolute path
|
||||
absPath, err := filepath.Abs(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error getting absolute path: %v", err)
|
||||
}
|
||||
|
||||
// Get the project directory name
|
||||
projectName := filepath.Base(absPath)
|
||||
|
||||
// Generate the output file name
|
||||
timestamp := time.Now().Format("20060102-150405")
|
||||
outputFileName := fmt.Sprintf("%s-%s.rollup.md", projectName, timestamp)
|
||||
|
||||
// Open the output file
|
||||
outputFile, err := os.Create(outputFileName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error creating output file: %v", err)
|
||||
}
|
||||
defer outputFile.Close()
|
||||
|
||||
startTime := time.Now()
|
||||
showProgress := false
|
||||
progressTicker := time.NewTicker(500 * time.Millisecond)
|
||||
defer progressTicker.Stop()
|
||||
|
||||
// Walk through the directory
|
||||
err = filepath.Walk(absPath, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if info.IsDir() {
|
||||
if strings.HasPrefix(info.Name(), ".") {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
return nil
|
||||
}
|
||||
relPath, _ := filepath.Rel(absPath, path)
|
||||
|
||||
// Check if the file should be ignored
|
||||
if isIgnored(relPath, ignoreList) {
|
||||
if verbose {
|
||||
fmt.Printf("Ignoring file: %s\n", relPath)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
ext := filepath.Ext(path)
|
||||
for _, t := range types {
|
||||
if ext == "."+t {
|
||||
// Verbose logging for processed file
|
||||
if verbose {
|
||||
size := humanReadableSize(info.Size())
|
||||
fmt.Printf("Processing file: %s (%s)\n", relPath, size)
|
||||
}
|
||||
|
||||
// Read file contents
|
||||
content, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
fmt.Printf("Error reading file %s: %v\n", path, err)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Check if the file is code-generated
|
||||
isCodeGen := isCodeGenerated(relPath, codeGenList)
|
||||
codeGenNote := ""
|
||||
if isCodeGen {
|
||||
codeGenNote = " (Code-generated, Read-only)"
|
||||
}
|
||||
|
||||
// Write file name and contents to the output file
|
||||
fmt.Fprintf(outputFile, "# File: %s%s\n\n```%s\n%s```\n\n", relPath, codeGenNote, t, string(content))
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if !showProgress && time.Since(startTime) > 5*time.Second {
|
||||
showProgress = true
|
||||
fmt.Print("This is taking a while (hold tight) ")
|
||||
}
|
||||
|
||||
select {
|
||||
case <-progressTicker.C:
|
||||
if showProgress {
|
||||
fmt.Print(".")
|
||||
}
|
||||
default:
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("error walking through directory: %v", err)
|
||||
}
|
||||
|
||||
if showProgress {
|
||||
fmt.Println() // Print a newline after the progress dots
|
||||
}
|
||||
|
||||
fmt.Printf("Rollup complete. Output file: %s\n", outputFileName)
|
||||
return nil
|
||||
}
|
||||
|
||||
func humanReadableSize(size int64) string {
|
||||
const unit = 1024
|
||||
if size < unit {
|
||||
return fmt.Sprintf("%d B", size)
|
||||
}
|
||||
div, exp := int64(unit), 0
|
||||
for n := size / unit; n >= unit; n /= unit {
|
||||
div *= unit
|
||||
exp++
|
||||
}
|
||||
return fmt.Sprintf("%.1f %cB", float64(size)/float64(div), "KMGTPE"[exp])
|
||||
}
|
||||
@@ -1,172 +0,0 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/tnypxl/rollup/internal/config"
|
||||
)
|
||||
|
||||
func TestMatchGlob(t *testing.T) {
|
||||
tests := []struct {
|
||||
pattern string
|
||||
path string
|
||||
expected bool
|
||||
}{
|
||||
{"*.go", "file.go", true},
|
||||
{"*.go", "file.txt", false},
|
||||
{"**/*.go", "dir/file.go", true},
|
||||
{"**/*.go", "dir/subdir/file.go", true},
|
||||
{"dir/*.go", "dir/file.go", true},
|
||||
{"dir/*.go", "otherdir/file.go", false},
|
||||
{"**/test_*.go", "internal/test_helper.go", true},
|
||||
{"docs/**/*.md", "docs/api/endpoints.md", true},
|
||||
{"docs/**/*.md", "src/docs/readme.md", false},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
result := matchGlob(test.pattern, test.path)
|
||||
if result != test.expected {
|
||||
t.Errorf("matchGlob(%q, %q) = %v; want %v", test.pattern, test.path, result, test.expected)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsCodeGenerated(t *testing.T) {
|
||||
patterns := []string{"generated_*.go", "**/auto_*.go", "**/*_gen.go"}
|
||||
tests := []struct {
|
||||
path string
|
||||
expected bool
|
||||
}{
|
||||
{"generated_file.go", true},
|
||||
{"normal_file.go", false},
|
||||
{"subdir/auto_file.go", true},
|
||||
{"subdir/normal_file.go", false},
|
||||
{"pkg/models_gen.go", true},
|
||||
{"pkg/handler.go", false},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
result := isCodeGenerated(test.path, patterns)
|
||||
if result != test.expected {
|
||||
t.Errorf("isCodeGenerated(%q, %v) = %v; want %v", test.path, patterns, result, test.expected)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsIgnored(t *testing.T) {
|
||||
patterns := []string{"*.tmp", "**/*.log", ".git/**", "vendor/**"}
|
||||
tests := []struct {
|
||||
path string
|
||||
expected bool
|
||||
}{
|
||||
{"file.tmp", true},
|
||||
{"file.go", false},
|
||||
{"subdir/file.log", true},
|
||||
{"subdir/file.txt", false},
|
||||
{".git/config", true},
|
||||
{"src/.git/config", false},
|
||||
{"vendor/package/file.go", true},
|
||||
{"internal/vendor/file.go", false},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
result := isIgnored(test.path, patterns)
|
||||
if result != test.expected {
|
||||
t.Errorf("isIgnored(%q, %v) = %v; want %v", test.path, patterns, result, test.expected)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunRollup(t *testing.T) {
|
||||
// Create a temporary directory for testing
|
||||
tempDir, err := os.MkdirTemp("", "rollup_test")
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create temp dir: %v", err)
|
||||
}
|
||||
defer os.RemoveAll(tempDir)
|
||||
|
||||
// Create some test files
|
||||
files := map[string]string{
|
||||
"file1.go": "package main\n\nfunc main() {}\n",
|
||||
"file2.txt": "This is a text file.\n",
|
||||
"subdir/file3.go": "package subdir\n\nfunc Func() {}\n",
|
||||
"subdir/file4.json": "{\"key\": \"value\"}\n",
|
||||
"generated_model.go": "// Code generated DO NOT EDIT.\n\npackage model\n",
|
||||
"docs/api/readme.md": "# API Documentation\n",
|
||||
".git/config": "[core]\n\trepositoryformatversion = 0\n",
|
||||
"vendor/lib/helper.go": "package lib\n\nfunc Helper() {}\n",
|
||||
}
|
||||
|
||||
for name, content := range files {
|
||||
path := filepath.Join(tempDir, name)
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
t.Fatalf("Failed to create directory: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
|
||||
t.Fatalf("Failed to write file: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Set up test configuration
|
||||
cfg = &config.Config{
|
||||
FileExtensions: []string{"go", "txt", "md"},
|
||||
IgnorePaths: []string{"*.json", ".git/**", "vendor/**"},
|
||||
CodeGeneratedPaths: []string{"generated_*.go"},
|
||||
}
|
||||
|
||||
// Change working directory to the temp directory
|
||||
originalWd, _ := os.Getwd()
|
||||
os.Chdir(tempDir)
|
||||
defer os.Chdir(originalWd)
|
||||
|
||||
// Run the rollup
|
||||
if err := runRollup(cfg); err != nil {
|
||||
t.Fatalf("runRollup() failed: %v", err)
|
||||
}
|
||||
|
||||
// Check if the output file was created
|
||||
outputFiles, err := filepath.Glob("*.rollup.md")
|
||||
if err != nil {
|
||||
t.Fatalf("Error globbing for output file: %v", err)
|
||||
}
|
||||
if len(outputFiles) == 0 {
|
||||
allFiles, _ := filepath.Glob("*")
|
||||
t.Fatalf("No rollup.md file found. Files in directory: %v", allFiles)
|
||||
}
|
||||
outputFile := outputFiles[0]
|
||||
|
||||
// Read the content of the output file
|
||||
content, err := os.ReadFile(outputFile)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to read output file: %v", err)
|
||||
}
|
||||
|
||||
// Check if the content includes the expected files
|
||||
expectedContent := []string{
|
||||
"# File: file1.go",
|
||||
"# File: file2.txt",
|
||||
"# File: subdir/file3.go",
|
||||
"# File: docs/api/readme.md",
|
||||
"# File: generated_model.go (Code-generated, Read-only)",
|
||||
}
|
||||
for _, expected := range expectedContent {
|
||||
if !strings.Contains(string(content), expected) {
|
||||
t.Errorf("Output file does not contain expected content: %s", expected)
|
||||
}
|
||||
}
|
||||
|
||||
// Check if the ignored files are not included
|
||||
ignoredContent := []string{
|
||||
"file4.json",
|
||||
".git/config",
|
||||
"vendor/lib/helper.go",
|
||||
}
|
||||
for _, ignored := range ignoredContent {
|
||||
if strings.Contains(string(content), ignored) {
|
||||
t.Errorf("Output file contains ignored file: %s", ignored)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,80 +0,0 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/tnypxl/rollup/internal/config"
|
||||
"gopkg.in/yaml.v2"
|
||||
)
|
||||
|
||||
var generateCmd = &cobra.Command{
|
||||
Use: "generate",
|
||||
Short: "Generate a rollup.yml config file",
|
||||
Long: `Scan the current directory for text and code files and generate a rollup.yml config file based on the found file extensions.`,
|
||||
RunE: runGenerate,
|
||||
}
|
||||
|
||||
func runGenerate(cmd *cobra.Command, args []string) error {
|
||||
fileTypes := make(map[string]bool)
|
||||
err := filepath.Walk(".", func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !info.IsDir() {
|
||||
ext := strings.TrimPrefix(filepath.Ext(path), ".")
|
||||
if isTextFile(ext) {
|
||||
fileTypes[ext] = true
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("error walking the path: %v", err)
|
||||
}
|
||||
|
||||
cfg := config.Config{
|
||||
FileExtensions: make([]string, 0, len(fileTypes)),
|
||||
IgnorePaths: []string{"node_modules/**", "vendor/**", ".git/**"},
|
||||
}
|
||||
|
||||
for ext := range fileTypes {
|
||||
cfg.FileExtensions = append(cfg.FileExtensions, ext)
|
||||
}
|
||||
|
||||
// Sort file types for consistency
|
||||
sort.Strings(cfg.FileExtensions)
|
||||
|
||||
yamlData, err := yaml.Marshal(&cfg)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error marshaling config: %v", err)
|
||||
}
|
||||
|
||||
outputPath := "rollup.yml"
|
||||
err = os.WriteFile(outputPath, yamlData, 0644)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error writing config file: %v", err)
|
||||
}
|
||||
|
||||
fmt.Printf("Generated %s file successfully.\n", outputPath)
|
||||
return nil
|
||||
}
|
||||
|
||||
func isTextFile(ext string) bool {
|
||||
textExtensions := map[string]bool{
|
||||
"txt": true, "md": true, "go": true, "py": true, "js": true, "html": true, "css": true,
|
||||
"json": true, "xml": true, "yaml": true, "yml": true, "toml": true, "ini": true,
|
||||
"sh": true, "bash": true, "zsh": true, "fish": true,
|
||||
"c": true, "cpp": true, "h": true, "hpp": true, "java": true, "kt": true, "scala": true,
|
||||
"rs": true, "rb": true, "php": true, "ts": true, "swift": true,
|
||||
}
|
||||
return textExtensions[ext]
|
||||
}
|
||||
|
||||
func init() {
|
||||
// Add any flags for the generate command here if needed
|
||||
}
|
||||
154
cmd/web_test.go
154
cmd/web_test.go
@@ -1,154 +0,0 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"strings"
|
||||
"github.com/tnypxl/rollup/internal/config"
|
||||
)
|
||||
|
||||
func TestConvertPathOverrides(t *testing.T) {
|
||||
configOverrides := []config.PathOverride{
|
||||
{
|
||||
Path: "/blog",
|
||||
CSSLocator: "article",
|
||||
ExcludeSelectors: []string{".ads", ".comments"},
|
||||
},
|
||||
{
|
||||
Path: "/products",
|
||||
CSSLocator: ".product-description",
|
||||
ExcludeSelectors: []string{".related-items"},
|
||||
},
|
||||
}
|
||||
|
||||
scraperOverrides := convertPathOverrides(configOverrides)
|
||||
|
||||
if len(scraperOverrides) != len(configOverrides) {
|
||||
t.Errorf("Expected %d overrides, got %d", len(configOverrides), len(scraperOverrides))
|
||||
}
|
||||
|
||||
for i, override := range scraperOverrides {
|
||||
if override.Path != configOverrides[i].Path {
|
||||
t.Errorf("Expected Path %s, got %s", configOverrides[i].Path, override.Path)
|
||||
}
|
||||
if override.CSSLocator != configOverrides[i].CSSLocator {
|
||||
t.Errorf("Expected CSSLocator %s, got %s", configOverrides[i].CSSLocator, override.CSSLocator)
|
||||
}
|
||||
if len(override.ExcludeSelectors) != len(configOverrides[i].ExcludeSelectors) {
|
||||
t.Errorf("Expected %d ExcludeSelectors, got %d", len(configOverrides[i].ExcludeSelectors), len(override.ExcludeSelectors))
|
||||
}
|
||||
for j, selector := range override.ExcludeSelectors {
|
||||
if selector != configOverrides[i].ExcludeSelectors[j] {
|
||||
t.Errorf("Expected ExcludeSelector %s, got %s", configOverrides[i].ExcludeSelectors[j], selector)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSanitizeFilename(t *testing.T) {
|
||||
tests := []struct {
|
||||
input string
|
||||
expected string
|
||||
}{
|
||||
{"Hello, World!", "Hello_World"},
|
||||
{"file/with/path", "file_with_path"},
|
||||
{"file.with.dots", "file_with_dots"},
|
||||
{"___leading_underscores___", "leading_underscores"},
|
||||
{"", "untitled"},
|
||||
{"!@#$%^&*()", "untitled"},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
result := sanitizeFilename(test.input)
|
||||
if result != test.expected {
|
||||
t.Errorf("sanitizeFilename(%q) = %q; want %q", test.input, result, test.expected)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetFilenameFromContent(t *testing.T) {
|
||||
tests := []struct {
|
||||
content string
|
||||
url string
|
||||
expected string
|
||||
expectErr bool
|
||||
}{
|
||||
{"<title>Test Page</title>", "http://example.com", "Test_Page.rollup.md", false},
|
||||
{"No title here", "http://example.com/page", "example_com_page.rollup.md", false},
|
||||
{"<title> Trim Me </title>", "http://example.com", "Trim_Me.rollup.md", false},
|
||||
{"<title></title>", "http://example.com", "example_com.rollup.md", false},
|
||||
{"<title> </title>", "http://example.com", "example_com.rollup.md", false},
|
||||
{"Invalid URL", "not a valid url", "", true},
|
||||
{"No host", "http://", "", true},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
result, err := getFilenameFromContent(test.content, test.url)
|
||||
if test.expectErr {
|
||||
if err == nil {
|
||||
t.Errorf("getFilenameFromContent(%q, %q) expected an error, but got none", test.content, test.url)
|
||||
}
|
||||
} else {
|
||||
if err != nil {
|
||||
t.Errorf("getFilenameFromContent(%q, %q) unexpected error: %v", test.content, test.url, err)
|
||||
}
|
||||
if result != test.expected {
|
||||
t.Errorf("getFilenameFromContent(%q, %q) = %q; want %q", test.content, test.url, result, test.expected)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Mock functions for testing
|
||||
func mockExtractAndConvertContent(urlStr string) (string, error) {
|
||||
return "Mocked content for " + urlStr, nil
|
||||
}
|
||||
|
||||
func mockExtractLinks() ([]string, error) {
|
||||
return []string{"http://example.com/link1", "http://example.com/link2"}, nil
|
||||
}
|
||||
|
||||
func TestScrapeURL(t *testing.T) {
|
||||
// Store the original functions
|
||||
originalExtractAndConvertContent := testExtractAndConvertContent
|
||||
originalExtractLinks := testExtractLinks
|
||||
|
||||
// Define mock functions
|
||||
testExtractAndConvertContent = func(urlStr string) (string, error) {
|
||||
return "Mocked content for " + urlStr, nil
|
||||
}
|
||||
testExtractLinks = func(urlStr string) ([]string, error) {
|
||||
return []string{"http://example.com/link1", "http://example.com/link2"}, nil
|
||||
}
|
||||
|
||||
// Defer the restoration of original functions
|
||||
defer func() {
|
||||
testExtractAndConvertContent = originalExtractAndConvertContent
|
||||
testExtractLinks = originalExtractLinks
|
||||
}()
|
||||
|
||||
tests := []struct {
|
||||
url string
|
||||
depth int
|
||||
expectedCalls int
|
||||
}{
|
||||
{"http://example.com", 0, 1},
|
||||
{"http://example.com", 1, 3},
|
||||
{"http://example.com", 2, 3}, // Same as depth 1 because our mock only returns 2 links
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
visited := make(map[string]bool)
|
||||
content, err := scrapeURL(test.url, test.depth, visited)
|
||||
if err != nil {
|
||||
t.Errorf("scrapeURL(%q, %d) returned error: %v", test.url, test.depth, err)
|
||||
continue
|
||||
}
|
||||
if len(visited) != test.expectedCalls {
|
||||
t.Errorf("scrapeURL(%q, %d) made %d calls, expected %d", test.url, test.depth, len(visited), test.expectedCalls)
|
||||
}
|
||||
expectedContent := "Mocked content for " + test.url
|
||||
if !strings.Contains(content, expectedContent) {
|
||||
t.Errorf("scrapeURL(%q, %d) content doesn't contain %q", test.url, test.depth, expectedContent)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user