From 0f5cb3e5059fcaffb8746ae139764bbda37e2f90 Mon Sep 17 00:00:00 2001
From: "Arik Jones (aider)" <github@tnypxl.com>
Date: Tue, 3 Sep 2024 11:25:15 -0500
Subject: [PATCH] feat: Implement web subcommand to fetch, summarize, and save
 web content

---
 cmd/web.go | 81 ++++++++++++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 75 insertions(+), 6 deletions(-)

diff --git a/cmd/web.go b/cmd/web.go
index d576b18..76d31dc 100644
--- a/cmd/web.go
+++ b/cmd/web.go
@@ -1,18 +1,87 @@
 package cmd
 
 import (
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/anthropics/anthropic-sdk-go"
 	"github.com/spf13/cobra"
 )
 
 var webCmd = &cobra.Command{
-	Use:   "web",
-	Short: "Web-related commands",
-	Long:  `This command is for web-related operations.`,
-	Run: func(cmd *cobra.Command, args []string) {
-		// This is left empty intentionally
-	},
+	Use:   "web <url>",
+	Short: "Fetch and summarize web content",
+	Long:  `This command fetches the content of a web page, summarizes it, and saves it as a markdown file.`,
+	Args:  cobra.ExactArgs(1),
+	Run:   runWeb,
 }
 
 func init() {
 	rootCmd.AddCommand(webCmd)
 }
+
+func runWeb(cmd *cobra.Command, args []string) {
+	url := args[0]
+	content, err := fetchWebContent(url)
+	if err != nil {
+		fmt.Printf("Error fetching web content: %v\n", err)
+		return
+	}
+
+	summary, err := summarizeContent(content)
+	if err != nil {
+		fmt.Printf("Error summarizing content: %v\n", err)
+		return
+	}
+
+	err = saveToMarkdown(url, summary)
+	if err != nil {
+		fmt.Printf("Error saving markdown: %v\n", err)
+		return
+	}
+
+	fmt.Println("Web content summarized and saved successfully.")
+}
+
+func fetchWebContent(url string) (string, error) {
+	resp, err := http.Get(url)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return "", err
+	}
+
+	return string(body), nil
+}
+
+func summarizeContent(content string) (string, error) {
+	client := anthropic.NewClient(os.Getenv("ANTHROPIC_API_KEY"))
+
+	resp, err := client.CreateCompletion(anthropic.CompletionRequest{
+		Model:     "claude-2",
+		Prompt:    fmt.Sprintf("Human: Summarize the following web content in markdown format:\n\n%s\n\nAssistant:", content),
+		MaxTokens: 1000,
+	})
+	if err != nil {
+		return "", err
+	}
+
+	return resp.Completion, nil
+}
+
+func saveToMarkdown(url string, content string) error {
+	pageName := strings.TrimPrefix(strings.TrimPrefix(url, "http://"), "https://")
+	pageName = strings.ReplaceAll(pageName, "/", "-")
+	timestamp := time.Now().Format("20060102150405")
+	filename := fmt.Sprintf("%s-web-rollup-%s.md", pageName, timestamp)
+
+	return ioutil.WriteFile(filename, []byte(content), 0644)
+}