feat(tools): Add web_search and fetch_url tools, wire into analyzer agent

agents/analyzer/analyzer.go

@@ -32,6 +32,8 @@ func NewAnalyzer(ctx context.Context, cfg *Config, llm model.LLM) (*Analyzer, er
 		tools.EditFile,      // Make targeted edits to existing files.
 		tools.ExecCommand,   // Run shell commands (build, extract, convert, etc.).
 		tools.SearchFiles,   // Search for text patterns across local files.
+		tools.WebSearch,     // Search the web via Brave Search API.
+		tools.FetchURL,      // Fetch and extract text from a URL.
 	}, &deps)
 	if err != nil {
 		return nil, err

agents/analyzer/prompt.go

@@ -8,7 +8,6 @@ You are a general-purpose agent that performs filesystem and command-line tasks,
 
 Working directory: {work_dir}
 Output path: {output_path}
-Task: {task}
 
 Your available tools:
 - list_dir: Explore directory trees before reading individual files.
@@ -18,17 +17,21 @@ Your available tools:
 - exec_command: Run CLI commands — use for building code, running scripts, extracting text
   from binary documents (e.g. pdftotext, pandoc, unzip), or any other shell task.
 - search_files: Search for text patterns across local files before reading them in full.
+- web_search: Search the internet via the Brave Search API (requires BRAVE_API_KEY env var).
+  Use when the task requires external information not present in local files.
+- fetch_url: Fetch the text content of any URL. Use after web_search to read a full page.
 
 General workflow:
-1. Understand the task from {task} and the files in {work_dir}.
+1. Understand the task from the user message and the files in {work_dir}.
 2. Use list_dir to explore the directory structure first.
 3. Use search_files to locate relevant content before reading files.
 4. Use read_local_file with line ranges; prefer snippets over full-file reads.
 5. For binary documents (PDF, DOCX, etc.), use exec_command to extract text first
    (e.g. "pdftotext", "pandoc --to plain"), then read the extracted output.
 6. Use edit_file for precise, targeted changes — never rewrite a whole file when a
    targeted edit will do.
-7. Write your final result with write_output_file.
+7. For tasks requiring external knowledge, use web_search then fetch_url to read results.
+8. Write your final result with write_output_file.
 
 Document analysis guidance:
 - PDFs: exec_command ["pdftotext", "-layout", "file.pdf", "-"] to extract text.

go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/spf13/cobra v1.10.2
 	github.com/spf13/viper v1.21.0
 	go.yaml.in/yaml/v3 v3.0.4
+	golang.org/x/net v0.50.0
 	google.golang.org/adk v0.6.0
 	google.golang.org/genai v1.50.0
 )
@@ -49,7 +50,6 @@ require (
 	go.opentelemetry.io/otel/metric v1.40.0 // indirect
 	go.opentelemetry.io/otel/trace v1.40.0 // indirect
 	golang.org/x/crypto v0.48.0 // indirect
-	golang.org/x/net v0.50.0 // indirect
 	golang.org/x/sys v0.41.0 // indirect
 	golang.org/x/text v0.34.0 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20260217215200-42d3e9bedb6d // indirect

tools/fetch_url.go

@@ -0,0 +1,139 @@
+package tools
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+	"time"
+	"unicode"
+
+	"golang.org/x/net/html"
+	"google.golang.org/adk/tool"
+	"google.golang.org/adk/tool/functiontool"
+)
+
+const (
+	fetchHTTPTimeout = 20 * time.Second
+	fetchMaxBytes    = 50 * 1024 // 50 KB of extracted text
+)
+
+// FetchURLArgs are the inputs to the fetch_url tool.
+type FetchURLArgs struct {
+	URL string `json:"url"`
+}
+
+// FetchURLResult is returned by the fetch_url tool.
+type FetchURLResult struct {
+	Content     string `json:"content"`
+	ContentType string `json:"content_type"`
+	StatusCode  int    `json:"status_code"`
+	Truncated   bool   `json:"truncated,omitempty"`
+}
+
+// NewFetchURLTool returns a fetch_url function tool that retrieves a URL and
+// returns its text content (HTML is stripped to plain text).
+func NewFetchURLTool() (tool.Tool, error) {
+	t, err := functiontool.New(
+		functiontool.Config{
+			Name: "fetch_url",
+			Description: "Fetch a URL and return its text content. " +
+				"HTML pages are converted to plain text. " +
+				"Use after web_search to read the full content of a search result, " +
+				"or to fetch API documentation, README files, or any web resource.",
+		},
+		fetchURLFunc(),
+	)
+	if err != nil {
+		return nil, fmt.Errorf("create fetch_url tool: %w", err)
+	}
+	return t, nil
+}
+
+func fetchURLFunc() func(tool.Context, FetchURLArgs) (FetchURLResult, error) {
+	return func(_ tool.Context, args FetchURLArgs) (FetchURLResult, error) {
+		if args.URL == "" {
+			return FetchURLResult{}, fmt.Errorf("fetch_url: url is required")
+		}
+
+		ctx, cancel := context.WithTimeout(context.Background(), fetchHTTPTimeout)
+		defer cancel()
+
+		req, err := http.NewRequestWithContext(ctx, http.MethodGet, args.URL, nil)
+		if err != nil {
+			return FetchURLResult{}, fmt.Errorf("fetch_url: build request: %w", err)
+		}
+		req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; agent-cli/1.0)")
+
+		resp, err := http.DefaultClient.Do(req)
+		if err != nil {
+			return FetchURLResult{}, fmt.Errorf("fetch_url: http: %w", err)
+		}
+		defer func() { _ = resp.Body.Close() }()
+
+		ct := resp.Header.Get("Content-Type")
+		body, err := io.ReadAll(io.LimitReader(resp.Body, 1*1024*1024)) // 1 MB raw cap
+		if err != nil {
+			return FetchURLResult{}, fmt.Errorf("fetch_url: read body: %w", err)
+		}
+
+		var text string
+		if strings.Contains(ct, "text/html") {
+			text = extractHTMLText(string(body))
+		} else {
+			text = string(body)
+		}
+
+		truncated := false
+		if len(text) > fetchMaxBytes {
+			text = text[:fetchMaxBytes]
+			truncated = true
+		}
+
+		return FetchURLResult{
+			Content:     text,
+			ContentType: ct,
+			StatusCode:  resp.StatusCode,
+			Truncated:   truncated,
+		}, nil
+	}
+}
+
+// extractHTMLText parses HTML and returns visible text, stripping tags and scripts.
+func extractHTMLText(htmlContent string) string {
+	doc, err := html.Parse(strings.NewReader(htmlContent))
+	if err != nil {
+		// Fall back to stripping angle brackets on parse failure.
+		return strings.Map(func(r rune) rune {
+			if r == '<' || r == '>' {
+				return ' '
+			}
+			return r
+		}, htmlContent)
+	}
+
+	var sb strings.Builder
+	var walk func(*html.Node)
+	walk = func(n *html.Node) {
+		if n.Type == html.ElementNode {
+			// Skip invisible elements.
+			switch n.Data {
+			case "script", "style", "noscript", "head", "meta", "link":
+				return
+			}
+		}
+		if n.Type == html.TextNode {
+			text := strings.TrimFunc(n.Data, unicode.IsSpace)
+			if text != "" {
+				sb.WriteString(text)
+				sb.WriteByte('\n')
+			}
+		}
+		for c := n.FirstChild; c != nil; c = c.NextSibling {
+			walk(c)
+		}
+	}
+	walk(doc)
+	return sb.String()
+}

tools/fetch_url_test.go

@@ -0,0 +1,100 @@
+package tools
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+)
+
+func TestFetchURL_EmptyURL(t *testing.T) {
+	fn := fetchURLFunc()
+	_, err := fn(nil, FetchURLArgs{URL: ""})
+	if err == nil {
+		t.Fatal("want error for empty URL")
+	}
+}
+
+func TestFetchURL_PlainText(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		w.Header().Set("Content-Type", "text/plain")
+		_, _ = w.Write([]byte("hello world"))
+	}))
+	defer srv.Close()
+
+	fn := fetchURLFunc()
+	result, err := fn(nil, FetchURLArgs{URL: srv.URL})
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if !strings.Contains(result.Content, "hello world") {
+		t.Errorf("want 'hello world' in content, got %q", result.Content)
+	}
+	if result.StatusCode != http.StatusOK {
+		t.Errorf("want 200, got %d", result.StatusCode)
+	}
+}
+
+func TestFetchURL_HTMLStripped(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		w.Header().Set("Content-Type", "text/html")
+		_, _ = w.Write([]byte(`<html><head><title>Test</title><script>alert(1)</script></head><body><h1>Hello</h1><p>World</p></body></html>`))
+	}))
+	defer srv.Close()
+
+	fn := fetchURLFunc()
+	result, err := fn(nil, FetchURLArgs{URL: srv.URL})
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if strings.Contains(result.Content, "<h1>") {
+		t.Error("HTML tags should be stripped from content")
+	}
+	if strings.Contains(result.Content, "alert(1)") {
+		t.Error("script content should be removed")
+	}
+	if !strings.Contains(result.Content, "Hello") {
+		t.Errorf("want visible text 'Hello' in content, got %q", result.Content)
+	}
+}
+
+func TestFetchURL_Truncation(t *testing.T) {
+	// Response larger than fetchMaxBytes should be truncated.
+	large := strings.Repeat("x", fetchMaxBytes+1000)
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		w.Header().Set("Content-Type", "text/plain")
+		_, _ = w.Write([]byte(large))
+	}))
+	defer srv.Close()
+
+	fn := fetchURLFunc()
+	result, err := fn(nil, FetchURLArgs{URL: srv.URL})
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if !result.Truncated {
+		t.Error("want Truncated=true for oversized response")
+	}
+	if len(result.Content) > fetchMaxBytes {
+		t.Errorf("content length %d exceeds cap %d", len(result.Content), fetchMaxBytes)
+	}
+}
+
+func TestExtractHTMLText_SkipsScript(t *testing.T) {
+	input := `<html><head><script>var x = 1;</script></head><body><p>visible</p></body></html>`
+	out := extractHTMLText(input)
+	if strings.Contains(out, "var x") {
+		t.Error("script content should not appear in extracted text")
+	}
+	if !strings.Contains(out, "visible") {
+		t.Errorf("want 'visible' in output, got %q", out)
+	}
+}
+
+func TestExtractHTMLText_InvalidHTML(t *testing.T) {
+	// Should not panic on malformed input.
+	out := extractHTMLText("<p>unclosed")
+	if out == "" {
+		t.Error("want non-empty output for partially valid HTML")
+	}
+}

tools/tools.go

@@ -21,6 +21,10 @@ const (
 	ReadLocalFile Kind = "read_local_file"
 	EditFile      Kind = "edit_file"
 	ExecCommand   Kind = "exec_command"
+
+	// Web tools.
+	WebSearch Kind = "web_search"
+	FetchURL  Kind = "fetch_url"
 )
 
 // GetToolByEnum returns the tool.Tool for the given Kind, initialised with any
@@ -54,6 +58,12 @@ func GetToolByEnum(kind Kind, deps *Deps) (tool.Tool, error) {
 	case ExecCommand:
 		return NewExecCommandTool()
 
+	// Web tools
+	case WebSearch:
+		return NewWebSearchTool()
+	case FetchURL:
+		return NewFetchURLTool()
+
 	default:
 		return nil, fmt.Errorf("invalid tool kind: %q", kind)
 	}