Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Cloud = false
39 changes: 39 additions & 0 deletions acceptance/cmd/aitools/auto-install/first-run/output.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@

=== first run of a new version: installs for the detected agents, one stderr line

>>> [CLI] jobs list
Installed Databricks AI tools for Claude Code, Cursor. Disable: databricks aitools auto-install disable

>>> print_telemetry
{"command":"jobs_list","exit_code":0,"aitools_install_event":{"agents":["CLAUDE_CODE","CURSOR"],"scope":"GLOBAL","error_category":"TYPE_UNSPECIFIED","agent_results":[{"agent":"CLAUDE_CODE","error_category":"CLI_NOT_ON_PATH"}],"trigger":"AUTO_FIRST_RUN"}}
{"command":"jobs_list","exit_code":0,"aitools_auto_install_event":{"outcome":"INSTALLED","holdout_arm":"TREATMENT","interactive":false}}

>>> cursor_skills
databricks-test

>>> auto_install_state
{
"last_version": "1.30.0",
"id": "treatment-machine"
}

=== second run of the same version: nothing

>>> [CLI] jobs list

>>> print_telemetry

=== uninstall turns it off: the next version installs nothing

>>> [CLI] aitools uninstall
Uninstalled 1 skill.

>>> DATABRICKS_AITOOLS_AUTO_INSTALL_CLI_VERSION=1.31.0 [CLI] jobs list

>>> print_telemetry
{"command":"aitools_uninstall","exit_code":0,"aitools_command_event":{"command":"UNINSTALL","stage":"STARTED","interactive":false}}
{"command":"aitools_uninstall","exit_code":0,"aitools_command_event":{"command":"UNINSTALL","stage":"FINISHED","outcome":"SUCCESS","interactive":false,"scope":"GLOBAL"}}
{"command":"jobs_list","exit_code":0,"aitools_auto_install_event":{"outcome":"SKIPPED_OPTOUT","holdout_arm":"TREATMENT","interactive":false}}

>>> cursor_skills
(no skills)
21 changes: 21 additions & 0 deletions acceptance/cmd/aitools/auto-install/first-run/script
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
use_home home
# Claude Code's config dir exists but its CLI is not on PATH (a desktop or IDE
# install), so it gets skill files, as `aitools install` gives it.
"$mkdir_bin" -p "${USERPROFILE:-$HOME}/.claude"
gate true

title "first run of a new version: installs for the detected agents, one stderr line"
trace $CLI jobs list
trace print_telemetry
trace cursor_skills
trace auto_install_state

title "second run of the same version: nothing"
trace $CLI jobs list
trace print_telemetry

title "uninstall turns it off: the next version installs nothing"
trace $CLI aitools uninstall
trace DATABRICKS_AITOOLS_AUTO_INSTALL_CLI_VERSION=1.31.0 $CLI jobs list
trace print_telemetry
trace cursor_skills
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Cloud = false
44 changes: 44 additions & 0 deletions acceptance/cmd/aitools/auto-install/retries/output.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@

=== a failed attempt is silent and retried on the next run, up to 10 attempts

>>> print_telemetry
9 {"command":"jobs_list","exit_code":0,"aitools_auto_install_event":{"outcome":"FAILED","holdout_arm":"TREATMENT","interactive":false,"error_category":"SKILLS_FETCH_FAILED"}}
9 {"command":"jobs_list","exit_code":0,"aitools_install_event":{"agents":["CURSOR"],"scope":"GLOBAL","error_category":"SKILLS_FETCH_FAILED","trigger":"AUTO_FIRST_RUN"}}

>>> auto_install_state
{
"last_version": "1.30.0",
"attempts": 9,
"installing": true,
"id": "treatment-machine"
}

>>> DATABRICKS_SKILLS_REF=broken-ref [CLI] jobs list

>>> print_telemetry
{"command":"jobs_list","exit_code":0,"aitools_install_event":{"agents":["CURSOR"],"scope":"GLOBAL","error_category":"SKILLS_FETCH_FAILED","trigger":"AUTO_FIRST_RUN"}}
{"command":"jobs_list","exit_code":0,"aitools_auto_install_event":{"outcome":"RETRIES_EXHAUSTED","holdout_arm":"TREATMENT","interactive":false,"error_category":"SKILLS_FETCH_FAILED"}}

=== after 10 attempts the version gets no more

>>> [CLI] jobs list

>>> print_telemetry

>>> cursor_skills
(no skills)

=== a new version starts over

>>> DATABRICKS_AITOOLS_AUTO_INSTALL_CLI_VERSION=1.31.0 [CLI] jobs list
Installed Databricks AI tools for Cursor. Disable: databricks aitools auto-install disable

>>> print_telemetry
{"command":"jobs_list","exit_code":0,"aitools_install_event":{"agents":["CURSOR"],"scope":"GLOBAL","error_category":"TYPE_UNSPECIFIED","trigger":"AUTO_FIRST_RUN"}}
{"command":"jobs_list","exit_code":0,"aitools_auto_install_event":{"outcome":"INSTALLED","holdout_arm":"TREATMENT","interactive":false}}

>>> auto_install_state
{
"last_version": "1.31.0",
"id": "treatment-machine"
}
22 changes: 22 additions & 0 deletions acceptance/cmd/aitools/auto-install/retries/script
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
use_home home
gate true

title "a failed attempt is silent and retried on the next run, up to 10 attempts"
for i in 1 2 3 4 5 6 7 8 9; do
DATABRICKS_SKILLS_REF=broken-ref $CLI jobs list
done
# One line per distinct event with its count (uniq -c pads differently by platform).
trace print_telemetry | sort | uniq -c | sed 's/^ *//'
trace auto_install_state
trace DATABRICKS_SKILLS_REF=broken-ref $CLI jobs list
trace print_telemetry

title "after 10 attempts the version gets no more"
trace $CLI jobs list
trace print_telemetry
trace cursor_skills

title "a new version starts over"
trace DATABRICKS_AITOOLS_AUTO_INSTALL_CLI_VERSION=1.31.0 $CLI jobs list
trace print_telemetry
trace auto_install_state
61 changes: 61 additions & 0 deletions acceptance/cmd/aitools/auto-install/script.prepare
Original file line number Diff line number Diff line change
Expand Up @@ -3,3 +3,64 @@ auto_install_state() {
local path="${USERPROFILE:-$HOME}/.databricks/aitools/auto-install.json"
if [ -f "$path" ]; then cat "$path"; else echo "(no auto-install state)"; fi
}

# The host's own CI flag or opt-out would change every case. CI detection also
# reads the providers' own variables (the SDK's useragent.CiCdProvider list).
unset CI DATABRICKS_AITOOLS_AUTO_INSTALL GITHUB_ACTIONS GITLAB_CI JENKINS_URL TF_BUILD CIRCLECI TRAVIS BITBUCKET_BUILD_NUMBER PROJECT_ID BUILD_ID PROJECT_NUMBER LOCATION CODEBUILD_BUILD_ARN TFC_RUN_ID
# Telemetry is printed by print_telemetry.
unset DATABRICKS_CLI_DISABLE_TELEMETRY DATABRICKS_CLI_TELEMETRY_TIMEOUT

# Test binaries are development builds, which never auto-install. Each case
# names the released version it simulates.
export DATABRICKS_AITOOLS_AUTO_INSTALL_CLI_VERSION=1.30.0

# use_home drops the PATH entries below, so resolve these first.
mkdir_bin=$(command -v mkdir)
rm_bin=$(command -v rm)
ls_bin=$(command -v ls)

# use_home HOME [ID] isolates HOME with Cursor's config dir present, and stores
# the machine id ID (default: treatment-machine, whose holdout bucket is in the
# treatment arm; holdout-machine is in the holdout), so the arm does not
# depend on a random id.
#
# Auto-install acts only on agents whose config directory exists, and installs
# plugins through agent CLIs found on PATH. Keep both inside the test: point
# every agent config dir at the test HOME, and drop every PATH entry holding an
# agent CLI. Windows also finds a command under each PATHEXT extension.
use_home() {
sethome "$1"
unset CLAUDE_CONFIG_DIR PI_CODING_AGENT_DIR GEMINI_CLI_HOME GOOSE_PATH_ROOT
export XDG_CONFIG_HOME="${USERPROFILE:-$HOME}/.config"
export APPDATA="${USERPROFILE:-$HOME}/AppData/Roaming"
local exts="" filtered_path="" dir bin ext keep
if [ -n "${PATHEXT:-}" ]; then exts=$(printf '%s' "$PATHEXT" | tr ';' ':'); fi
local IFS=:
for dir in $PATH; do
keep=1
for bin in claude cursor-agent codex opencode copilot pi gemini goose kiro; do
if [ -x "$dir/$bin" ]; then keep=; fi
for ext in $exts; do
if [ -f "$dir/$bin$ext" ]; then keep=; fi
done
done
if [ -n "$keep" ]; then filtered_path="$filtered_path:$dir"; fi
done
export PATH="${filtered_path#:}"
Comment on lines +39 to +49

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[nit] use_home drops every PATH directory that holds an agent CLI, but print_telemetry needs jq (and print_requests.py needs python3) from PATH. On a machine where jq comes only from the same directory as an agent CLI, every auto-install case fails. An example is Homebrew's /opt/homebrew/bin with an npm-installed claude. Resolve these first, as is done for mkdir/rm/ls. Line 9 also has a run of spaces between CIRCLECI and TRAVIS.

"$mkdir_bin" -p "${USERPROFILE:-$HOME}/.cursor" "${USERPROFILE:-$HOME}/.databricks/aitools"
echo "{\"id\": \"${2:-treatment-machine}\"}" > "${USERPROFILE:-$HOME}/.databricks/aitools/auto-install.json"
}

# gate true|false writes a compatibility manifest into the CLI cache with that
# auto_install flag. A fresh cache file replaces the remote manifest.
gate() {
"$mkdir_bin" -p "$DATABRICKS_CACHE_DIR"
echo "{\"0.0.0\": {\"appkit\": \"0.84.0\", \"skills\": \"latest\", \"auto_install\": $1}}" > "$DATABRICKS_CACHE_DIR/compat-manifest.json"
}

# cursor_skills lists the skills installed for Cursor.
cursor_skills() {
local dir="${USERPROFILE:-$HOME}/.cursor/skills" skills=""
if [ -d "$dir" ]; then skills=$("$ls_bin" "$dir"); fi
echo "${skills:-(no skills)}"
}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Cloud = false
96 changes: 96 additions & 0 deletions acceptance/cmd/aitools/auto-install/skipped/output.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@

=== CI: not installed, reported once for the version

>>> CI=true [CLI] jobs list

>>> print_telemetry
{"command":"jobs_list","exit_code":0,"aitools_auto_install_event":{"outcome":"SKIPPED_CI","holdout_arm":"TREATMENT","interactive":false}}

>>> CI=true [CLI] jobs list

>>> print_telemetry

>>> cursor_skills
(no skills)

=== DATABRICKS_AITOOLS_AUTO_INSTALL=false: not installed

>>> DATABRICKS_AITOOLS_AUTO_INSTALL=false [CLI] jobs list

>>> print_telemetry
{"command":"jobs_list","exit_code":0,"aitools_auto_install_event":{"outcome":"SKIPPED_OPTOUT","holdout_arm":"TREATMENT","interactive":false}}

=== auto-install disable: not installed

>>> [CLI] aitools auto-install disable
Automatic installation of Databricks AI tools is disabled.

>>> [CLI] jobs list

>>> print_telemetry
{"command":"jobs_list","exit_code":0,"aitools_auto_install_event":{"outcome":"SKIPPED_OPTOUT","holdout_arm":"TREATMENT","interactive":false}}

>>> cursor_skills
(no skills)

>>> [CLI] aitools auto-install enable
Automatic installation of Databricks AI tools is enabled.

=== commands that never trigger it

>>> [CLI] version
Databricks CLI v[CLI_VERSION]

>>> [CLI] aitools auto-install enable
Automatic installation of Databricks AI tools is enabled.

>>> print_telemetry

>>> DATABRICKS_CACHE_ENABLED=false [CLI] jobs list

>>> print_telemetry

>>> auto_install_state
{
"last_version": "1.32.0",
"id": "treatment-machine"
}

>>> cursor_skills
(no skills)

=== the gate is off: nothing printed or reported, and the version is settled

>>> [CLI] jobs list

>>> print_telemetry

>>> auto_install_state
{
"last_version": "1.33.0",
"id": "treatment-machine"
}

>>> [CLI] jobs list

>>> print_telemetry

>>> cursor_skills
(no skills)

=== a machine in the holdout: not installed

>>> [CLI] jobs list

>>> print_telemetry
{"command":"jobs_list","exit_code":0,"aitools_auto_install_event":{"outcome":"SKIPPED_HOLDOUT","holdout_arm":"HOLDOUT","interactive":false}}

>>> cursor_skills
(no skills)

=== no agent config directory: nothing to install

>>> [CLI] jobs list

>>> print_telemetry
{"command":"jobs_list","exit_code":0,"aitools_auto_install_event":{"outcome":"SKIPPED_NO_AGENTS","holdout_arm":"TREATMENT","interactive":false}}
58 changes: 58 additions & 0 deletions acceptance/cmd/aitools/auto-install/skipped/script
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
use_home home
gate true

title "CI: not installed, reported once for the version"
trace CI=true $CLI jobs list
trace print_telemetry
trace CI=true $CLI jobs list
trace print_telemetry
trace cursor_skills

export DATABRICKS_AITOOLS_AUTO_INSTALL_CLI_VERSION=1.31.0
title "DATABRICKS_AITOOLS_AUTO_INSTALL=false: not installed"
trace DATABRICKS_AITOOLS_AUTO_INSTALL=false $CLI jobs list
trace print_telemetry

export DATABRICKS_AITOOLS_AUTO_INSTALL_CLI_VERSION=1.32.0
title "auto-install disable: not installed"
trace $CLI aitools auto-install disable
trace $CLI jobs list
trace print_telemetry
trace cursor_skills
trace $CLI aitools auto-install enable

export DATABRICKS_AITOOLS_AUTO_INSTALL_CLI_VERSION=1.33.0
title "commands that never trigger it"
trace $CLI version
trace $CLI aitools auto-install enable
$CLI completion bash &> LOG.completion
$CLI __complete jobs &> LOG.complete
$CLI help jobs &> LOG.help
trace print_telemetry
trace DATABRICKS_CACHE_ENABLED=false $CLI jobs list
trace print_telemetry
trace auto_install_state
trace cursor_skills

title "the gate is off: nothing printed or reported, and the version is settled"
gate false
trace $CLI jobs list
trace print_telemetry
trace auto_install_state
# Turning the gate on reaches this machine on its next new version.
gate true
trace $CLI jobs list
trace print_telemetry
trace cursor_skills

title "a machine in the holdout: not installed"
use_home holdout-home holdout-machine
trace $CLI jobs list
trace print_telemetry
trace cursor_skills

title "no agent config directory: nothing to install"
use_home no-agents-home
"$rm_bin" -r "${USERPROFILE:-$HOME}/.cursor"
trace $CLI jobs list
trace print_telemetry
10 changes: 9 additions & 1 deletion acceptance/cmd/aitools/auto-install/test.toml
Original file line number Diff line number Diff line change
@@ -1,13 +1,17 @@
# Automatic installation of Databricks AI tools on the first run of a new CLI
# version. The mock server replaces raw.githubusercontent.com (skills manifest
# and files).
# and files) and the GitHub API (latest skills release).
Env.DATABRICKS_SKILLS_BASE_URL = "$DATABRICKS_HOST"
Env.DATABRICKS_CLI_GITHUB_API_URL = "$DATABRICKS_HOST"
Env.DATABRICKS_SKILLS_REF = "test-ref"
RecordRequests = true
# Pin it so an opt-out exported in the developer's shell does not change the output.
Env.DATABRICKS_AITOOLS_AUTO_INSTALL = "true"

Ignore = [
"home",
"holdout-home",
"no-agents-home",
]

[[Server]]
Expand Down Expand Up @@ -35,3 +39,7 @@ name: databricks-test

# Test skill
'''

[[Server]]
Pattern = "GET /broken-ref/manifest.json"
Response.StatusCode = 500
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Cloud = false
Loading
Loading