diff --git a/acceptance/cmd/aitools/auto-install/first-run/out.test.toml b/acceptance/cmd/aitools/auto-install/first-run/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/cmd/aitools/auto-install/first-run/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/cmd/aitools/auto-install/first-run/output.txt b/acceptance/cmd/aitools/auto-install/first-run/output.txt new file mode 100644 index 00000000000..3a85c3b02e8 --- /dev/null +++ b/acceptance/cmd/aitools/auto-install/first-run/output.txt @@ -0,0 +1,39 @@ + +=== first run of a new version: installs for the detected agents, one stderr line + +>>> [CLI] jobs list +Installed Databricks AI tools for Claude Code, Cursor. Disable: databricks aitools auto-install disable + +>>> print_telemetry +{"command":"jobs_list","exit_code":0,"aitools_install_event":{"agents":["CLAUDE_CODE","CURSOR"],"scope":"GLOBAL","error_category":"TYPE_UNSPECIFIED","agent_results":[{"agent":"CLAUDE_CODE","error_category":"CLI_NOT_ON_PATH"}],"trigger":"AUTO_FIRST_RUN"}} +{"command":"jobs_list","exit_code":0,"aitools_auto_install_event":{"outcome":"INSTALLED","holdout_arm":"TREATMENT","interactive":false}} + +>>> cursor_skills +databricks-test + +>>> auto_install_state +{ + "last_version": "1.30.0", + "id": "treatment-machine" +} + +=== second run of the same version: nothing + +>>> [CLI] jobs list + +>>> print_telemetry + +=== uninstall turns it off: the next version installs nothing + +>>> [CLI] aitools uninstall +Uninstalled 1 skill. + +>>> DATABRICKS_AITOOLS_AUTO_INSTALL_CLI_VERSION=1.31.0 [CLI] jobs list + +>>> print_telemetry +{"command":"aitools_uninstall","exit_code":0,"aitools_command_event":{"command":"UNINSTALL","stage":"STARTED","interactive":false}} +{"command":"aitools_uninstall","exit_code":0,"aitools_command_event":{"command":"UNINSTALL","stage":"FINISHED","outcome":"SUCCESS","interactive":false,"scope":"GLOBAL"}} +{"command":"jobs_list","exit_code":0,"aitools_auto_install_event":{"outcome":"SKIPPED_OPTOUT","holdout_arm":"TREATMENT","interactive":false}} + +>>> cursor_skills +(no skills) diff --git a/acceptance/cmd/aitools/auto-install/first-run/script b/acceptance/cmd/aitools/auto-install/first-run/script new file mode 100644 index 00000000000..383f46fc466 --- /dev/null +++ b/acceptance/cmd/aitools/auto-install/first-run/script @@ -0,0 +1,21 @@ +use_home home +# Claude Code's config dir exists but its CLI is not on PATH (a desktop or IDE +# install), so it gets skill files, as `aitools install` gives it. +"$mkdir_bin" -p "${USERPROFILE:-$HOME}/.claude" +gate true + +title "first run of a new version: installs for the detected agents, one stderr line" +trace $CLI jobs list +trace print_telemetry +trace cursor_skills +trace auto_install_state + +title "second run of the same version: nothing" +trace $CLI jobs list +trace print_telemetry + +title "uninstall turns it off: the next version installs nothing" +trace $CLI aitools uninstall +trace DATABRICKS_AITOOLS_AUTO_INSTALL_CLI_VERSION=1.31.0 $CLI jobs list +trace print_telemetry +trace cursor_skills diff --git a/acceptance/cmd/aitools/auto-install/retries/out.test.toml b/acceptance/cmd/aitools/auto-install/retries/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/cmd/aitools/auto-install/retries/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/cmd/aitools/auto-install/retries/output.txt b/acceptance/cmd/aitools/auto-install/retries/output.txt new file mode 100644 index 00000000000..8bd4aa922db --- /dev/null +++ b/acceptance/cmd/aitools/auto-install/retries/output.txt @@ -0,0 +1,44 @@ + +=== a failed attempt is silent and retried on the next run, up to 10 attempts + +>>> print_telemetry +9 {"command":"jobs_list","exit_code":0,"aitools_auto_install_event":{"outcome":"FAILED","holdout_arm":"TREATMENT","interactive":false,"error_category":"SKILLS_FETCH_FAILED"}} +9 {"command":"jobs_list","exit_code":0,"aitools_install_event":{"agents":["CURSOR"],"scope":"GLOBAL","error_category":"SKILLS_FETCH_FAILED","trigger":"AUTO_FIRST_RUN"}} + +>>> auto_install_state +{ + "last_version": "1.30.0", + "attempts": 9, + "installing": true, + "id": "treatment-machine" +} + +>>> DATABRICKS_SKILLS_REF=broken-ref [CLI] jobs list + +>>> print_telemetry +{"command":"jobs_list","exit_code":0,"aitools_install_event":{"agents":["CURSOR"],"scope":"GLOBAL","error_category":"SKILLS_FETCH_FAILED","trigger":"AUTO_FIRST_RUN"}} +{"command":"jobs_list","exit_code":0,"aitools_auto_install_event":{"outcome":"RETRIES_EXHAUSTED","holdout_arm":"TREATMENT","interactive":false,"error_category":"SKILLS_FETCH_FAILED"}} + +=== after 10 attempts the version gets no more + +>>> [CLI] jobs list + +>>> print_telemetry + +>>> cursor_skills +(no skills) + +=== a new version starts over + +>>> DATABRICKS_AITOOLS_AUTO_INSTALL_CLI_VERSION=1.31.0 [CLI] jobs list +Installed Databricks AI tools for Cursor. Disable: databricks aitools auto-install disable + +>>> print_telemetry +{"command":"jobs_list","exit_code":0,"aitools_install_event":{"agents":["CURSOR"],"scope":"GLOBAL","error_category":"TYPE_UNSPECIFIED","trigger":"AUTO_FIRST_RUN"}} +{"command":"jobs_list","exit_code":0,"aitools_auto_install_event":{"outcome":"INSTALLED","holdout_arm":"TREATMENT","interactive":false}} + +>>> auto_install_state +{ + "last_version": "1.31.0", + "id": "treatment-machine" +} diff --git a/acceptance/cmd/aitools/auto-install/retries/script b/acceptance/cmd/aitools/auto-install/retries/script new file mode 100644 index 00000000000..e987c9f90c4 --- /dev/null +++ b/acceptance/cmd/aitools/auto-install/retries/script @@ -0,0 +1,22 @@ +use_home home +gate true + +title "a failed attempt is silent and retried on the next run, up to 10 attempts" +for i in 1 2 3 4 5 6 7 8 9; do + DATABRICKS_SKILLS_REF=broken-ref $CLI jobs list +done +# One line per distinct event with its count (uniq -c pads differently by platform). +trace print_telemetry | sort | uniq -c | sed 's/^ *//' +trace auto_install_state +trace DATABRICKS_SKILLS_REF=broken-ref $CLI jobs list +trace print_telemetry + +title "after 10 attempts the version gets no more" +trace $CLI jobs list +trace print_telemetry +trace cursor_skills + +title "a new version starts over" +trace DATABRICKS_AITOOLS_AUTO_INSTALL_CLI_VERSION=1.31.0 $CLI jobs list +trace print_telemetry +trace auto_install_state diff --git a/acceptance/cmd/aitools/auto-install/script.prepare b/acceptance/cmd/aitools/auto-install/script.prepare index 8f64b39a860..8a130b71809 100644 --- a/acceptance/cmd/aitools/auto-install/script.prepare +++ b/acceptance/cmd/aitools/auto-install/script.prepare @@ -3,3 +3,64 @@ auto_install_state() { local path="${USERPROFILE:-$HOME}/.databricks/aitools/auto-install.json" if [ -f "$path" ]; then cat "$path"; else echo "(no auto-install state)"; fi } + +# The host's own CI flag or opt-out would change every case. CI detection also +# reads the providers' own variables (the SDK's useragent.CiCdProvider list). +unset CI DATABRICKS_AITOOLS_AUTO_INSTALL GITHUB_ACTIONS GITLAB_CI JENKINS_URL TF_BUILD CIRCLECI TRAVIS BITBUCKET_BUILD_NUMBER PROJECT_ID BUILD_ID PROJECT_NUMBER LOCATION CODEBUILD_BUILD_ARN TFC_RUN_ID +# Telemetry is printed by print_telemetry. +unset DATABRICKS_CLI_DISABLE_TELEMETRY DATABRICKS_CLI_TELEMETRY_TIMEOUT + +# Test binaries are development builds, which never auto-install. Each case +# names the released version it simulates. +export DATABRICKS_AITOOLS_AUTO_INSTALL_CLI_VERSION=1.30.0 + +# use_home drops the PATH entries below, so resolve these first. +mkdir_bin=$(command -v mkdir) +rm_bin=$(command -v rm) +ls_bin=$(command -v ls) + +# use_home HOME [ID] isolates HOME with Cursor's config dir present, and stores +# the machine id ID (default: treatment-machine, whose holdout bucket is in the +# treatment arm; holdout-machine is in the holdout), so the arm does not +# depend on a random id. +# +# Auto-install acts only on agents whose config directory exists, and installs +# plugins through agent CLIs found on PATH. Keep both inside the test: point +# every agent config dir at the test HOME, and drop every PATH entry holding an +# agent CLI. Windows also finds a command under each PATHEXT extension. +use_home() { + sethome "$1" + unset CLAUDE_CONFIG_DIR PI_CODING_AGENT_DIR GEMINI_CLI_HOME GOOSE_PATH_ROOT + export XDG_CONFIG_HOME="${USERPROFILE:-$HOME}/.config" + export APPDATA="${USERPROFILE:-$HOME}/AppData/Roaming" + local exts="" filtered_path="" dir bin ext keep + if [ -n "${PATHEXT:-}" ]; then exts=$(printf '%s' "$PATHEXT" | tr ';' ':'); fi + local IFS=: + for dir in $PATH; do + keep=1 + for bin in claude cursor-agent codex opencode copilot pi gemini goose kiro; do + if [ -x "$dir/$bin" ]; then keep=; fi + for ext in $exts; do + if [ -f "$dir/$bin$ext" ]; then keep=; fi + done + done + if [ -n "$keep" ]; then filtered_path="$filtered_path:$dir"; fi + done + export PATH="${filtered_path#:}" + "$mkdir_bin" -p "${USERPROFILE:-$HOME}/.cursor" "${USERPROFILE:-$HOME}/.databricks/aitools" + echo "{\"id\": \"${2:-treatment-machine}\"}" > "${USERPROFILE:-$HOME}/.databricks/aitools/auto-install.json" +} + +# gate true|false writes a compatibility manifest into the CLI cache with that +# auto_install flag. A fresh cache file replaces the remote manifest. +gate() { + "$mkdir_bin" -p "$DATABRICKS_CACHE_DIR" + echo "{\"0.0.0\": {\"appkit\": \"0.84.0\", \"skills\": \"latest\", \"auto_install\": $1}}" > "$DATABRICKS_CACHE_DIR/compat-manifest.json" +} + +# cursor_skills lists the skills installed for Cursor. +cursor_skills() { + local dir="${USERPROFILE:-$HOME}/.cursor/skills" skills="" + if [ -d "$dir" ]; then skills=$("$ls_bin" "$dir"); fi + echo "${skills:-(no skills)}" +} diff --git a/acceptance/cmd/aitools/auto-install/skipped/out.test.toml b/acceptance/cmd/aitools/auto-install/skipped/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/cmd/aitools/auto-install/skipped/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/cmd/aitools/auto-install/skipped/output.txt b/acceptance/cmd/aitools/auto-install/skipped/output.txt new file mode 100644 index 00000000000..55321b2a457 --- /dev/null +++ b/acceptance/cmd/aitools/auto-install/skipped/output.txt @@ -0,0 +1,96 @@ + +=== CI: not installed, reported once for the version + +>>> CI=true [CLI] jobs list + +>>> print_telemetry +{"command":"jobs_list","exit_code":0,"aitools_auto_install_event":{"outcome":"SKIPPED_CI","holdout_arm":"TREATMENT","interactive":false}} + +>>> CI=true [CLI] jobs list + +>>> print_telemetry + +>>> cursor_skills +(no skills) + +=== DATABRICKS_AITOOLS_AUTO_INSTALL=false: not installed + +>>> DATABRICKS_AITOOLS_AUTO_INSTALL=false [CLI] jobs list + +>>> print_telemetry +{"command":"jobs_list","exit_code":0,"aitools_auto_install_event":{"outcome":"SKIPPED_OPTOUT","holdout_arm":"TREATMENT","interactive":false}} + +=== auto-install disable: not installed + +>>> [CLI] aitools auto-install disable +Automatic installation of Databricks AI tools is disabled. + +>>> [CLI] jobs list + +>>> print_telemetry +{"command":"jobs_list","exit_code":0,"aitools_auto_install_event":{"outcome":"SKIPPED_OPTOUT","holdout_arm":"TREATMENT","interactive":false}} + +>>> cursor_skills +(no skills) + +>>> [CLI] aitools auto-install enable +Automatic installation of Databricks AI tools is enabled. + +=== commands that never trigger it + +>>> [CLI] version +Databricks CLI v[CLI_VERSION] + +>>> [CLI] aitools auto-install enable +Automatic installation of Databricks AI tools is enabled. + +>>> print_telemetry + +>>> DATABRICKS_CACHE_ENABLED=false [CLI] jobs list + +>>> print_telemetry + +>>> auto_install_state +{ + "last_version": "1.32.0", + "id": "treatment-machine" +} + +>>> cursor_skills +(no skills) + +=== the gate is off: nothing printed or reported, and the version is settled + +>>> [CLI] jobs list + +>>> print_telemetry + +>>> auto_install_state +{ + "last_version": "1.33.0", + "id": "treatment-machine" +} + +>>> [CLI] jobs list + +>>> print_telemetry + +>>> cursor_skills +(no skills) + +=== a machine in the holdout: not installed + +>>> [CLI] jobs list + +>>> print_telemetry +{"command":"jobs_list","exit_code":0,"aitools_auto_install_event":{"outcome":"SKIPPED_HOLDOUT","holdout_arm":"HOLDOUT","interactive":false}} + +>>> cursor_skills +(no skills) + +=== no agent config directory: nothing to install + +>>> [CLI] jobs list + +>>> print_telemetry +{"command":"jobs_list","exit_code":0,"aitools_auto_install_event":{"outcome":"SKIPPED_NO_AGENTS","holdout_arm":"TREATMENT","interactive":false}} diff --git a/acceptance/cmd/aitools/auto-install/skipped/script b/acceptance/cmd/aitools/auto-install/skipped/script new file mode 100644 index 00000000000..e4b76797c0c --- /dev/null +++ b/acceptance/cmd/aitools/auto-install/skipped/script @@ -0,0 +1,58 @@ +use_home home +gate true + +title "CI: not installed, reported once for the version" +trace CI=true $CLI jobs list +trace print_telemetry +trace CI=true $CLI jobs list +trace print_telemetry +trace cursor_skills + +export DATABRICKS_AITOOLS_AUTO_INSTALL_CLI_VERSION=1.31.0 +title "DATABRICKS_AITOOLS_AUTO_INSTALL=false: not installed" +trace DATABRICKS_AITOOLS_AUTO_INSTALL=false $CLI jobs list +trace print_telemetry + +export DATABRICKS_AITOOLS_AUTO_INSTALL_CLI_VERSION=1.32.0 +title "auto-install disable: not installed" +trace $CLI aitools auto-install disable +trace $CLI jobs list +trace print_telemetry +trace cursor_skills +trace $CLI aitools auto-install enable + +export DATABRICKS_AITOOLS_AUTO_INSTALL_CLI_VERSION=1.33.0 +title "commands that never trigger it" +trace $CLI version +trace $CLI aitools auto-install enable +$CLI completion bash &> LOG.completion +$CLI __complete jobs &> LOG.complete +$CLI help jobs &> LOG.help +trace print_telemetry +trace DATABRICKS_CACHE_ENABLED=false $CLI jobs list +trace print_telemetry +trace auto_install_state +trace cursor_skills + +title "the gate is off: nothing printed or reported, and the version is settled" +gate false +trace $CLI jobs list +trace print_telemetry +trace auto_install_state +# Turning the gate on reaches this machine on its next new version. +gate true +trace $CLI jobs list +trace print_telemetry +trace cursor_skills + +title "a machine in the holdout: not installed" +use_home holdout-home holdout-machine +trace $CLI jobs list +trace print_telemetry +trace cursor_skills + +title "no agent config directory: nothing to install" +use_home no-agents-home +"$rm_bin" -r "${USERPROFILE:-$HOME}/.cursor" +trace $CLI jobs list +trace print_telemetry diff --git a/acceptance/cmd/aitools/auto-install/test.toml b/acceptance/cmd/aitools/auto-install/test.toml index da6d94db55c..1883cb3dbb2 100644 --- a/acceptance/cmd/aitools/auto-install/test.toml +++ b/acceptance/cmd/aitools/auto-install/test.toml @@ -1,13 +1,17 @@ # Automatic installation of Databricks AI tools on the first run of a new CLI # version. The mock server replaces raw.githubusercontent.com (skills manifest -# and files). +# and files) and the GitHub API (latest skills release). Env.DATABRICKS_SKILLS_BASE_URL = "$DATABRICKS_HOST" +Env.DATABRICKS_CLI_GITHUB_API_URL = "$DATABRICKS_HOST" Env.DATABRICKS_SKILLS_REF = "test-ref" +RecordRequests = true # Pin it so an opt-out exported in the developer's shell does not change the output. Env.DATABRICKS_AITOOLS_AUTO_INSTALL = "true" Ignore = [ "home", + "holdout-home", + "no-agents-home", ] [[Server]] @@ -35,3 +39,7 @@ name: databricks-test # Test skill ''' + +[[Server]] +Pattern = "GET /broken-ref/manifest.json" +Response.StatusCode = 500 diff --git a/acceptance/cmd/aitools/auto-install/upgrade/out.test.toml b/acceptance/cmd/aitools/auto-install/upgrade/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/cmd/aitools/auto-install/upgrade/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/cmd/aitools/auto-install/upgrade/output.txt b/acceptance/cmd/aitools/auto-install/upgrade/output.txt new file mode 100644 index 00000000000..f608694937f --- /dev/null +++ b/acceptance/cmd/aitools/auto-install/upgrade/output.txt @@ -0,0 +1,66 @@ + +=== first run of 1.30.0 installs skills release 0.3.0 + +>>> [CLI] jobs list +Installed Databricks AI tools for Cursor. Disable: databricks aitools auto-install disable + +>>> print_telemetry +{"command":"jobs_list","exit_code":0,"aitools_install_event":{"agents":["CURSOR"],"scope":"GLOBAL","error_category":"TYPE_UNSPECIFIED","trigger":"AUTO_FIRST_RUN"}} +{"command":"jobs_list","exit_code":0,"aitools_auto_install_event":{"outcome":"INSTALLED","holdout_arm":"TREATMENT","interactive":false}} + +>>> cursor_skills +databricks-dropped +databricks-edited +databricks-kept + +=== upgrade to 1.31.0 with release 0.4.0: updates like aitools update, keeping the edited skills + +>>> DATABRICKS_AITOOLS_AUTO_INSTALL_CLI_VERSION=1.31.0 DATABRICKS_CLI_GITHUB_API_URL=[DATABRICKS_URL]/newer [CLI] jobs list +Updated Databricks AI tools for Cursor. Disable: databricks aitools auto-install disable + +>>> print_telemetry +{"command":"jobs_list","exit_code":0,"aitools_auto_install_event":{"outcome":"INSTALLED","holdout_arm":"TREATMENT","interactive":false}} + +>>> cursor_skills +databricks-dropped +databricks-edited +databricks-kept + +>>> skill_file databricks-kept +kept 2.0.0 + +>>> skill_file databricks-dropped +dropped 1.0.0 +my notes + +>>> skill_file databricks-edited +edited 1.0.0 +my notes + +=== upgrade to 1.32.0 while the latest release is 0.2.0: no downgrade, nothing printed, not retried + +>>> DATABRICKS_AITOOLS_AUTO_INSTALL_CLI_VERSION=1.32.0 DATABRICKS_CLI_GITHUB_API_URL=[DATABRICKS_URL]/older [CLI] jobs list + +>>> print_telemetry +{"command":"jobs_list","exit_code":0,"aitools_auto_install_event":{"outcome":"KEPT","holdout_arm":"TREATMENT","interactive":false}} + +>>> skill_file databricks-kept +kept 2.0.0 + +>>> auto_install_state +{ + "last_version": "1.32.0", + "id": "treatment-machine" +} + +=== an older CLI on the same machine (e.g. one an IDE ships) does nothing + +>>> DATABRICKS_AITOOLS_AUTO_INSTALL_CLI_VERSION=1.31.0 [CLI] jobs list + +>>> print_telemetry + +>>> auto_install_state +{ + "last_version": "1.32.0", + "id": "treatment-machine" +} diff --git a/acceptance/cmd/aitools/auto-install/upgrade/script b/acceptance/cmd/aitools/auto-install/upgrade/script new file mode 100644 index 00000000000..0be68b26379 --- /dev/null +++ b/acceptance/cmd/aitools/auto-install/upgrade/script @@ -0,0 +1,36 @@ +use_home home +# Track the latest skills release, as the compatibility manifest does. +unset DATABRICKS_SKILLS_REF +gate true +skills_dir="${USERPROFILE:-$HOME}/.databricks/aitools/skills" +skill_file() { + cat "$skills_dir/$1/SKILL.md" +} + +title "first run of 1.30.0 installs skills release 0.3.0" +trace $CLI jobs list +trace print_telemetry +trace cursor_skills + +# The user edits a skill that the next release drops, and one that it updates. +echo "my notes" >> "$skills_dir/databricks-dropped/SKILL.md" +echo "my notes" >> "$skills_dir/databricks-edited/SKILL.md" + +title "upgrade to 1.31.0 with release 0.4.0: updates like aitools update, keeping the edited skills" +trace DATABRICKS_AITOOLS_AUTO_INSTALL_CLI_VERSION=1.31.0 DATABRICKS_CLI_GITHUB_API_URL=$DATABRICKS_HOST/newer $CLI jobs list +trace print_telemetry +trace cursor_skills +trace skill_file databricks-kept +trace skill_file databricks-dropped +trace skill_file databricks-edited + +title "upgrade to 1.32.0 while the latest release is 0.2.0: no downgrade, nothing printed, not retried" +trace DATABRICKS_AITOOLS_AUTO_INSTALL_CLI_VERSION=1.32.0 DATABRICKS_CLI_GITHUB_API_URL=$DATABRICKS_HOST/older $CLI jobs list +trace print_telemetry +trace skill_file databricks-kept +trace auto_install_state + +title "an older CLI on the same machine (e.g. one an IDE ships) does nothing" +trace DATABRICKS_AITOOLS_AUTO_INSTALL_CLI_VERSION=1.31.0 $CLI jobs list +trace print_telemetry +trace auto_install_state diff --git a/acceptance/cmd/aitools/auto-install/upgrade/test.toml b/acceptance/cmd/aitools/auto-install/upgrade/test.toml new file mode 100644 index 00000000000..1511d4aaad2 --- /dev/null +++ b/acceptance/cmd/aitools/auto-install/upgrade/test.toml @@ -0,0 +1,73 @@ +# Skills releases: the latest release is v0.3.0, under /newer it is v0.4.0 and +# under /older it is v0.2.0 (DATABRICKS_CLI_GITHUB_API_URL picks one). + +[[Server]] +Pattern = "GET /repos/databricks/databricks-agent-skills/releases/latest" +Response.Body = '{"tag_name": "v0.3.0"}' + +[[Server]] +Pattern = "GET /newer/repos/databricks/databricks-agent-skills/releases/latest" +Response.Body = '{"tag_name": "v0.4.0"}' + +[[Server]] +Pattern = "GET /older/repos/databricks/databricks-agent-skills/releases/latest" +Response.Body = '{"tag_name": "v0.2.0"}' + +[[Server]] +Pattern = "GET /v0.3.0/manifest.json" +Response.Body = ''' +{ + "version": "2", + "updated_at": "2026-01-01T00:00:00Z", + "skills": { + "databricks-kept": {"version": "1.0.0", "description": "Kept", "files": ["SKILL.md"], "repo_dir": "skills"}, + "databricks-dropped": {"version": "1.0.0", "description": "Dropped", "files": ["SKILL.md"], "repo_dir": "skills"}, + "databricks-edited": {"version": "1.0.0", "description": "Edited", "files": ["SKILL.md"], "repo_dir": "skills"} + } +} +''' + +[[Server]] +Pattern = "GET /v0.3.0/skills/databricks-kept/SKILL.md" +Response.Body = "kept 1.0.0\n" + +[[Server]] +Pattern = "GET /v0.3.0/skills/databricks-dropped/SKILL.md" +Response.Body = "dropped 1.0.0\n" + +[[Server]] +Pattern = "GET /v0.3.0/skills/databricks-edited/SKILL.md" +Response.Body = "edited 1.0.0\n" + +[[Server]] +Pattern = "GET /v0.4.0/manifest.json" +Response.Body = ''' +{ + "version": "2", + "updated_at": "2026-02-01T00:00:00Z", + "skills": { + "databricks-kept": {"version": "2.0.0", "description": "Kept", "files": ["SKILL.md"], "repo_dir": "skills"}, + "databricks-edited": {"version": "2.0.0", "description": "Edited", "files": ["SKILL.md"], "repo_dir": "skills"} + } +} +''' + +[[Server]] +Pattern = "GET /v0.4.0/skills/databricks-kept/SKILL.md" +Response.Body = "kept 2.0.0\n" + +[[Server]] +Pattern = "GET /v0.4.0/skills/databricks-edited/SKILL.md" +Response.Body = "edited 2.0.0\n" + +[[Server]] +Pattern = "GET /v0.2.0/manifest.json" +Response.Body = ''' +{ + "version": "2", + "updated_at": "2025-12-01T00:00:00Z", + "skills": { + "databricks-kept": {"version": "0.5.0", "description": "Kept", "files": ["SKILL.md"], "repo_dir": "skills"} + } +} +''' diff --git a/acceptance/cmd/aitools/script.prepare b/acceptance/cmd/aitools/script.prepare new file mode 100644 index 00000000000..48a1478966f --- /dev/null +++ b/acceptance/cmd/aitools/script.prepare @@ -0,0 +1,13 @@ +# print_telemetry prints each telemetry event uploaded since the last call, one +# per line: the command it ran under, its exit code, and the event without the +# rest of the execution context (timings and versions vary). print_requests.py +# clears the recorded requests, so the next call sees only the next command's +# upload. It fails when no request was recorded, hence the file check. +print_telemetry() { + if [ -f "$OUT_REQUESTS" ]; then + print_requests.py --oneline //telemetry-ext | jq -c '.body.protoLogs[] | fromjson + | .entry.databricks_cli_log + | {command: .execution_context.command, exit_code: .execution_context.exit_code} + + (del(.execution_context))' + fi +} diff --git a/acceptance/cmd/aitools/telemetry/script.prepare b/acceptance/cmd/aitools/telemetry/script.prepare index f55e1cef710..9d3b1d18c4a 100644 --- a/acceptance/cmd/aitools/telemetry/script.prepare +++ b/acceptance/cmd/aitools/telemetry/script.prepare @@ -1,17 +1,3 @@ # A host that disables telemetry or sets a zero upload timeout would upload # nothing, so these cases would record no events. unset DATABRICKS_CLI_DISABLE_TELEMETRY DATABRICKS_CLI_TELEMETRY_TIMEOUT - -# print_telemetry prints each telemetry event uploaded since the last call, one -# per line: the command it ran under, its exit code, and the event without the -# rest of the execution context (timings and versions vary). print_requests.py -# clears the recorded requests, so the next call sees only the next command's -# upload. It fails when no request was recorded, hence the file check. -print_telemetry() { - if [ -f "$OUT_REQUESTS" ]; then - print_requests.py --oneline //telemetry-ext | jq -c '.body.protoLogs[] | fromjson - | .entry.databricks_cli_log - | {command: .execution_context.command, exit_code: .execution_context.exit_code} - + (del(.execution_context))' - fi -} diff --git a/cmd/aitools/autoinstall.go b/cmd/aitools/autoinstall.go index 9cafbf4d5cf..bdd174deffe 100644 --- a/cmd/aitools/autoinstall.go +++ b/cmd/aitools/autoinstall.go @@ -19,11 +19,12 @@ func newAutoInstallCmd() *cobra.Command { On the first run of a new CLI version, the CLI can install the Databricks AI tools (skills and plugins) for the coding agents already on this machine, as 'databricks aitools install --scope=global' does, or update an existing -installation, as 'databricks aitools update' does. It never runs in CI and only -touches the global scope. +installation, as 'databricks aitools update' does. It never runs in CI or on +Databricks Runtime, and only touches the global scope. -It stays off after 'databricks aitools auto-install disable' or a global-scope -'databricks aitools uninstall', until 'databricks aitools auto-install enable'. +It stays off after 'databricks aitools auto-install disable', a global-scope +'databricks aitools uninstall', or a "no" to the skills prompt of +'databricks apps init', until 'databricks aitools auto-install enable'. Setting ` + autoinstall.EnvVar + ` to anything but a true value (true, 1, yes, on) turns it off in that environment.`, RunE: root.ReportUnknownSubcommand, diff --git a/cmd/aitools/autoinstall_plan_test.go b/cmd/aitools/autoinstall_plan_test.go new file mode 100644 index 00000000000..14dc7ef3411 --- /dev/null +++ b/cmd/aitools/autoinstall_plan_test.go @@ -0,0 +1,163 @@ +package aitools + +import ( + "maps" + "os" + "path/filepath" + "testing" + + "github.com/databricks/cli/libs/aitools/agents" + "github.com/databricks/cli/libs/aitools/autoinstall" + "github.com/databricks/cli/libs/aitools/installer" + "github.com/databricks/cli/libs/telemetry/protos" + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// autoInstallHome isolates HOME with Claude Code (its CLI on PATH) and Cursor +// detected. +func autoInstallHome(t *testing.T) string { + t.Helper() + tmp := t.TempDir() + isolateAgents(t, "HOME", tmp, "USERPROFILE", tmp, "XDG_CONFIG_HOME", filepath.Join(tmp, ".config")) + require.NoError(t, os.MkdirAll(filepath.Join(tmp, ".claude"), 0o755)) + require.NoError(t, os.MkdirAll(filepath.Join(tmp, ".cursor"), 0o755)) + fakeBinsOnPath(t, "claude") + return tmp +} + +func setAgentCLIsStoppable(t *testing.T, v bool) { + t.Helper() + orig := agentCLIsStoppable + t.Cleanup(func() { agentCLIsStoppable = orig }) + agentCLIsStoppable = v +} + +func planAgentNames(plan []agentPlanItem) []string { + var names []string + for _, it := range plan { + names = append(names, it.agent.DisplayName) + } + return names +} + +func TestPlanToolsRunsNoAgentCLIWhereTimeoutsCannotStopThem(t *testing.T) { + autoInstallHome(t) + + setAgentCLIsStoppable(t, true) + tools, err := planTools(t.Context(), autoinstall.State{}) + require.NoError(t, err) + assert.Equal(t, []string{"Claude Code", "Cursor"}, planAgentNames(tools.plan)) + + // The plugin install would run Claude Code's CLI. + setAgentCLIsStoppable(t, false) + tools, err = planTools(t.Context(), autoinstall.State{}) + require.NoError(t, err) + assert.Equal(t, []string{"Cursor"}, planAgentNames(tools.plan)) + assert.Empty(t, tools.skip) + + // Updating the recorded plugins would run agent CLIs too. + saveGlobalState(t, map[string]string{"databricks-sql": "0.1.0"}) + tools, err = planTools(t.Context(), autoinstall.State{}) + require.NoError(t, err) + assert.Nil(t, tools.existing) + assert.Equal(t, protos.AitoolsAutoInstallOutcomeSkippedPlatform, tools.skip) + + setAgentCLIsStoppable(t, true) + tools, err = planTools(t.Context(), autoinstall.State{}) + require.NoError(t, err) + assert.NotNil(t, tools.existing) + assert.Empty(t, tools.skip) +} + +func TestPlanToolsReportsWhyNothingIsInstalled(t *testing.T) { + home := autoInstallHome(t) + + // Claude Code alone installs through its CLI, which is not run here. + require.NoError(t, os.RemoveAll(filepath.Join(home, ".cursor"))) + setAgentCLIsStoppable(t, false) + tools, err := planTools(t.Context(), autoinstall.State{}) + require.NoError(t, err) + assert.Equal(t, protos.AitoolsAutoInstallOutcomeSkippedPlatform, tools.skip) + + require.NoError(t, os.RemoveAll(filepath.Join(home, ".claude"))) + tools, err = planTools(t.Context(), autoinstall.State{}) + require.NoError(t, err) + assert.Equal(t, protos.AitoolsAutoInstallOutcomeSkippedNoAgents, tools.skip) +} + +func TestPlanToolsKeepsSkillsWithoutInstallState(t *testing.T) { + autoInstallHome(t) + dir, err := installer.GlobalSkillsDir(t.Context()) + require.NoError(t, err) + // No checksums record what the CLI wrote, so edits cannot be told apart. + require.NoError(t, os.MkdirAll(filepath.Join(dir, "databricks-sql"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "databricks-sql", "SKILL.md"), []byte("user edit"), 0o644)) + + tools, err := planTools(t.Context(), autoinstall.State{}) + require.NoError(t, err) + assert.Equal(t, protos.AitoolsAutoInstallOutcomeKept, tools.skip) + assert.Empty(t, tools.plan) + + result, err := tools.install(t.Context()) + require.NoError(t, err) + assert.Equal(t, protos.AitoolsAutoInstallOutcomeKept, result.outcome) + assert.Empty(t, result.agents) + data, err := os.ReadFile(filepath.Join(dir, "databricks-sql", "SKILL.md")) + require.NoError(t, err) + assert.Equal(t, "user edit", string(data)) + + // Skills that an unfinished install of auto-install left are its own, so + // a later run finishes the install. + tools, err = planTools(t.Context(), autoinstall.State{Installing: true}) + require.NoError(t, err) + assert.Empty(t, tools.skip) + assert.True(t, tools.fresh()) +} + +func TestChangedAgents(t *testing.T) { + autoInstallHome(t) + dir := saveGlobalState(t, map[string]string{"databricks-sql": "0.1.0"}) + require.NoError(t, os.MkdirAll(filepath.Join(dir, "databricks-sql"), 0o755)) + before, err := installer.LoadState(dir) + require.NoError(t, err) + + // A new release in which every outdated skill was kept changed nothing. + after := *before + after.Release = "v0.3.0" + assert.Empty(t, changedAgents(t.Context(), before, &after)) + + // Only the agent whose plugin was updated changed. + after.Plugins = maps.Clone(before.Plugins) + rec := after.Plugins[agents.NameClaudeCode] + rec.Version = "0.3.0" + after.Plugins[agents.NameClaudeCode] = rec + assert.Equal(t, []string{"Claude Code"}, changedAgents(t.Context(), before, &after)) +} + +func TestSkipsAutoInstall(t *testing.T) { + root := &cobra.Command{Use: "databricks"} + add := func(parent *cobra.Command, name string) *cobra.Command { + c := &cobra.Command{Use: name} + parent.AddCommand(c) + return c + } + jobsList := add(add(root, "jobs"), "list") + auth := add(root, "auth") + authToken := add(auth, "token") + dockerToken := add(add(auth, "docker"), "token") + authDescribe := add(auth, "describe") + aitoolsList := add(add(root, "aitools"), "list") + experimentalAitools := add(add(add(root, "experimental"), "aitools"), "tools") + version := add(root, "version") + completion := add(add(root, "completion"), "zsh") + complete := add(root, cobra.ShellCompRequestCmd) + + for _, cmd := range []*cobra.Command{authToken, dockerToken, aitoolsList, experimentalAitools, version, completion, complete} { + assert.True(t, skipsAutoInstall(cmd), cmd.CommandPath()) + } + for _, cmd := range []*cobra.Command{jobsList, authDescribe} { + assert.False(t, skipsAutoInstall(cmd), cmd.CommandPath()) + } +} diff --git a/cmd/aitools/autoinstall_run.go b/cmd/aitools/autoinstall_run.go new file mode 100644 index 00000000000..88cb1444bd0 --- /dev/null +++ b/cmd/aitools/autoinstall_run.go @@ -0,0 +1,431 @@ +package aitools + +import ( + "context" + "errors" + "fmt" + "io" + "log/slog" + "maps" + "runtime" + "slices" + "strings" + "time" + + "github.com/databricks/cli/libs/aitools/agents" + "github.com/databricks/cli/libs/aitools/autoinstall" + "github.com/databricks/cli/libs/aitools/installer" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/dbr" + "github.com/databricks/cli/libs/flags" + "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/telemetry" + "github.com/databricks/cli/libs/telemetry/protos" + "github.com/databricks/cli/libs/versioncheck" + "github.com/spf13/cobra" +) + +const ( + // gateTimeout bounds the gate lookup, one compatibility manifest fetch. + // Every machine pays it once per version while the gate is off, so it is + // kept short. A lookup that fails counts as a failed attempt. + gateTimeout = 3 * time.Second + + // installTimeout bounds the install or update (its downloads and the + // agent CLIs it runs). One that runs out of time counts as a failed + // attempt and is retried on a later run. + installTimeout = 10 * time.Second +) + +// autoInstallDisable is the opt-out command named in the notice. +const autoInstallDisable = "databricks aitools auto-install disable" + +// AutoInstall installs Databricks AI tools for the coding agents on this +// machine, or updates an existing installation, on the first run of a new CLI +// version. The root command runs it after a command succeeded. It never fails +// the command and never writes to stdout: it prints at most one line to +// stderr, and why it skipped or failed only to the debug log. +func AutoInstall(cmd *cobra.Command) { + ctx := cmd.Context() + defer func() { + if r := recover(); r != nil { + log.Debugf(ctx, "auto-install: panic: %v", r) + } + }() + autoInstall(ctx, skipsAutoInstall(cmd)) +} + +// skipsAutoInstall reports whether cmd never triggers auto-install: the +// commands the new-release notice skips (`version` is how a user checks what +// was just installed; help and shell completion output is read by machines or +// shown on every keystroke), the credential helpers that other tools call for +// a token, and the aitools commands, which manage the tools explicitly. +// +// Runs without a terminal and runs with JSON output are not skipped, unlike +// for the notice: coding agents run the CLI that way, and they are what the +// tools are for. Auto-install writes nothing to stdout. +func skipsAutoInstall(cmd *cobra.Command) bool { + if versioncheck.IsExemptCommand(cmd) || agents.IsCredentialHelperCommand(cmd) { + return true + } + for c := cmd; c != nil; c = c.Parent() { + if c.Name() == "aitools" { + return true + } + } + return false +} + +func autoInstall(ctx context.Context, skippedCommand bool) { + version := autoinstall.Version(ctx) + in := autoinstall.Inputs{ + SkippedCommand: skippedCommand, + OnRuntime: dbr.RunsOnRuntime(ctx), + CacheDisabled: autoinstall.CacheDisabled(ctx), + Version: version, + CI: agents.InCI(ctx), + // A "no" earlier in this process counts even if storing it failed. + EnvOptOut: autoinstall.EnvOptedOut(ctx) || autoinstall.Declined(), + } + // What never auto-installs skips before the state is read, and without a + // log line: these are most runs. + if slices.Contains(stateFreeSkips, autoinstall.Decide(in)) { + return + } + var err error + in.State, err = autoinstall.Load(ctx) + if err != nil { + log.Debugf(ctx, "auto-install: skipped: %v", err) + return + } + if reason := autoinstall.Decide(in); !reason.Due() { + log.Debugf(ctx, "auto-install: skipped: %s", reason) + return + } + + unlock, ok := autoinstall.TryLock(ctx) + if !ok { + log.Debugf(ctx, "auto-install: skipped: another CLI process is running it") + return + } + defer unlock() + // Another process may have finished a run since the state was read. + in.State, err = autoinstall.Load(ctx) + if err != nil { + log.Debugf(ctx, "auto-install: skipped: %v", err) + return + } + id := in.State.ID + if id == "" { + id = autoinstall.NewID() + } + in.Bucket = autoinstall.Bucket(id) + + // The gate costs a manifest lookup, so it is resolved only when it is + // what decides the run. + reason := autoinstall.Decide(in) + if reason == autoinstall.ReasonGateOff { + gateCtx, cancel := context.WithTimeout(ctx, gateTimeout) + in.GateOn, err = autoinstall.GateOn(gateCtx, version) + cancel() + if err != nil { + // The gate is unknown, so the version is not settled: a later run + // looks it up again. Like a failed install, the lookup uses up an + // attempt, so a machine that is offline stops paying for it. The + // feature may be off, so nothing is reported. + attempt := autoinstall.NextAttempt(in.State, version) + log.Debugf(ctx, "auto-install: attempt %d of %d: could not resolve the gate: %v", attempt, autoinstall.MaxAttempts, err) + if err := autoinstall.Record(ctx, version, attempt, id); err != nil { + log.Debugf(ctx, "auto-install: could not save the state: %v", err) + } + return + } + reason = autoinstall.Decide(in) + } + run := autoInstallRun{ + version: version, + id: id, + arm: holdoutArm(in.Bucket), + interactive: installPromptsSupported(ctx), + } + if !reason.Due() { + // Another process settled the version while this one waited. + log.Debugf(ctx, "auto-install: skipped: %s", reason) + return + } + if reason == autoinstall.ReasonGateOff { + // The feature is not launched, so the skip is not reported. + log.Debugf(ctx, "auto-install: skipped: %s", reason) + if err := autoinstall.Record(ctx, version, 0, id); err != nil { + log.Debugf(ctx, "auto-install: could not save the state: %v", err) + } + return + } + if outcome, ok := skipOutcomes[reason]; ok { + run.settle(ctx, outcome) + return + } + run.attempt(ctx, in.State) +} + +// attempt installs or updates the tools and reports the outcome. prior is the +// state the run was decided on. +func (r autoInstallRun) attempt(ctx context.Context, prior autoinstall.State) { + attempt := autoinstall.NextAttempt(prior, r.version) + qctx := quietContext(ctx) + tools, err := planTools(qctx, prior) + // Count the attempt before making it: an attempt that never finishes (the + // process is killed) still counts, and a state that cannot be saved stops + // auto-install instead of retrying it on every run. + if recErr := autoinstall.Record(ctx, r.version, attempt, r.id); recErr != nil { + log.Debugf(ctx, "auto-install: skipped: could not save the state: %v", recErr) + return + } + if err == nil && tools.fresh() { + // Skills an unfinished install leaves have no install state; the + // marker tells a later run that they are its own. + err = autoinstall.MarkInstalling(ctx) + } + var result installResult + if err == nil { + runCtx, cancel := context.WithTimeout(qctx, installTimeout) + result, err = tools.install(runCtx) + cancel() + } + if _, refused := errors.AsType[*installer.RefusedError](err); refused { + // The installed skills are newer than the release this CLI would + // install (the downgrade guard). Retrying changes nothing until a + // newer release, which a later version gets. + log.Debugf(ctx, "auto-install: kept the installed tools: %v", err) + result, err = installResult{outcome: protos.AitoolsAutoInstallOutcomeKept}, nil + } + switch { + case err != nil: + outcome := protos.AitoolsAutoInstallOutcomeFailed + if attempt >= autoinstall.MaxAttempts { + outcome = protos.AitoolsAutoInstallOutcomeRetriesExhausted + } + log.Debugf(ctx, "auto-install: attempt %d of %d failed: %v", attempt, autoinstall.MaxAttempts, err) + r.report(ctx, outcome, classifyInstallError(err)) + default: + r.settle(ctx, result.outcome) + if result.outcome == protos.AitoolsAutoInstallOutcomeInstalled && len(result.agents) > 0 { + verb := "Installed" + if result.updated { + verb = "Updated" + } + cmdio.LogString(ctx, fmt.Sprintf("%s Databricks AI tools for %s. Disable: %s", verb, strings.Join(result.agents, ", "), autoInstallDisable)) + } + } +} + +// stateFreeSkips are the reasons Decide finds without the state. +var stateFreeSkips = []autoinstall.Reason{ + autoinstall.ReasonSkippedCommand, + autoinstall.ReasonOnRuntime, + autoinstall.ReasonCacheDisabled, + autoinstall.ReasonDevelopmentBuild, +} + +// skipOutcomes are the reported outcomes of the skips that settle a version. +var skipOutcomes = map[autoinstall.Reason]protos.AitoolsAutoInstallOutcome{ + autoinstall.ReasonSkippedCI: protos.AitoolsAutoInstallOutcomeSkippedCI, + autoinstall.ReasonSkippedOptOut: protos.AitoolsAutoInstallOutcomeSkippedOptOut, + autoinstall.ReasonSkippedHoldout: protos.AitoolsAutoInstallOutcomeSkippedHoldout, +} + +// autoInstallRun is what the outcome of one auto-install run is reported with. +type autoInstallRun struct { + version string + id string + arm protos.AitoolsHoldoutArm + interactive bool +} + +// settle records the version as done and reports outcome. +func (r autoInstallRun) settle(ctx context.Context, outcome protos.AitoolsAutoInstallOutcome) { + log.Debugf(ctx, "auto-install: %s", strings.ToLower(string(outcome))) + if err := autoinstall.Record(ctx, r.version, 0, r.id); err != nil { + log.Debugf(ctx, "auto-install: could not save the state: %v", err) + } + r.report(ctx, outcome, protos.AitoolsErrorCategoryUnspecified) +} + +// report buffers the auto-install event; cmd/root uploads it at exit when the +// command resolved auth. +func (r autoInstallRun) report(ctx context.Context, outcome protos.AitoolsAutoInstallOutcome, category protos.AitoolsErrorCategory) { + if category == protos.AitoolsErrorCategoryUnspecified { + category = "" + } + telemetry.Log(ctx, protos.DatabricksCliLog{ + AitoolsAutoInstallEvent: &protos.AitoolsAutoInstallEvent{ + Outcome: outcome, + HoldoutArm: r.arm, + Interactive: r.interactive, + ErrorCategory: category, + }, + }) +} + +func holdoutArm(bucket int) protos.AitoolsHoldoutArm { + if autoinstall.InHoldout(bucket) { + return protos.AitoolsHoldoutArmHoldout + } + return protos.AitoolsHoldoutArmTreatment +} + +// installResult is what one auto-install attempt did. +type installResult struct { + // outcome is the reported outcome of an attempt that did not fail. + outcome protos.AitoolsAutoInstallOutcome + // updated is set when an existing installation was updated. + updated bool + // agents names the agents that got new or updated tools. + agents []string +} + +// toolsPlan is what an auto-install attempt does: update the existing global +// installation, install for the agents in plan, or leave the tools alone. +type toolsPlan struct { + globalDir string + existing *installer.InstallState + plan []agentPlanItem + // skip, when set, is the outcome of a plan that changes nothing, and + // skipWhy says why. + skip protos.AitoolsAutoInstallOutcome + skipWhy string +} + +// fresh reports whether the plan is a first install. +func (t toolsPlan) fresh() bool { + return t.skip == "" && t.existing == nil && len(t.plan) > 0 +} + +// planTools plans what `databricks aitools install --scope=global` runs +// without prompts, for the agents whose config directory exists. An existing +// global installation is updated as `databricks aitools update --scope=global` +// does instead, which keeps its downgrade guard, keeps the skills whose files +// the user modified at their version, and does not migrate skills to plugins. +// Planning reads only local state; prior is the auto-install state. +func planTools(ctx context.Context, prior autoinstall.State) (toolsPlan, error) { + globalDir, err := installer.GlobalSkillsDir(ctx) + if err != nil { + return toolsPlan{}, err + } + existing, err := installer.LoadState(globalDir) + if err != nil { + return toolsPlan{}, err + } + if existing != nil { + // The update runs agent CLIs to update the recorded plugins. + if !agentCLIsStoppable && len(existing.Plugins) > 0 { + return toolsPlan{skip: protos.AitoolsAutoInstallOutcomeSkippedPlatform, skipWhy: "the update would run agent CLIs"}, nil + } + return toolsPlan{globalDir: globalDir, existing: existing}, nil + } + // Skills without install state have no checksums to tell the user's edits + // from ours, and an install would replace them. The ones an unfinished + // install of auto-install left are its own. + if !prior.Installing && installer.HasUntrackedSkills(ctx, installer.ScopeGlobal) { + return toolsPlan{skip: protos.AitoolsAutoInstallOutcomeKept, skipWhy: "skills without install state are on disk"}, nil + } + plan := buildPlan(ctx, agents.DetectInstalled(ctx), installer.ScopeGlobal, false, false) + if len(plan) == 0 { + return toolsPlan{skip: protos.AitoolsAutoInstallOutcomeSkippedNoAgents, skipWhy: "no supported coding agent has a config directory"}, nil + } + if !agentCLIsStoppable { + plan = slices.DeleteFunc(plan, func(it agentPlanItem) bool { + return it.delivery == deliveryPlugin && !it.cliMissing + }) + if len(plan) == 0 { + return toolsPlan{skip: protos.AitoolsAutoInstallOutcomeSkippedPlatform, skipWhy: "every agent installs through its CLI"}, nil + } + } + return toolsPlan{globalDir: globalDir, plan: plan}, nil +} + +// agentCLIsStoppable reports whether a timeout stops an agent CLI together +// with its descendants. On Windows it stops only the CLI, and a descendant +// left running could still be installing when the next run takes the lock, +// so auto-install there runs no agent CLI. +var agentCLIsStoppable = runtime.GOOS != "windows" + +// install carries out the plan. +func (t toolsPlan) install(ctx context.Context) (installResult, error) { + if t.skip != "" { + log.Debugf(ctx, "auto-install: nothing to do: %s", t.skipWhy) + return installResult{outcome: t.skip}, nil + } + if t.existing != nil { + return updateTools(ctx, t.globalDir, t.existing) + } + opts := installer.InstallOptions{Scope: installer.ScopeGlobal} + outcomes, err := executePlan(ctx, &installer.GitHubManifestSource{}, t.plan, opts, installInvocation{scope: installer.ScopeGlobal}, true) + topErr := topLevelFailure(err) + logInstallEvent(ctx, t.plan, installOpts{Scope: opts.Scope, Trigger: protos.AitoolsInstallTriggerAutoFirstRun}, classifyInstallError(topErr), outcomes) + if err != nil && topErr == nil { + // Reported per agent: take the category from the outcomes, as install does. + err = withCategory(err, firstFailureCategory(outcomes)) + } + var names []string + for _, o := range outcomes { + if o.status == outcomeInstalled { + names = append(names, o.agent.DisplayName) + } + } + return installResult{outcome: protos.AitoolsAutoInstallOutcomeInstalled, agents: names}, err +} + +// updateTools runs the update command's body for the global scope. Its funnel +// events and its wait for telemetry auth belong to the command, so they are +// detached: the auto-install event reports the run. +func updateTools(ctx context.Context, globalDir string, before *installer.InstallState) (installResult, error) { + uctx := telemetry.WithNewLogger(ctx) + uctx = context.WithValue(uctx, funnelKey{}, &commandFunnel{command: protos.AitoolsCommandUpdate}) + uctx = context.WithValue(uctx, telemetryAuthKey{}, &telemetryAuth{}) + if err := runUpdate(uctx, updateFlags{scope: installer.ScopeGlobal, keepModified: true}); err != nil { + return installResult{}, err + } + after, err := installer.LoadState(globalDir) + if err != nil { + return installResult{}, err + } + result := installResult{outcome: protos.AitoolsAutoInstallOutcomeUpToDate, updated: true} + if after != nil { + result.agents = changedAgents(ctx, before, after) + } + if len(result.agents) > 0 { + result.outcome = protos.AitoolsAutoInstallOutcomeInstalled + } + return result, nil +} + +// changedAgents names the agents an update changed: those with skills when a +// skill version changed, and those whose plugin version changed. The release +// is not compared: it advances even when every outdated skill was kept. +func changedAgents(ctx context.Context, before, after *installer.InstallState) []string { + skillsChanged := !maps.Equal(before.Skills, after.Skills) + var names []string + for _, e := range buildAgentEntries(ctx, map[string]*installer.InstallState{installer.ScopeGlobal: after}) { + info, ok := e.Installed[installer.ScopeGlobal] + if !ok { + continue + } + changed := skillsChanged + if info.Delivery == deliveryPlugin.String() { + changed = before.Plugins[e.Name].Version != after.Plugins[e.Name].Version + } + if changed { + names = append(names, e.DisplayName) + } + } + return names +} + +// quietContext discards what the installer prints and logs its warnings at +// debug level, so auto-install shows only its notice and its failures appear +// only in the debug log. +func quietContext(ctx context.Context) context.Context { + ctx = cmdio.InContext(ctx, cmdio.NewIO(ctx, flags.OutputText, nil, io.Discard, io.Discard, "", "")) + return log.NewContext(ctx, slog.New(debugOnlyHandler{log.GetLogger(ctx).Handler()})) +} diff --git a/cmd/aitools/telemetry.go b/cmd/aitools/telemetry.go index 4e451feb6eb..25d9be40308 100644 --- a/cmd/aitools/telemetry.go +++ b/cmd/aitools/telemetry.go @@ -190,6 +190,7 @@ func (h debugOnlyHandler) WithGroup(name string) slog.Handler { type installOpts struct { Scope string Experimental bool + Trigger protos.AitoolsInstallTrigger } // logInstallEvent buffers an install event; cmd/root uploads it at exit. @@ -205,6 +206,7 @@ func logInstallEvent(ctx context.Context, plan []agentPlanItem, opts installOpts Experimental: opts.Experimental, ErrorCategory: errCategory, AgentResults: agentResultsField(outcomes), + Trigger: opts.Trigger, }, }) } diff --git a/cmd/aitools/update.go b/cmd/aitools/update.go index bdddafbc56c..8ab71a87e65 100644 --- a/cmd/aitools/update.go +++ b/cmd/aitools/update.go @@ -42,9 +42,7 @@ type pluginMigrationResult struct { } func NewUpdateCmd() *cobra.Command { - var check, force, noNew, noPrune bool - var skillsFlag, scopeFlag string - var projectFlag, globalFlag bool + var f updateFlags cmd := &cobra.Command{ Use: "update", @@ -58,212 +56,237 @@ preview what would change without downloading. The command exits with code 1 when every update or check it attempts fails.`, Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { - ctx := cmd.Context() + return runUpdate(cmd.Context(), f) + }, + } + + cmd.Flags().BoolVar(&f.check, "check", false, "Show what would be updated without downloading") + cmd.Flags().BoolVar(&f.force, "force", false, "Re-download even if versions match") + cmd.Flags().BoolVar(&f.noNew, "no-new", false, "Don't auto-install new skills from manifest") + cmd.Flags().BoolVar(&f.noPrune, "no-prune", false, "Keep skills that vanished from the manifest instead of removing them") + cmd.Flags().StringVar(&f.skills, "skills", "", "Specific skills to update (comma-separated)") + cmd.Flags().StringVar(&f.scope, "scope", "", "Update scope: project, global, or both") + cmd.Flags().BoolVar(&f.project, "project", false, "Update project-scoped skills") + cmd.Flags().BoolVar(&f.global, "global", false, "Update globally-scoped skills") + markScopeBoolsDeprecated(cmd) + return instrumentCommand(cmd, protos.AitoolsCommandUpdate) +} + +// updateFlags are the flags of `databricks aitools update`. +type updateFlags struct { + check, force, noNew, noPrune bool + skills, scope string + project, global bool + // keepModified is not a flag: auto-install sets it (see + // installer.UpdateOptions.KeepModified). It also keeps the raw skills of + // agents that have, or could get, the plugin. + keepModified bool +} + +// runUpdate is the body of `databricks aitools update`. Auto-install runs it +// too, to update an existing installation. +func runUpdate(ctx context.Context, f updateFlags) error { + globalDir, err := installer.GlobalSkillsDir(ctx) + if err != nil { + return err + } + projectDir, err := installer.ProjectSkillsDir(ctx) + if err != nil { + return err + } + + projectFlag, globalFlag, err := parseScopeFlag(f.scope, f.project, f.global, true) + if err != nil { + return withCategory(err, protos.AitoolsErrorCategoryInvalidFlags) + } + // Record a scope flag before resolving, so a scope error still reports it. + funnel := funnelFrom(ctx) + funnel.setScope(explicitScope(projectFlag, globalFlag)) + + scopes, err := resolveScopeForUpdate(ctx, projectFlag, globalFlag, globalDir, projectDir) + if err != nil { + return err + } + // The scope enum has no "both"; a run over both scopes reports none. + if len(scopes) == 1 { + funnel.setScope(scopes[0]) + } + + installed := agents.DetectInstalled(ctx) + src := &installer.GitHubManifestSource{} + skills := splitAndTrim(f.skills) + + // The plugin version recorded on update tracks the skills release. + // Tolerate a resolution error (skip plugin updates, still do skills). + ref, _, refErr := updateGetSkillsRefFn(ctx) + + // A failed or skipped plugin update, or a check whose manifest could + // not be fetched, is only a warning, unless every attempt failed and + // nothing else ran. A plugin record for an agent this CLI does not know + // is not an attempt: no command can act on it. + // firstErr is the first failure of any kind; firstAttemptErr the first + // one of an attempt (a record of an unknown agent is not one). + var firstErr, firstAttemptErr error + updatedAnything := false + + for _, scope := range scopes { + if len(scopes) > 1 { + cmdio.LogString(ctx, fmt.Sprintf("Updating %s skills...", scope)) + } - globalDir, err := installer.GlobalSkillsDir(ctx) + dir := globalDir + if scope == installer.ScopeProject { + dir = projectDir + } + state, err := installer.LoadState(dir) + if err != nil { + return err + } + // A scope with something installed fetches releases and runs agent + // CLIs, so the run waits for the telemetry auth. Without state the + // run only reports that nothing (or a legacy install) is there. + if state != nil { + waitForTelemetryAuth(ctx) + } + + if f.check && refErr == nil && state != nil && len(state.Plugins) > 0 { + printPluginCheckResults(ctx, state, installer.DisplaySkillsVersion(ref)) + updatedAnything = true + } + + // With keepModified (auto-install), raw skills are not migrated to + // the plugin: the migration removes them without checking whether + // the user modified them, and the agent would load the plugin's + // copies instead. + var migrationCandidates []pluginMigrationCandidate + if !f.keepModified { + migrationCandidates, err = pluginMigrationCandidatesForScope(ctx, scope, state) if err != nil { return err } - projectDir, err := installer.ProjectSkillsDir(ctx) + } + if f.check { + printPluginMigrationCheckResults(ctx, migrationCandidates) + } + + // Without a ref the recorded plugins are skipped (with --check, + // not checked). + if refErr != nil && state != nil && hasKnownPluginRecord(state) { + firstAttemptErr = cmp.Or(firstAttemptErr, refErr) + firstErr = cmp.Or(firstErr, refErr) + } + + // Update plugin agents through their own CLI (check mode skips this). + if !f.check && refErr == nil { + pluginUpdates, err := updatePluginsFn(ctx, scope, ref) if err != nil { return err } - - projectFlag, globalFlag, err := parseScopeFlag(scopeFlag, projectFlag, globalFlag, true) - if err != nil { - return withCategory(err, protos.AitoolsErrorCategoryInvalidFlags) + for _, pu := range pluginUpdates { + if pu.Err != nil { + if !errors.Is(pu.Err, installer.ErrUnknownAgent) { + firstAttemptErr = cmp.Or(firstAttemptErr, pu.Err) + } + firstErr = cmp.Or(firstErr, pu.Err) + continue + } + updatedAnything = true + cmdio.LogString(ctx, fmt.Sprintf(" %s databricks plugin %s", pu.Agent, versionToken(pu.Version))) } - // Record a scope flag before resolving, so a scope error still reports it. - funnel := funnelFrom(ctx) - funnel.setScope(explicitScope(projectFlag, globalFlag)) - scopes, err := resolveScopeForUpdate(ctx, projectFlag, globalFlag, globalDir, projectDir) + migrated, err := migrateLegacyRawSkillsToPlugins(ctx, scope, ref, migrationCandidates) if err != nil { return err } - // The scope enum has no "both"; a run over both scopes reports none. - if len(scopes) == 1 { - funnel.setScope(scopes[0]) - } - - installed := agents.DetectInstalled(ctx) - src := &installer.GitHubManifestSource{} - skills := splitAndTrim(skillsFlag) - - // The plugin version recorded on update tracks the skills release. - // Tolerate a resolution error (skip plugin updates, still do skills). - ref, _, refErr := updateGetSkillsRefFn(ctx) - - // A failed or skipped plugin update, or a check whose manifest could - // not be fetched, is only a warning, unless every attempt failed and - // nothing else ran. A plugin record for an agent this CLI does not know - // is not an attempt: no command can act on it. - // firstErr is the first failure of any kind; firstAttemptErr the first - // one of an attempt (a record of an unknown agent is not one). - var firstErr, firstAttemptErr error - updatedAnything := false - - for _, scope := range scopes { - if len(scopes) > 1 { - cmdio.LogString(ctx, fmt.Sprintf("Updating %s skills...", scope)) - } - - dir := globalDir - if scope == installer.ScopeProject { - dir = projectDir - } - state, err := installer.LoadState(dir) + if migrated { + updatedAnything = true + state, err = installer.LoadState(dir) if err != nil { return err } - // A scope with something installed fetches releases and runs agent - // CLIs, so the run waits for the telemetry auth. Without state the - // run only reports that nothing (or a legacy install) is there. - if state != nil { - waitForTelemetryAuth(ctx) - } + } + } - if check && refErr == nil && state != nil && len(state.Plugins) > 0 { - printPluginCheckResults(ctx, state, installer.DisplaySkillsVersion(ref)) - updatedAnything = true - } + // Reconcile file skills for non-plugin agents. Skip entirely for a + // pure-plugin install (no file skills to maintain); keep calling for + // no-state installs so the "no skills or plugins installed" guidance still fires. + if state == nil || len(state.Skills) > 0 { + opts := installer.UpdateOptions{ + Check: f.check, + Force: f.force, + NoNew: f.noNew, + NoPrune: f.noPrune, + Scope: scope, + + KeepModified: f.keepModified, + } + opts.Skills = skills - migrationCandidates, err := pluginMigrationCandidatesForScope(ctx, scope, state) + targetAgents := installed + if scope == installer.ScopeProject { + cwd, err := os.Getwd() if err != nil { return err } - if check { - printPluginMigrationCheckResults(ctx, migrationCandidates) - } - - // Without a ref the recorded plugins are skipped (with --check, - // not checked). - if refErr != nil && state != nil && hasKnownPluginRecord(state) { - firstAttemptErr = cmp.Or(firstAttemptErr, refErr) - firstErr = cmp.Or(firstErr, refErr) - } - - // Update plugin agents through their own CLI (check mode skips this). - if !check && refErr == nil { - pluginUpdates, err := updatePluginsFn(ctx, scope, ref) - if err != nil { - return err - } - for _, pu := range pluginUpdates { - if pu.Err != nil { - if !errors.Is(pu.Err, installer.ErrUnknownAgent) { - firstAttemptErr = cmp.Or(firstAttemptErr, pu.Err) - } - firstErr = cmp.Or(firstErr, pu.Err) - continue - } - updatedAnything = true - cmdio.LogString(ctx, fmt.Sprintf(" %s databricks plugin %s", pu.Agent, versionToken(pu.Version))) - } + targetAgents = mergeAgents(installed, agents.DetectProjectInstalled(cwd)) + } - migrated, err := migrateLegacyRawSkillsToPlugins(ctx, scope, ref, migrationCandidates) - if err != nil { - return err - } - if migrated { - updatedAnything = true - state, err = installer.LoadState(dir) - if err != nil { - return err - } + fileAgents, manifestPlugins := excludePluginAgents(ctx, scope, targetAgents, state) + // Raw skills an earlier install gave an agent that now has the + // plugin would show every skill twice, so remove them as + // install does when it keeps the plugin. Auto-install leaves + // them, as it does not migrate them either. + if !f.check && !f.keepModified { + for _, a := range manifestPlugins { + if err := updateCleanupLegacyFn(ctx, a, scope); err != nil { + log.Debugf(ctx, "Legacy skill cleanup for %s failed: %v", a.DisplayName, err) } } + } + // The plugin agents left no agent that can take file skills in + // this scope, so there is nothing to reconcile. Without this, a + // project whose only project-capable agent has the plugin would + // fail the update with "no agents support project-scoped skills". + // Without install state the reconcile still runs, for the + // "no skills or plugins installed" and legacy-install guidance. + if state != nil && len(manifestPlugins) > 0 && !slices.ContainsFunc(fileAgents, func(a *agents.Agent) bool { + return scope != installer.ScopeProject || a.SupportsProjectScope + }) { + continue + } - // Reconcile file skills for non-plugin agents. Skip entirely for a - // pure-plugin install (no file skills to maintain); keep calling for - // no-state installs so the "no skills or plugins installed" guidance still fires. - if state == nil || len(state.Skills) > 0 { - opts := installer.UpdateOptions{ - Check: check, - Force: force, - NoNew: noNew, - NoPrune: noPrune, - Scope: scope, - } - opts.Skills = skills - - targetAgents := installed - if scope == installer.ScopeProject { - cwd, err := os.Getwd() - if err != nil { - return err - } - targetAgents = mergeAgents(installed, agents.DetectProjectInstalled(cwd)) - } - - fileAgents, manifestPlugins := excludePluginAgents(ctx, scope, targetAgents, state) - // Raw skills an earlier install gave an agent that now has the - // plugin would show every skill twice, so remove them as - // install does when it keeps the plugin. - if !check { - for _, a := range manifestPlugins { - if err := updateCleanupLegacyFn(ctx, a, scope); err != nil { - log.Debugf(ctx, "Legacy skill cleanup for %s failed: %v", a.DisplayName, err) - } - } - } - // The plugin agents left no agent that can take file skills in - // this scope, so there is nothing to reconcile. Without this, a - // project whose only project-capable agent has the plugin would - // fail the update with "no agents support project-scoped skills". - // Without install state the reconcile still runs, for the - // "no skills or plugins installed" and legacy-install guidance. - if state != nil && len(manifestPlugins) > 0 && !slices.ContainsFunc(fileAgents, func(a *agents.Agent) bool { - return scope != installer.ScopeProject || a.SupportsProjectScope - }) { - continue - } - - result, err := updateSkillsFn(ctx, src, fileAgents, opts) - if err != nil { - return err - } - if result != nil && result.CheckErr != nil { - firstAttemptErr = cmp.Or(firstAttemptErr, result.CheckErr) - firstErr = cmp.Or(firstErr, result.CheckErr) - } else { - updatedAnything = true - } - // A check that could not fetch the manifest has nothing to report. - if result != nil && result.CheckErr == nil && (check || len(result.Updated) > 0 || len(result.Added) > 0 || len(result.Removed) > 0) { - text := installer.FormatUpdateResult(result, check) - if text != "No changes." || len(migrationCandidates) == 0 { - cmdio.LogString(ctx, text) - } - } - } + result, err := updateSkillsFn(ctx, src, fileAgents, opts) + if err != nil { + return err } - if firstErr != nil && !updatedAnything { - if firstAttemptErr == nil { - // Only unknown-agent records: warned about above, but not a run - // that failed to do what it could. - funnel.finishWith(ctx, protos.AitoolsCommandOutcomeFailure, classifyInstallError(firstErr)) - return nil - } - if check { - return fmt.Errorf("nothing was checked: %w", firstAttemptErr) + if result != nil && result.CheckErr != nil { + firstAttemptErr = cmp.Or(firstAttemptErr, result.CheckErr) + firstErr = cmp.Or(firstErr, result.CheckErr) + } else { + updatedAnything = true + } + // A check that could not fetch the manifest has nothing to report. + if result != nil && result.CheckErr == nil && (f.check || len(result.Updated) > 0 || len(result.Added) > 0 || len(result.Removed) > 0) { + text := installer.FormatUpdateResult(result, f.check) + if text != "No changes." || len(migrationCandidates) == 0 { + cmdio.LogString(ctx, text) } - return fmt.Errorf("nothing was updated: %w", firstAttemptErr) } + } + } + if firstErr != nil && !updatedAnything { + if firstAttemptErr == nil { + // Only unknown-agent records: warned about above, but not a run + // that failed to do what it could. + funnel.finishWith(ctx, protos.AitoolsCommandOutcomeFailure, classifyInstallError(firstErr)) return nil - }, + } + if f.check { + return fmt.Errorf("nothing was checked: %w", firstAttemptErr) + } + return fmt.Errorf("nothing was updated: %w", firstAttemptErr) } - - cmd.Flags().BoolVar(&check, "check", false, "Show what would be updated without downloading") - cmd.Flags().BoolVar(&force, "force", false, "Re-download even if versions match") - cmd.Flags().BoolVar(&noNew, "no-new", false, "Don't auto-install new skills from manifest") - cmd.Flags().BoolVar(&noPrune, "no-prune", false, "Keep skills that vanished from the manifest instead of removing them") - cmd.Flags().StringVar(&skillsFlag, "skills", "", "Specific skills to update (comma-separated)") - cmd.Flags().StringVar(&scopeFlag, "scope", "", "Update scope: project, global, or both") - cmd.Flags().BoolVar(&projectFlag, "project", false, "Update project-scoped skills") - cmd.Flags().BoolVar(&globalFlag, "global", false, "Update globally-scoped skills") - markScopeBoolsDeprecated(cmd) - return instrumentCommand(cmd, protos.AitoolsCommandUpdate) + return nil } func pluginMigrationCandidatesForScope(ctx context.Context, scope string, state *installer.InstallState) ([]pluginMigrationCandidate, error) { diff --git a/cmd/apps/init.go b/cmd/apps/init.go index ed79271d25f..5e704e53e4c 100644 --- a/cmd/apps/init.go +++ b/cmd/apps/init.go @@ -18,6 +18,7 @@ import ( "github.com/charmbracelet/huh" "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/aitools/agents" + "github.com/databricks/cli/libs/aitools/autoinstall" "github.com/databricks/cli/libs/aitools/installer" "github.com/databricks/cli/libs/apps/generator" "github.com/databricks/cli/libs/apps/initializer" @@ -1964,13 +1965,20 @@ func runPostCreateDeploy(ctx context.Context, profile string) error { args = append(args, "--profile", profile) } cmd := exec.CommandContext(ctx, executable, args...) - cmd.Env = agents.SelfExecEnv(os.Environ()) + cmd.Env = selfExecEnv(os.Environ()) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr cmd.Stdin = os.Stdin return cmd.Run() } +// selfExecEnv returns environ for a CLI process apps init starts: it prints no +// coding-agent hint (the parent's output already has one), and it does not +// auto-install when the user turned the tools down in this process. +func selfExecEnv(environ []string) []string { + return agents.SelfExecEnv(autoinstall.ChildEnv(environ)) +} + // runPostCreateDev runs the dev or dev-remote command in the current directory. func runPostCreateDev(ctx context.Context, mode prompt.RunMode, projectInit initializer.Initializer, workDir, profile string) error { switch mode { @@ -1993,7 +2001,7 @@ func runPostCreateDev(ctx context.Context, mode prompt.RunMode, projectInit init args = append(args, "--profile", profile) } cmd := exec.CommandContext(ctx, executable, args...) - cmd.Env = agents.SelfExecEnv(os.Environ()) + cmd.Env = selfExecEnv(os.Environ()) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr cmd.Stdin = os.Stdin diff --git a/cmd/apps/init_test.go b/cmd/apps/init_test.go index 013e3a9253a..3551102727a 100644 --- a/cmd/apps/init_test.go +++ b/cmd/apps/init_test.go @@ -13,6 +13,7 @@ import ( "strings" "testing" + "github.com/databricks/cli/libs/aitools/autoinstall" "github.com/databricks/cli/libs/apps/manifest" "github.com/databricks/cli/libs/apps/pkgmanager" "github.com/databricks/cli/libs/apps/prompt" @@ -1810,3 +1811,17 @@ func TestPackageManagerValidation(t *testing.T) { }) } } + +func TestSelfExecEnvCarriesBothMarkers(t *testing.T) { + t.Cleanup(autoinstall.ResetDeclined) + ctx := env.WithUserHomeDir(t.Context(), t.TempDir()) + + // The child prints no coding-agent hint, as before a "no" was given. + assert.Equal(t, []string{"HOME=/home/user", "DATABRICKS_CLI_SELF_EXEC=1"}, selfExecEnv([]string{"HOME=/home/user"})) + + // After a "no", the child also does not auto-install. + require.NoError(t, autoinstall.Decline(ctx)) + environ := selfExecEnv([]string{"HOME=/home/user"}) + assert.Contains(t, environ, "DATABRICKS_CLI_SELF_EXEC=1") + assert.Contains(t, environ, autoinstall.EnvVar+"=false") +} diff --git a/cmd/root/autoinstall_test.go b/cmd/root/autoinstall_test.go new file mode 100644 index 00000000000..86b25091a65 --- /dev/null +++ b/cmd/root/autoinstall_test.go @@ -0,0 +1,62 @@ +package root_test + +import ( + "context" + "errors" + "io" + "testing" + + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/aitools/autoinstall" + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// setAutoInstallRun replaces autoinstall.Run for the test. The variable is +// global, so tests that call it must not run in parallel. +func setAutoInstallRun(t *testing.T, run func(*cobra.Command)) { + prev := autoinstall.Run + autoinstall.Run = run + t.Cleanup(func() { autoinstall.Run = prev }) +} + +func newAutoInstallTestRoot(t *testing.T) *root.Tree { + r := root.NewTree(t.Context()) + r.SetOut(io.Discard) + r.SetErr(io.Discard) + r.Add(root.CommandSpec{ + Use: "jobs", + Short: "jobs commands", + Build: func(ctx context.Context, cmd *cobra.Command) { + cmd.AddCommand(&cobra.Command{Use: "ok", RunE: func(*cobra.Command, []string) error { return nil }}) + cmd.AddCommand(&cobra.Command{Use: "fail", RunE: func(*cobra.Command, []string) error { return errors.New("failed") }}) + }, + }) + return r +} + +func TestPostRunCallsAutoInstallWithTheCommand(t *testing.T) { + var ran []string + setAutoInstallRun(t, func(cmd *cobra.Command) { ran = append(ran, cmd.CommandPath()) }) + + err := newAutoInstallTestRoot(t).Execute(t.Context(), []string{"jobs", "ok"}) + require.NoError(t, err) + assert.Equal(t, []string{"databricks jobs ok"}, ran) +} + +func TestPostRunSkipsAutoInstallAfterAFailedCommand(t *testing.T) { + var ran []string + setAutoInstallRun(t, func(cmd *cobra.Command) { ran = append(ran, cmd.CommandPath()) }) + + err := newAutoInstallTestRoot(t).Execute(t.Context(), []string{"jobs", "fail"}) + require.Error(t, err) + assert.Empty(t, ran) +} + +func TestPostRunWithoutAutoInstall(t *testing.T) { + setAutoInstallRun(t, nil) + + err := newAutoInstallTestRoot(t).Execute(t.Context(), []string{"jobs", "ok"}) + assert.NoError(t, err) +} diff --git a/cmd/root/root.go b/cmd/root/root.go index a988c22d285..5fc80ec87a4 100644 --- a/cmd/root/root.go +++ b/cmd/root/root.go @@ -13,6 +13,7 @@ import ( "github.com/databricks/cli/internal/build" "github.com/databricks/cli/libs/aitools/agents" + "github.com/databricks/cli/libs/aitools/autoinstall" "github.com/databricks/cli/libs/aitools/installer" "github.com/databricks/cli/libs/auth" "github.com/databricks/cli/libs/cmdctx" @@ -102,6 +103,12 @@ func newRootCommand(ctx context.Context) *cobra.Command { cmd.PersistentPostRunE = func(cmd *cobra.Command, args []string) error { // Wait for any active Bubble Tea programs to finish and restore terminal state cmdio.Wait(cmd.Context()) + + // On the first run of a new CLI version, install Databricks AI tools. + // Cobra runs post-run hooks only when the command succeeded. + if autoinstall.Run != nil { + autoinstall.Run(cmd) + } return nil } diff --git a/libs/aitools/agents/detect.go b/libs/aitools/agents/detect.go index 53ae8d085f8..cb4ba22cfb4 100644 --- a/libs/aitools/agents/detect.go +++ b/libs/aitools/agents/detect.go @@ -112,7 +112,7 @@ func (a *Agent) ProbePlugin(ctx context.Context) error { cctx, cancel := context.WithTimeout(ctx, pluginProbeTimeout) defer cancel() - if _, err := process.Background(cctx, []string{path, "plugin", "--help"}); err != nil { + if _, err := process.BackgroundInGroup(cctx, []string{path, "plugin", "--help"}); err != nil { return fmt.Errorf("%s plugin subcommand unavailable: %w", a.Binary, errors.Join(ErrPluginUnsupported, err)) } return nil diff --git a/libs/aitools/agents/hint.go b/libs/aitools/agents/hint.go index 9045dddcdf1..9add2a607d4 100644 --- a/libs/aitools/agents/hint.go +++ b/libs/aitools/agents/hint.go @@ -213,7 +213,7 @@ func MaybeHint(ctx context.Context, cmd *cobra.Command, pluginRecorded PluginRec if isHintExemptCommand(cmd) { return } - if isCredentialHelperCommand(cmd) { + if IsCredentialHelperCommand(cmd) { return } if env.Get(ctx, selfExecEnv) != "" { @@ -282,7 +282,7 @@ func (h *agentHint) claim(ctx context.Context, key string) bool { // stdin even when the output goes through a pipe (`2>&1 | less`), so any // terminal means a human may read the tag raw. func claudeNativeHintLine(ctx context.Context) string { - if env.Get(ctx, claudeChildSessionEnv) == "" || inCI(ctx) { + if env.Get(ctx, claudeChildSessionEnv) == "" || InCI(ctx) { return "" } if cmdio.HasTerminal(ctx) { @@ -291,13 +291,13 @@ func claudeNativeHintLine(ctx context.Context) string { return claudeNativeHint } -// inCI reports whether the CLI runs in CI: the generic CI variable set to +// InCI reports whether the CLI runs in CI: the generic CI variable set to // anything but an explicit false (some providers set it to their own name), or // any provider the SDK detects for its user agent (e.g. Azure Pipelines' // TF_BUILD, which doesn't set CI). The false values are the ones env.GetBool // accepts; GetBool itself can't be used because it also maps a provider name // to false. -func inCI(ctx context.Context) bool { +func InCI(ctx context.Context) bool { switch strings.ToLower(env.Get(ctx, ciEnv)) { case "", "0", "f", "false", "no", "off": default: @@ -306,9 +306,9 @@ func inCI(ctx context.Context) bool { return useragent.CiCdProvider() != "" } -// isCredentialHelperCommand reports whether cmd is `databricks auth token` or +// IsCredentialHelperCommand reports whether cmd is `databricks auth token` or // `databricks auth docker token`. -func isCredentialHelperCommand(cmd *cobra.Command) bool { +func IsCredentialHelperCommand(cmd *cobra.Command) bool { if cmd.Name() != tokenCommandName { return false } diff --git a/libs/aitools/agents/recommend.go b/libs/aitools/agents/recommend.go index ea66ebe4016..31d3b94f064 100644 --- a/libs/aitools/agents/recommend.go +++ b/libs/aitools/agents/recommend.go @@ -3,6 +3,7 @@ package agents import ( "context" + "github.com/databricks/cli/libs/aitools/autoinstall" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/dbr" "github.com/databricks/cli/libs/log" @@ -53,7 +54,8 @@ func MaybeVersionCheckTip(ctx context.Context, installedPlugins func(context.Con // RecommendSkillsInstall checks if coding agents are detected without Databricks tooling (skills // or the databricks plugin) installed. -// In interactive mode, prompts the user to install now using installFn. In non-interactive mode, prints a hint. +// In interactive mode, prompts the user to install now using installFn; a "no" also turns +// auto-install off. In non-interactive mode, prints a hint. func RecommendSkillsInstall(ctx context.Context, installFn func(context.Context) error) error { if HasDatabricksToolingInstalled(ctx) { return nil @@ -70,6 +72,14 @@ func RecommendSkillsInstall(ctx context.Context, installFn func(context.Context) return err } if !yes { + // The automatic install after this command (and after later upgrades) + // must not install what the user just turned down. + if err := autoinstall.Decline(ctx); err != nil { + log.Warnf(ctx, "Could not turn off the automatic installation of Databricks AI tools: %v", err) + return nil + } + // The prompt asked about now; say that the answer also holds later. + cmdio.LogString(ctx, "Databricks AI tools won't be installed automatically either. To turn that back on, run: databricks aitools auto-install enable") return nil } diff --git a/libs/aitools/agents/recommend_test.go b/libs/aitools/agents/recommend_test.go index aaf978bb9d1..5430b147c0b 100644 --- a/libs/aitools/agents/recommend_test.go +++ b/libs/aitools/agents/recommend_test.go @@ -8,6 +8,7 @@ import ( "testing" "github.com/databricks/cli/internal/testutil" + "github.com/databricks/cli/libs/aitools/autoinstall" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/dbr" "github.com/databricks/cli/libs/telemetry" @@ -100,6 +101,7 @@ func TestRecommendSkillsInstallNonInteractive(t *testing.T) { } func TestRecommendSkillsInstallInteractiveDecline(t *testing.T) { + t.Cleanup(autoinstall.ResetDeclined) tmpDir := t.TempDir() t.Setenv("HOME", tmpDir) t.Setenv("USERPROFILE", tmpDir) @@ -119,7 +121,11 @@ func TestRecommendSkillsInstallInteractiveDecline(t *testing.T) { ctx = telemetry.WithNewLogger(ctx) // Drain stderr so the prompt write doesn't block. - go func() { _, _ = io.Copy(io.Discard, testIO.Stderr) }() + stderr := make(chan string, 1) + go func() { + b, _ := io.ReadAll(testIO.Stderr) + stderr <- string(b) + }() errc := make(chan error, 1) go func() { @@ -133,6 +139,14 @@ func TestRecommendSkillsInstallInteractiveDecline(t *testing.T) { assert.NoError(t, <-errc) // The prompt is logged when shown, so a declined prompt still counts. assert.Equal(t, []protos.AitoolsHintKind{protos.AitoolsHintKindAppsInitPrompt}, hintKinds(t, ctx)) + // The "no" also keeps the automatic install from installing. + state, err := autoinstall.Load(ctx) + require.NoError(t, err) + assert.True(t, state.OptedOut) + assert.True(t, autoinstall.Declined()) + // The user is told that the answer also holds for later installs. + testIO.Done() + assert.Contains(t, <-stderr, "databricks aitools auto-install enable") } func TestMaybeVersionCheckTip(t *testing.T) { diff --git a/libs/aitools/autoinstall/decide.go b/libs/aitools/autoinstall/decide.go new file mode 100644 index 00000000000..0b4d2832a48 --- /dev/null +++ b/libs/aitools/autoinstall/decide.go @@ -0,0 +1,127 @@ +package autoinstall + +import ( + "github.com/databricks/cli/internal/build" + "golang.org/x/mod/semver" +) + +const ( + // MaxAttempts is how many failed attempts one CLI version gets. A new + // version starts over. + MaxAttempts = 10 + + // HoldoutPercent is the share of machines (by Bucket) that never + // auto-install, so the install rate can be compared with a control group. + HoldoutPercent = 10 +) + +// Reason is what Decide concluded. The reasons that are reported are also the +// outcome values of the auto-install telemetry event. +type Reason string + +const ( + // Not due: the run records and reports nothing. + ReasonSkippedCommand Reason = "skipped_command" + ReasonOnRuntime Reason = "on_runtime" + ReasonCacheDisabled Reason = "cache_disabled" + ReasonDevelopmentBuild Reason = "development_build" + ReasonAlreadyRan Reason = "already_ran" + ReasonRetriesExhausted Reason = "retries_exhausted" + + // Settle the version without installing. All but ReasonGateOff are + // reported: the feature is not launched while the gate is off. + ReasonSkippedCI Reason = "skipped_ci" + ReasonSkippedOptOut Reason = "skipped_optout" + ReasonGateOff Reason = "gate_off" + ReasonSkippedHoldout Reason = "skipped_holdout" + + // Install, or update an existing installation. + ReasonRun Reason = "run" +) + +// Due reports whether the run acts on the decision: it installs, or it +// settles the version. A run that is not due changes nothing. +func (r Reason) Due() bool { + switch r { + case ReasonSkippedCommand, ReasonOnRuntime, ReasonCacheDisabled, ReasonDevelopmentBuild, ReasonAlreadyRan, ReasonRetriesExhausted: + return false + case ReasonSkippedCI, ReasonSkippedOptOut, ReasonGateOff, ReasonSkippedHoldout, ReasonRun: + return true + } + return true +} + +// Inputs is everything Decide depends on. +type Inputs struct { + // SkippedCommand is set for a command that never triggers auto-install. + SkippedCommand bool + + // OnRuntime is set on Databricks Runtime, where the home directory does + // not outlive the cluster and the CLI version is the platform's. + OnRuntime bool + + // CacheDisabled is DATABRICKS_CACHE_ENABLED=false. + CacheDisabled bool + + // Version is the running CLI version. + Version string + + // State is the persisted state: last version, attempts and opt-out. + State State + + // CI is set when the CLI runs in continuous integration. + CI bool + + // EnvOptOut is DATABRICKS_AITOOLS_AUTO_INSTALL=false, or a "no" earlier in + // this process (see Declined). + EnvOptOut bool + + // GateOn is the remote gate for Version. + GateOn bool + + // Bucket is the machine's stable holdout bucket, in [0, 100). + Bucket int +} + +// Decide returns whether this run should install, settle the version, or do +// nothing. The checks run from cheapest to most expensive, so callers can +// leave GateOn false and look the gate up only when Decide returns +// ReasonGateOff. Every skip reason precedes the gate except the holdout: +// the holdout arm only exists once the feature is on. +func Decide(in Inputs) Reason { + switch { + case in.SkippedCommand: + return ReasonSkippedCommand + case in.OnRuntime: + return ReasonOnRuntime + case in.CacheDisabled: + return ReasonCacheDisabled + // Development and snapshot builds change version with every commit, so + // they would trigger on every build. + case build.IsDevelopmentVersion(in.Version): + return ReasonDevelopmentBuild + case in.State.LastVersion == in.Version && in.State.Attempts >= MaxAttempts: + return ReasonRetriesExhausted + case in.State.LastVersion == in.Version && in.State.Attempts == 0: + return ReasonAlreadyRan + // A newer CLI already ran here, e.g. an IDE ships its own older CLI next + // to the user's: alternating between the two must not run on every + // command. + case in.State.LastVersion != "" && semver.Compare("v"+in.Version, "v"+in.State.LastVersion) < 0: + return ReasonAlreadyRan + case in.CI: + return ReasonSkippedCI + case in.EnvOptOut, in.State.OptedOut: + return ReasonSkippedOptOut + case !in.GateOn: + return ReasonGateOff + case InHoldout(in.Bucket): + return ReasonSkippedHoldout + } + return ReasonRun +} + +// InHoldout reports whether a machine in bucket belongs to the holdout. +func InHoldout(bucket int) bool { + return bucket < HoldoutPercent +} diff --git a/libs/aitools/autoinstall/decide_test.go b/libs/aitools/autoinstall/decide_test.go new file mode 100644 index 00000000000..352f78637b1 --- /dev/null +++ b/libs/aitools/autoinstall/decide_test.go @@ -0,0 +1,76 @@ +package autoinstall + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestDecide(t *testing.T) { + // eligible is a run of a released version never seen before, with the gate on, in the treatment arm. + eligible := Inputs{ + Version: "1.30.0", + GateOn: true, + Bucket: HoldoutPercent, + } + tests := []struct { + name string + modify func(in *Inputs) + want Reason + }{ + {name: "eligible", modify: func(in *Inputs) {}, want: ReasonRun}, + + {name: "skipped command", modify: func(in *Inputs) { in.SkippedCommand = true }, want: ReasonSkippedCommand}, + {name: "Databricks Runtime", modify: func(in *Inputs) { in.OnRuntime = true }, want: ReasonOnRuntime}, + + {name: "cache disabled", modify: func(in *Inputs) { in.CacheDisabled = true }, want: ReasonCacheDisabled}, + {name: "development build", modify: func(in *Inputs) { in.Version = "1.31.0-dev+abcdef123456" }, want: ReasonDevelopmentBuild}, + {name: "snapshot build", modify: func(in *Inputs) { in.Version = "1.30.1-dev+abcdef1" }, want: ReasonDevelopmentBuild}, + + {name: "same version, settled", modify: func(in *Inputs) { in.State.LastVersion = "1.30.0" }, want: ReasonAlreadyRan}, + {name: "same version, failed before", modify: func(in *Inputs) { in.State = State{LastVersion: "1.30.0", Attempts: 3} }, want: ReasonRun}, + {name: "same version, last attempt left", modify: func(in *Inputs) { in.State = State{LastVersion: "1.30.0", Attempts: MaxAttempts - 1} }, want: ReasonRun}, + {name: "same version, retries exhausted", modify: func(in *Inputs) { in.State = State{LastVersion: "1.30.0", Attempts: MaxAttempts} }, want: ReasonRetriesExhausted}, + {name: "new version after retries exhausted", modify: func(in *Inputs) { in.State = State{LastVersion: "1.29.0", Attempts: MaxAttempts} }, want: ReasonRun}, + {name: "older CLI after a newer one ran", modify: func(in *Inputs) { in.State.LastVersion = "1.31.0" }, want: ReasonAlreadyRan}, + {name: "older CLI while a newer one retries", modify: func(in *Inputs) { in.State = State{LastVersion: "1.31.0", Attempts: 2} }, want: ReasonAlreadyRan}, + {name: "newer CLI after an older one ran", modify: func(in *Inputs) { in.State.LastVersion = "1.29.5" }, want: ReasonRun}, + + {name: "CI", modify: func(in *Inputs) { in.CI = true }, want: ReasonSkippedCI}, + {name: "CI before the opt-out", modify: func(in *Inputs) { in.CI = true; in.EnvOptOut = true }, want: ReasonSkippedCI}, + {name: "env opt-out", modify: func(in *Inputs) { in.EnvOptOut = true }, want: ReasonSkippedOptOut}, + {name: "persisted opt-out", modify: func(in *Inputs) { in.State.OptedOut = true }, want: ReasonSkippedOptOut}, + {name: "opt-out before the gate", modify: func(in *Inputs) { in.State.OptedOut = true; in.GateOn = false }, want: ReasonSkippedOptOut}, + {name: "gate off", modify: func(in *Inputs) { in.GateOn = false }, want: ReasonGateOff}, + {name: "gate off hides the holdout", modify: func(in *Inputs) { in.GateOn = false; in.Bucket = 0 }, want: ReasonGateOff}, + {name: "holdout", modify: func(in *Inputs) { in.Bucket = HoldoutPercent - 1 }, want: ReasonSkippedHoldout}, + {name: "first bucket is holdout", modify: func(in *Inputs) { in.Bucket = 0 }, want: ReasonSkippedHoldout}, + {name: "last bucket is treatment", modify: func(in *Inputs) { in.Bucket = 99 }, want: ReasonRun}, + + {name: "skipped command before everything", modify: func(in *Inputs) { + in.SkippedCommand = true + in.OnRuntime = true + in.CacheDisabled = true + in.CI = true + }, want: ReasonSkippedCommand}, + {name: "settled version before CI", modify: func(in *Inputs) { in.State.LastVersion = "1.30.0"; in.CI = true }, want: ReasonAlreadyRan}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + in := eligible + tc.modify(&in) + assert.Equal(t, tc.want, Decide(in)) + }) + } +} + +func TestReasonDue(t *testing.T) { + notDue := []Reason{ReasonSkippedCommand, ReasonOnRuntime, ReasonCacheDisabled, ReasonDevelopmentBuild, ReasonAlreadyRan, ReasonRetriesExhausted} + due := []Reason{ReasonSkippedCI, ReasonSkippedOptOut, ReasonGateOff, ReasonSkippedHoldout, ReasonRun} + for _, r := range notDue { + assert.False(t, r.Due(), r) + } + for _, r := range due { + assert.True(t, r.Due(), r) + } +} diff --git a/libs/aitools/autoinstall/hook.go b/libs/aitools/autoinstall/hook.go new file mode 100644 index 00000000000..c573cc7bd42 --- /dev/null +++ b/libs/aitools/autoinstall/hook.go @@ -0,0 +1,10 @@ +package autoinstall + +import "github.com/spf13/cobra" + +// Run, when set, is called by the root command after a command succeeded. It +// runs auto-install, which needs the install and update code in cmd/aitools. +// cmd/root cannot import that package (cmd/aitools imports cmd/root), so +// main.go sets Run. Runs that don't start in main.go, such as in-process +// tests, leave it nil and never auto-install. +var Run func(cmd *cobra.Command) diff --git a/libs/aitools/autoinstall/lock_unix.go b/libs/aitools/autoinstall/lock_unix.go new file mode 100644 index 00000000000..c183be7d109 --- /dev/null +++ b/libs/aitools/autoinstall/lock_unix.go @@ -0,0 +1,21 @@ +//go:build unix + +package autoinstall + +import ( + "errors" + "os" + + "golang.org/x/sys/unix" +) + +// tryLockFile takes an exclusive lock on f, failing at once with errLockHeld +// when another process holds it. The lock is released when f is closed, or by +// the OS when the process exits. +func tryLockFile(f *os.File) error { + err := unix.Flock(int(f.Fd()), unix.LOCK_EX|unix.LOCK_NB) + if errors.Is(err, unix.EWOULDBLOCK) { + return errLockHeld + } + return err +} diff --git a/libs/aitools/autoinstall/lock_windows.go b/libs/aitools/autoinstall/lock_windows.go new file mode 100644 index 00000000000..b49b8e0eeff --- /dev/null +++ b/libs/aitools/autoinstall/lock_windows.go @@ -0,0 +1,22 @@ +//go:build windows + +package autoinstall + +import ( + "errors" + "os" + + "golang.org/x/sys/windows" +) + +// tryLockFile takes an exclusive lock on f, failing at once with errLockHeld +// when another process holds it. The lock is released when f is closed, or by +// the OS when the process exits. +func tryLockFile(f *os.File) error { + flags := uint32(windows.LOCKFILE_EXCLUSIVE_LOCK | windows.LOCKFILE_FAIL_IMMEDIATELY) + err := windows.LockFileEx(windows.Handle(f.Fd()), flags, 0, 1, 0, &windows.Overlapped{}) + if errors.Is(err, windows.ERROR_LOCK_VIOLATION) { + return errLockHeld + } + return err +} diff --git a/libs/aitools/autoinstall/run.go b/libs/aitools/autoinstall/run.go new file mode 100644 index 00000000000..38856ce660a --- /dev/null +++ b/libs/aitools/autoinstall/run.go @@ -0,0 +1,168 @@ +package autoinstall + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "errors" + "fmt" + "os" + "path/filepath" + "time" + + "github.com/databricks/cli/internal/build" + "github.com/databricks/cli/libs/clicompat" + "github.com/databricks/cli/libs/env" + "github.com/databricks/cli/libs/log" +) + +const ( + // versionEnv sets the CLI version auto-install sees. Acceptance tests use + // it to simulate releases and upgrades: their binaries are development + // builds, which never auto-install. Not a documented, user-facing setting. + versionEnv = "DATABRICKS_AITOOLS_AUTO_INSTALL_CLI_VERSION" + + // cacheEnabledEnv mirrors libs/cache's env knob: with the cache turned + // off, auto-install does nothing. + cacheEnabledEnv = "DATABRICKS_CACHE_ENABLED" + + // lockFileName is the lock of a run, held while it installs. + lockFileName = "auto-install.lock" + + // stateLockFileName is the lock of the state file, held while it is read + // and rewritten. + stateLockFileName = "auto-install.json.lock" + + // stateLockWait bounds the wait for the state lock. A live holder keeps it + // only to read or rewrite a small file. + stateLockWait = time.Second + + // lockRetryInterval is how often a held lock is tried again. + lockRetryInterval = 10 * time.Millisecond +) + +// Version returns the CLI version auto-install runs for. +func Version(ctx context.Context) string { + if v := env.Get(ctx, versionEnv); v != "" { + return v + } + return build.GetInfo().Version +} + +// CacheDisabled reports whether DATABRICKS_CACHE_ENABLED turns the CLI's cache +// off. +func CacheDisabled(ctx context.Context) bool { + enabled, ok := env.GetBool(ctx, cacheEnabledEnv) + return ok && !enabled +} + +// GateOn reports whether the compatibility manifest turns auto-install on for +// version. It fails when the manifest could not be fetched: the gate is then +// unknown, not off. +func GateOn(ctx context.Context, version string) (bool, error) { + return clicompat.ResolveAutoInstall(ctx, version) +} + +// NextAttempt returns the number of the attempt a run for version makes after +// prior: 1 for a version the state has not failed for. +func NextAttempt(prior State, version string) int { + if prior.LastVersion == version { + return prior.Attempts + 1 + } + return 1 +} + +// NewID returns a random machine identifier for State.ID. +func NewID() string { + b := make([]byte, 16) + _, _ = rand.Read(b) + return hex.EncodeToString(b) +} + +// Bucket maps a machine identifier to its stable holdout bucket in [0, 100). +func Bucket(id string) int { + sum := sha256.Sum256([]byte("aitools-auto-install:" + id)) + return int(binary.BigEndian.Uint64(sum[:8]) % 100) +} + +// Record stores the outcome of a run for version: attempts is 0 when the +// version is settled, or the number of failed attempts so far. It re-reads the +// state and changes only the fields a run owns, so an opt-out stored while the +// run was installing is kept. +func Record(ctx context.Context, version string, attempts int, id string) error { + return updateState(ctx, func(s *State) { + s.LastVersion = version + s.Attempts = attempts + if attempts == 0 { + s.Installing = false + } + if s.ID == "" { + s.ID = id + } + }) +} + +// MarkInstalling sets State.Installing before a first install starts. +func MarkInstalling(ctx context.Context) error { + return updateState(ctx, func(s *State) { s.Installing = true }) +} + +// TryLock claims the machine-wide auto-install lock so that concurrent CLI +// processes (agents often run several at once) do not install at the same +// time. ok is false when another process holds it or it cannot be taken; that +// run then skips auto-install. It is an OS file lock, so a process that dies +// mid-run releases it. +func TryLock(ctx context.Context) (unlock func(), ok bool) { + unlock, err := lock(ctx, lockFileName, 0) + if err != nil { + log.Debugf(ctx, "auto-install: could not take the lock: %v", err) + return nil, false + } + return unlock, true +} + +// lock takes the exclusive lock file name next to the state. It retries for +// up to wait, so a holder that is stopped (a suspended process, say) delays +// the caller by at most wait, then fails it. The file is never removed: a +// process that removed it could let a third one lock a new file while a +// second still holds the old. +func lock(ctx context.Context, name string, wait time.Duration) (unlock func(), err error) { + path, err := statePath(ctx) + if err != nil { + return nil, err + } + path = filepath.Join(filepath.Dir(path), name) + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return nil, err + } + f, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0o600) + if err != nil { + return nil, err + } + deadline := time.Now().Add(wait) + for { + err = tryLock(f) + if err == nil { + return func() { _ = f.Close() }, nil + } + // Only a lock another process holds can come free; any other error + // (a filesystem without locks, say) fails the same way on every try. + if !errors.Is(err, errLockHeld) { + _ = f.Close() + return nil, fmt.Errorf("lock %s: %w", filepath.ToSlash(path), err) + } + if time.Now().After(deadline) || ctx.Err() != nil { + _ = f.Close() + return nil, fmt.Errorf("%s: %w", filepath.ToSlash(path), err) + } + time.Sleep(lockRetryInterval) + } +} + +// errLockHeld is the error of tryLockFile when another process holds the lock. +var errLockHeld = errors.New("held by another CLI process") + +// tryLock is tryLockFile; tests replace it. +var tryLock = tryLockFile diff --git a/libs/aitools/autoinstall/run_test.go b/libs/aitools/autoinstall/run_test.go new file mode 100644 index 00000000000..b73b51d6eb5 --- /dev/null +++ b/libs/aitools/autoinstall/run_test.go @@ -0,0 +1,149 @@ +package autoinstall + +import ( + "errors" + "os" + "sync" + "testing" + "time" + + "github.com/databricks/cli/libs/env" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestTryLockExcludesOtherRuns(t *testing.T) { + ctx := env.WithUserHomeDir(t.Context(), t.TempDir()) + + unlock, ok := TryLock(ctx) + require.True(t, ok) + _, ok = TryLock(ctx) + assert.False(t, ok, "a second run must not take a held lock") + + unlock() + unlock, ok = TryLock(ctx) + require.True(t, ok, "a released lock can be taken again") + unlock() +} + +func TestRecordKeepsConcurrentOptOut(t *testing.T) { + ctx := env.WithUserHomeDir(t.Context(), t.TempDir()) + + // Without the state lock, a Record that read the state before the opt-out + // was saved would write opted_out=false back. + var wg sync.WaitGroup + for range 20 { + wg.Go(func() { assert.NoError(t, Record(ctx, "1.30.0", 1, "machine")) }) + } + wg.Go(func() { assert.NoError(t, SetOptedOut(ctx, true)) }) + wg.Wait() + + s, err := Load(ctx) + require.NoError(t, err) + assert.True(t, s.OptedOut) + assert.Equal(t, "1.30.0", s.LastVersion) +} + +func TestLoadWaitsForStateLock(t *testing.T) { + ctx := env.WithUserHomeDir(t.Context(), t.TempDir()) + + // A writer holds the state lock while it replaces the file, which fails on + // Windows while another process has the file open. + unlock, err := lock(ctx, stateLockFileName, 0) + require.NoError(t, err) + done := make(chan struct{}) + go func() { + _, err := Load(ctx) + assert.NoError(t, err) + close(done) + }() + select { + case <-done: + t.Fatal("Load read the state while the state lock was held") + case <-time.After(100 * time.Millisecond): + } + unlock() + <-done +} + +func TestLoadFailsWhenStateLockStaysHeld(t *testing.T) { + ctx := env.WithUserHomeDir(t.Context(), t.TempDir()) + + // A stopped holder must not block every later command. + unlock, err := lock(ctx, stateLockFileName, 0) + require.NoError(t, err) + defer unlock() + start := time.Now() + _, err = Load(ctx) + assert.ErrorIs(t, err, errLockHeld) + assert.Less(t, time.Since(start), stateLockWait+2*time.Second) +} + +func TestLockReturnsOtherErrorsAtOnce(t *testing.T) { + ctx := env.WithUserHomeDir(t.Context(), t.TempDir()) + + // A filesystem without locks fails every try, so it is not retried. + errNoLocks := errors.New("no locks here") + orig := tryLock + t.Cleanup(func() { tryLock = orig }) + tries := 0 + tryLock = func(*os.File) error { + tries++ + return errNoLocks + } + _, err := lock(ctx, stateLockFileName, stateLockWait) + assert.ErrorIs(t, err, errNoLocks) + assert.Equal(t, 1, tries) +} + +func TestSetOptedOutWithoutLocks(t *testing.T) { + ctx := env.WithUserHomeDir(t.Context(), t.TempDir()) + + // An explicit opt-out must work where the filesystem has no locks. + orig := tryLock + t.Cleanup(func() { tryLock = orig }) + tryLock = func(*os.File) error { return errors.New("no locks here") } + require.NoError(t, SetOptedOut(ctx, true)) + + s, err := load(ctx) + require.NoError(t, err) + assert.True(t, s.OptedOut) + + // A lock another process holds is waited for, not skipped. + tryLock = func(*os.File) error { return errLockHeld } + assert.ErrorIs(t, SetOptedOut(ctx, false), errLockHeld) +} + +func TestRecordClearsInstallingWhenSettled(t *testing.T) { + ctx := env.WithUserHomeDir(t.Context(), t.TempDir()) + + require.NoError(t, MarkInstalling(ctx)) + // A failed attempt leaves the marker for the next run. + require.NoError(t, Record(ctx, "1.30.0", 1, "machine")) + s, err := Load(ctx) + require.NoError(t, err) + assert.True(t, s.Installing) + + require.NoError(t, Record(ctx, "1.30.0", 0, "machine")) + s, err = Load(ctx) + require.NoError(t, err) + assert.False(t, s.Installing) +} + +func TestNextAttempt(t *testing.T) { + assert.Equal(t, 1, NextAttempt(State{}, "1.30.0")) + assert.Equal(t, 1, NextAttempt(State{LastVersion: "1.29.0", Attempts: 4}, "1.30.0")) + assert.Equal(t, 5, NextAttempt(State{LastVersion: "1.30.0", Attempts: 4}, "1.30.0")) +} + +func TestChildEnvCarriesTheRefusal(t *testing.T) { + t.Cleanup(ResetDeclined) + + environ := []string{"HOME=/home/user"} + assert.Equal(t, environ, ChildEnv(environ)) + + // A CLI child started after a "no" must not install either, even when + // the opt-out could not be stored. + declined.Store(true) + assert.Equal(t, []string{"HOME=/home/user", EnvVar + "=false"}, ChildEnv(environ)) +} diff --git a/libs/aitools/autoinstall/state.go b/libs/aitools/autoinstall/state.go index 313b429434e..c13982fa129 100644 --- a/libs/aitools/autoinstall/state.go +++ b/libs/aitools/autoinstall/state.go @@ -10,6 +10,7 @@ import ( "io/fs" "os" "path/filepath" + "sync/atomic" "github.com/databricks/cli/libs/atomicfile" "github.com/databricks/cli/libs/env" @@ -35,9 +36,21 @@ type State struct { Attempts int `json:"attempts,omitempty"` // OptedOut turns auto-install off for every later version. It is set by - // `databricks aitools auto-install disable` and by a global-scope - // `databricks aitools uninstall`. + // `databricks aitools auto-install disable`, by a global-scope + // `databricks aitools uninstall` and by a "no" to the skills prompt of + // `databricks apps init`. OptedOut bool `json:"opted_out,omitempty"` + + // Installing is set while auto-install makes a first install, and stays + // set when the install did not finish. The skills such an install left + // have no install state, like the ones the user put there, so a later run + // uses it to finish the install instead of leaving the skills alone. + Installing bool `json:"installing,omitempty"` + + // ID is a random per-machine identifier, created on the first run that + // decides anything. Only its Bucket is used, and it never leaves the + // machine. The CLI has no other stable machine identifier. + ID string `json:"id,omitempty"` } // statePath returns ~/.databricks/aitools/auto-install.json. @@ -49,8 +62,20 @@ func statePath(ctx context.Context) (string, error) { return filepath.Join(home, ".databricks", "aitools", stateFileName), nil } -// Load reads the state. A missing file is the zero State. +// Load reads the state. A missing file is the zero State. It holds the state +// lock while it reads: on Windows a file another process has open cannot be +// replaced, so an unlocked read could make a concurrent write fail. func Load(ctx context.Context) (State, error) { + unlock, err := lock(ctx, stateLockFileName, stateLockWait) + if err != nil { + return State{}, err + } + defer unlock() + return load(ctx) +} + +// load reads the state; the caller holds the state lock. +func load(ctx context.Context) (State, error) { path, err := statePath(ctx) if err != nil { return State{}, err @@ -69,8 +94,8 @@ func Load(ctx context.Context) (State, error) { return s, nil } -// Save writes the state atomically. -func Save(ctx context.Context, s State) error { +// save writes the state atomically; the caller holds the state lock. +func save(ctx context.Context, s State) error { path, err := statePath(ctx) if err != nil { return err @@ -84,15 +109,76 @@ func Save(ctx context.Context, s State) error { // SetOptedOut turns auto-install off (true) or back on (false), keeping the // rest of the state. A state file that cannot be read is replaced: the user's -// explicit choice matters more than a broken record. +// explicit choice matters more than a broken record. Where the state lock +// cannot be taken at all (a filesystem without locks), it writes without it: +// auto-install cannot run there, so nothing else writes the state. func SetOptedOut(ctx context.Context, optedOut bool) error { - s, err := Load(ctx) + unlock, err := lock(ctx, stateLockFileName, stateLockWait) + switch { + case errors.Is(err, errLockHeld): + return err + case err != nil: + log.Debugf(ctx, "Saving the auto-install state without its lock: %v", err) + default: + defer unlock() + } + s, err := load(ctx) if err != nil { log.Debugf(ctx, "Replacing unreadable auto-install state: %v", err) s = State{} } s.OptedOut = optedOut - return Save(ctx, s) + return save(ctx, s) +} + +// declined is set once the user turned the tools down in this process. +var declined atomic.Bool + +// Decline turns auto-install off after the user turned the tools down: for +// this process at once, and for later ones through the stored opt-out. The +// error is that of storing the opt-out; this process honors the answer either +// way. +func Decline(ctx context.Context) error { + declined.Store(true) + return SetOptedOut(ctx, true) +} + +// Declined reports whether Decline ran in this process. +func Declined() bool { + return declined.Load() +} + +// ResetDeclined forgets a Decline in this process. Tests that decline use it +// to clean up. +func ResetDeclined() { + declined.Store(false) +} + +// ChildEnv returns environ for a CLI process this one starts, with +// auto-install turned off when the user declined in this process, so the child +// honors the answer even if storing it failed. +func ChildEnv(environ []string) []string { + if Declined() { + environ = append(environ, EnvVar+"=false") + } + return environ +} + +// updateState applies change to the stored state and saves it. Every read and +// change of the state goes through the state lock, so that concurrent CLI +// processes do not overwrite each other's changes. +func updateState(ctx context.Context, change func(*State)) error { + unlock, err := lock(ctx, stateLockFileName, stateLockWait) + if err != nil { + return err + } + defer unlock() + s, err := load(ctx) + if err != nil { + return err + } + change(&s) + return save(ctx, s) } // EnvOptedOut reports whether EnvVar turns auto-install off. diff --git a/libs/aitools/autoinstall/state_test.go b/libs/aitools/autoinstall/state_test.go index ea4e5c8476c..b18a472ae8f 100644 --- a/libs/aitools/autoinstall/state_test.go +++ b/libs/aitools/autoinstall/state_test.go @@ -31,7 +31,7 @@ func TestLoadCorruptFileFails(t *testing.T) { func TestSetOptedOutKeepsOtherFields(t *testing.T) { ctx := env.WithUserHomeDir(t.Context(), t.TempDir()) - require.NoError(t, Save(ctx, State{LastVersion: "1.2.3", Attempts: 2})) + require.NoError(t, save(ctx, State{LastVersion: "1.2.3", Attempts: 2})) require.NoError(t, SetOptedOut(ctx, true)) s, err := Load(ctx) diff --git a/libs/aitools/installer/plugin.go b/libs/aitools/installer/plugin.go index d623a75c496..75ad7e5edd5 100644 --- a/libs/aitools/installer/plugin.go +++ b/libs/aitools/installer/plugin.go @@ -81,7 +81,9 @@ func (e *BlockedError) Error() string { func runAgentCmd(ctx context.Context, timeout time.Duration, argv []string) (string, error) { cctx, cancel := context.WithTimeout(ctx, timeout) defer cancel() - out, err := process.Background(cctx, argv) + // A timed-out agent CLI is stopped with its descendants: one left running + // would hold the output pipes open and block past the timeout. + out, err := process.BackgroundInGroup(cctx, argv) if err != nil && errors.Is(cctx.Err(), context.DeadlineExceeded) { return out, &agentCmdTimeoutError{timeout: timeout, err: err} } diff --git a/libs/aitools/installer/plugin_unix_test.go b/libs/aitools/installer/plugin_unix_test.go new file mode 100644 index 00000000000..b31e16aa328 --- /dev/null +++ b/libs/aitools/installer/plugin_unix_test.go @@ -0,0 +1,23 @@ +//go:build unix + +package installer + +import ( + "errors" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestRunAgentCmdTimeoutStopsDescendants(t *testing.T) { + // The agent CLI leaves a child holding its output pipes. Killing only the + // CLI would leave the command waiting for the child to exit. + start := time.Now() + _, err := runAgentCmd(t.Context(), 100*time.Millisecond, []string{"sh", "-c", "sleep 30 & sleep 30"}) + require.Error(t, err) + assert.Less(t, time.Since(start), 5*time.Second) + _, timedOut := errors.AsType[*agentCmdTimeoutError](err) + assert.True(t, timedOut) +} diff --git a/libs/aitools/installer/update.go b/libs/aitools/installer/update.go index f6f45adf957..cbd3a48bd89 100644 --- a/libs/aitools/installer/update.go +++ b/libs/aitools/installer/update.go @@ -30,6 +30,12 @@ type UpdateOptions struct { Check bool // dry run: show what would change without downloading Skills []string // empty = all installed Scope string // ScopeGlobal or ScopeProject (default: global) + + // KeepModified keeps an installed skill at its version when its files are + // not exactly the ones the CLI recorded (the user edited them, or there is + // no record to check against), instead of replacing it. Automatic updates + // set it: they must never overwrite what the user changed. + KeepModified bool } // UpdateResult describes what UpdateSkills did (or would do in check mode). @@ -173,6 +179,12 @@ func UpdateSkills(ctx context.Context, src ManifestSource, targetAgents []*agent continue } + if wasInstalled && opts.KeepModified && !skillPrunable(ctx, baseDir, name, scope, cwd, state) { + log.Warnf(ctx, "Warning: keeping %q at %s: its installed files were modified.", name, oldVersion) + result.Unchanged = append(result.Unchanged, name) + continue + } + update := SkillUpdate{ Name: name, OldVersion: oldVersion, diff --git a/libs/aitools/installer/update_test.go b/libs/aitools/installer/update_test.go index 4e55246e34b..5871e8fc417 100644 --- a/libs/aitools/installer/update_test.go +++ b/libs/aitools/installer/update_test.go @@ -127,6 +127,42 @@ func TestUpdateVersionDiffDetected(t *testing.T) { assert.Equal(t, "0.2.0", state.Skills["databricks-sql"]) } +func TestUpdateKeepModifiedKeepsEditedSkill(t *testing.T) { + tmp := setupTestHome(t) + ctx := cmdio.MockDiscard(t.Context()) + setupFetchMock(t) + t.Setenv("DATABRICKS_SKILLS_REF", testSkillsRef) + + src := &mockManifestSource{manifest: testManifest()} + agent := testAgent(tmp) + require.NoError(t, InstallSkillsForAgents(ctx, src, []*agents.Agent{agent}, InstallOptions{})) + + // The user edits databricks-sql; a new release updates both skills. + globalDir := filepath.Join(tmp, ".databricks", "aitools", "skills") + edited := filepath.Join(globalDir, "databricks-sql", "SKILL.md") + require.NoError(t, os.WriteFile(edited, []byte("user edit"), 0o644)) + t.Setenv("DATABRICKS_SKILLS_REF", "v0.2.0") + updatedManifest := testManifest() + for name, meta := range updatedManifest.Skills { + meta.Version = "0.2.0" + updatedManifest.Skills[name] = meta + } + + result, err := UpdateSkills(ctx, &mockManifestSource{manifest: updatedManifest}, []*agents.Agent{agent}, UpdateOptions{KeepModified: true}) + require.NoError(t, err) + + require.Len(t, result.Updated, 1) + assert.Equal(t, "databricks-jobs", result.Updated[0].Name) + assert.Contains(t, result.Unchanged, "databricks-sql") + data, err := os.ReadFile(edited) + require.NoError(t, err) + assert.Equal(t, "user edit", string(data)) + state, err := LoadState(globalDir) + require.NoError(t, err) + assert.Equal(t, "0.1.0", state.Skills["databricks-sql"]) + assert.Equal(t, "0.2.0", state.Skills["databricks-jobs"]) +} + func TestUpdateCheckDryRun(t *testing.T) { tmp := setupTestHome(t) ctx := cmdio.MockDiscard(t.Context()) diff --git a/libs/clicompat/clicompat.go b/libs/clicompat/clicompat.go index 7d9f6f746fa..b59a0e2222f 100644 --- a/libs/clicompat/clicompat.go +++ b/libs/clicompat/clicompat.go @@ -97,9 +97,18 @@ var httpClient = &http.Client{Timeout: fetchTimeout} // Set DATABRICKS_CACHE_ENABLED=false to bypass the local cache (tiers 1 and 3a), // which is useful to recover from a bad cached manifest. func FetchManifest(ctx context.Context) (Manifest, error) { + m, _, err := fetchManifest(ctx) + return m, err +} + +// fetchManifest is FetchManifest. fallback is set when the remote and the +// local cache both failed and the manifest is the embedded one, which may be +// older than either. +func fetchManifest(ctx context.Context) (m Manifest, fallback bool, err error) { if force, ok := env.GetBool(ctx, "DATABRICKS_FORCE_EMBEDDED_COMPAT"); ok && force { log.Debugf(ctx, "Using embedded manifest (DATABRICKS_FORCE_EMBEDDED_COMPAT=true)") - return parseEmbeddedManifest() + m, err := parseEmbeddedManifest() + return m, false, err } cacheEnabled := true @@ -121,7 +130,7 @@ func FetchManifest(ctx context.Context) (Manifest, error) { // Tier 1: local file is fresh. if localErr == nil && local.isFresh(cacheTTL) { log.Debugf(ctx, "Using cached manifest from %s", localPath) - return local.manifest, nil + return local.manifest, false, nil } // Tier 2: fetch from remote (local file missing or stale). @@ -130,23 +139,23 @@ func FetchManifest(ctx context.Context) (Manifest, error) { if cacheEnabled { writeLocalManifest(ctx, localPath, m) } - return m, nil + return m, false, nil } // Tier 3a: local file exists but stale — use it anyway. if localErr == nil { log.Debugf(ctx, "Using stale cached manifest (remote failed: %v)", fetchErr) - return local.manifest, nil + return local.manifest, false, nil } // Tier 3b: embedded manifest. m, embeddedErr := parseEmbeddedManifest() if embeddedErr == nil { log.Debugf(ctx, "Using embedded manifest (remote and local cache failed)") - return m, nil + return m, true, nil } - return nil, fmt.Errorf("all manifest sources failed (remote: %w, embedded: %w)", fetchErr, embeddedErr) + return nil, false, fmt.Errorf("all manifest sources failed (remote: %w, embedded: %w)", fetchErr, embeddedErr) } // parseEmbeddedManifest parses the embedded manifest once and caches the result. @@ -295,9 +304,18 @@ func ResolveAgentSkillsVersion(ctx context.Context) (string, error) { } // ResolveAutoInstall reports whether the manifest entry for cliVersion turns on -// the automatic installation of Databricks AI tools. +// the automatic installation of Databricks AI tools. It fails when only the +// embedded manifest is available: that one predates any remote change of the +// gate, so it cannot tell off from not known yet. func ResolveAutoInstall(ctx context.Context, cliVersion string) (bool, error) { - entry, err := resolveEntry(ctx, cliVersion) + m, fallback, err := fetchManifest(ctx) + if err != nil { + return false, err + } + if fallback { + return false, errors.New("the compatibility manifest could not be fetched") + } + entry, err := Resolve(m, cliVersion) if err != nil { return false, err } diff --git a/libs/clicompat/clicompat_test.go b/libs/clicompat/clicompat_test.go index f4ce7613f71..d69299c2802 100644 --- a/libs/clicompat/clicompat_test.go +++ b/libs/clicompat/clicompat_test.go @@ -353,6 +353,20 @@ func TestResolveAutoInstall(t *testing.T) { } } +func TestResolveAutoInstallFailsOnTheEmbeddedFallback(t *testing.T) { + ctx := testContext(t) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + })) + defer srv.Close() + redirectToServer(t, srv) + + // Neither the remote nor a cached manifest is there: the gate is unknown, + // not off. + _, err := ResolveAutoInstall(ctx, "1.30.0") + assert.Error(t, err) +} + // TestManifestLeavesAutoInstallOff checks that no entry turns auto_install on. // The remote manifest is this same file on main, so turning it on anywhere is a // launch decision: change this test in the PR that makes it. diff --git a/libs/process/interrupt_other.go b/libs/process/interrupt_other.go new file mode 100644 index 00000000000..73625fa5060 --- /dev/null +++ b/libs/process/interrupt_other.go @@ -0,0 +1,12 @@ +//go:build !unix + +package process + +import "context" + +// BackgroundInGroup runs args as Background does, with WithProcessGroup. On +// these platforms the child stays in the console's process group, so Ctrl-C +// reaches it directly and nothing is forwarded. +func BackgroundInGroup(ctx context.Context, args []string, opts ...execOption) (string, error) { + return Background(ctx, args, append(opts, WithProcessGroup())...) +} diff --git a/libs/process/interrupt_unix.go b/libs/process/interrupt_unix.go new file mode 100644 index 00000000000..26fd8ffdbd3 --- /dev/null +++ b/libs/process/interrupt_unix.go @@ -0,0 +1,47 @@ +//go:build unix + +package process + +import ( + "context" + "os" + "os/signal" + "syscall" +) + +// BackgroundInGroup runs args as Background does, with WithProcessGroup, so a +// cancelled ctx stops the child's descendants too. +// +// The new process group takes the child out of the terminal's foreground +// group, so Ctrl-C no longer reaches it. While the child runs, the first SIGINT +// or SIGTERM is caught instead: it stops the group, and is then raised again +// in this process, which ends as it would have without the group. The default +// disposition is restored before the group is stopped, so a second signal ends +// this process at once. +func BackgroundInGroup(ctx context.Context, args []string, opts ...execOption) (string, error) { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + sigCh := make(chan os.Signal, 1) + signal.Notify(sigCh, os.Interrupt, syscall.SIGTERM) + caught := make(chan os.Signal, 1) + done := make(chan struct{}) + go func() { + defer close(caught) + select { + case sig := <-sigCh: + signal.Stop(sigCh) + caught <- sig + cancel() + case <-done: + } + }() + + out, err := Background(ctx, args, append(opts, WithProcessGroup())...) + close(done) + signal.Stop(sigCh) + if sig, ok := <-caught; ok { + _ = syscall.Kill(os.Getpid(), sig.(syscall.Signal)) + } + return out, err +} diff --git a/libs/process/interrupt_unix_test.go b/libs/process/interrupt_unix_test.go new file mode 100644 index 00000000000..090a3477b1b --- /dev/null +++ b/libs/process/interrupt_unix_test.go @@ -0,0 +1,62 @@ +//go:build unix + +package process + +import ( + "errors" + "os" + "os/exec" + "path/filepath" + "syscall" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// interruptHelperEnv makes TestBackgroundInGroupInterruptHelper run +// BackgroundInGroup in a helper process, which the test interrupts. Raising +// the signal again ends the process, so it cannot run in the test process. +const interruptHelperEnv = "PROCESS_TEST_INTERRUPT_HELPER_PIDFILE" + +func TestBackgroundInGroupInterruptHelper(t *testing.T) { + pidFile := os.Getenv(interruptHelperEnv) + if pidFile == "" { + t.Skip("run by TestBackgroundInGroupForwardsInterrupt") + } + _, _ = BackgroundInGroup(t.Context(), []string{"sh", "-c", `sleep 300 & echo $! > "$1"; wait`, "sh", pidFile}) + // The raised signal ends the process asynchronously. + time.Sleep(5 * time.Second) + t.Fatal("the interrupt did not end the process") +} + +func TestBackgroundInGroupForwardsInterrupt(t *testing.T) { + pidFile := filepath.Join(t.TempDir(), "grandchild.pid") + helper := exec.Command(os.Args[0], "-test.run=^TestBackgroundInGroupInterruptHelper$") + helper.Env = append(os.Environ(), interruptHelperEnv+"="+pidFile) + require.NoError(t, helper.Start()) + + grandchildPid := waitForPid(t, pidFile) + // A Ctrl-C in the terminal signals the helper, which is in the foreground + // group, but not the child, which has its own group. + require.NoError(t, helper.Process.Signal(os.Interrupt)) + + waitErr := make(chan error, 1) + go func() { waitErr <- helper.Wait() }() + select { + case err := <-waitErr: + exitErr, ok := errors.AsType[*exec.ExitError](err) + require.True(t, ok, "helper exited with %v", err) + status := exitErr.Sys().(syscall.WaitStatus) + assert.True(t, status.Signaled(), "the helper must end as the interrupt would have, got %v", err) + assert.Equal(t, syscall.SIGINT, status.Signal()) + case <-time.After(processGroupGracePeriod + 5*time.Second): + _ = helper.Process.Kill() + t.Fatal("the interrupt did not end the helper") + } + + assert.Eventually(t, func() bool { + return errors.Is(syscall.Kill(grandchildPid, 0), syscall.ESRCH) + }, 5*time.Second, 20*time.Millisecond, "grandchild %d outlived the interrupt", grandchildPid) +} diff --git a/libs/telemetry/protos/README.md b/libs/telemetry/protos/README.md index a6ae4c7cbd8..75091a8146d 100644 --- a/libs/telemetry/protos/README.md +++ b/libs/telemetry/protos/README.md @@ -30,3 +30,16 @@ wait for it. Neither does a run of `install`, `update` or `uninstall` that ends confirmation step (a flag error, a cancelled prompt, no agent detected, a `--path` dump), nor an `update` or `uninstall` that finds nothing installed. A run that does not wait still uploads, within the upload timeout, when auth has already resolved. + +On the first run of a new CLI version, the CLI can install Databricks AI tools for the coding agents +on the machine (see `databricks aitools auto-install --help`). It logs an `aitools_auto_install_event` +with the outcome (`INSTALLED`, `UP_TO_DATE`, `KEPT`, `SKIPPED_CI`, `SKIPPED_OPTOUT`, +`SKIPPED_HOLDOUT`, `SKIPPED_NO_AGENTS`, `SKIPPED_PLATFORM`, `FAILED`, `RETRIES_EXHAUSTED`), the +machine's holdout arm and, for a failure, its error category: once per version and machine, plus +once per retry of a failed attempt. Nothing is logged for a version whose outcome is settled, on +Databricks Runtime, or while the feature is turned off remotely (the CI and opt-out checks come +first and are logged either way). An install with no tools installed yet also logs an +`aitools_install_event` with `trigger` `AUTO_FIRST_RUN`; installs by `databricks aitools install` +leave `trigger` unset. The holdout arm comes from a random machine id kept in +`~/.databricks/aitools/auto-install.json`; the id itself is never sent. Like the other aitools +events, these upload at exit only when the command resolved auth. diff --git a/libs/telemetry/protos/aitools_events.go b/libs/telemetry/protos/aitools_events.go index 4084430eff1..eb67e1cce93 100644 --- a/libs/telemetry/protos/aitools_events.go +++ b/libs/telemetry/protos/aitools_events.go @@ -160,3 +160,71 @@ type AitoolsCommandEvent struct { // omitted when none or both scopes are selected. Scope AitoolsInstallScope `json:"scope,omitempty"` } + +// AitoolsAutoInstallOutcome mirrors AitoolsAutoInstallOutcome.Type: how the +// automatic installation of Databricks AI tools on the first run of a new CLI +// version ended. +type AitoolsAutoInstallOutcome string + +const ( + AitoolsAutoInstallOutcomeUnspecified AitoolsAutoInstallOutcome = "TYPE_UNSPECIFIED" + // The tools were installed, or an existing installation was updated and + // changed. + AitoolsAutoInstallOutcomeInstalled AitoolsAutoInstallOutcome = "INSTALLED" + // An existing installation was updated and nothing changed: it was current, + // or every outdated skill was kept because the user modified it. + AitoolsAutoInstallOutcomeUpToDate AitoolsAutoInstallOutcome = "UP_TO_DATE" + // The installed tools were left as they are: skills without install state + // (installed by hand, or by an older CLI), or skills newer than the release + // this CLI installs. + AitoolsAutoInstallOutcomeKept AitoolsAutoInstallOutcome = "KEPT" + // Every change would have run an agent CLI, which auto-install does not do + // on Windows: a plan of only plugin installs, or an update of an + // installation with plugins. + AitoolsAutoInstallOutcomeSkippedPlatform AitoolsAutoInstallOutcome = "SKIPPED_PLATFORM" + // The run was in CI (the CI env var, or a CI provider's own). + AitoolsAutoInstallOutcomeSkippedCI AitoolsAutoInstallOutcome = "SKIPPED_CI" + // The user turned auto-install off: DATABRICKS_AITOOLS_AUTO_INSTALL=false, + // `aitools auto-install disable`, a global `aitools uninstall`, or "no" + // to the skills prompt of `apps init`. + AitoolsAutoInstallOutcomeSkippedOptOut AitoolsAutoInstallOutcome = "SKIPPED_OPTOUT" + // The machine is in the holdout cohort. + AitoolsAutoInstallOutcomeSkippedHoldout AitoolsAutoInstallOutcome = "SKIPPED_HOLDOUT" + // No supported coding agent has a config directory on the machine. + AitoolsAutoInstallOutcomeSkippedNoAgents AitoolsAutoInstallOutcome = "SKIPPED_NO_AGENTS" + // The attempt failed or timed out; a later run of the same version retries. + AitoolsAutoInstallOutcomeFailed AitoolsAutoInstallOutcome = "FAILED" + // The attempt failed and was the last one this CLI version gets. + AitoolsAutoInstallOutcomeRetriesExhausted AitoolsAutoInstallOutcome = "RETRIES_EXHAUSTED" +) + +// AitoolsHoldoutArm mirrors AitoolsHoldoutArm.Type: the auto-install +// experiment arm of the machine, from a stable per-machine bucket. +type AitoolsHoldoutArm string + +const ( + AitoolsHoldoutArmUnspecified AitoolsHoldoutArm = "TYPE_UNSPECIFIED" + // The machine never auto-installs (control group). + AitoolsHoldoutArmHoldout AitoolsHoldoutArm = "HOLDOUT" + // The machine auto-installs. + AitoolsHoldoutArmTreatment AitoolsHoldoutArm = "TREATMENT" +) + +// AitoolsAutoInstallEvent is emitted when the CLI decides about installing +// Databricks AI tools on the first run of a new version: once per version and +// machine, plus once per retry after a failed attempt. A version whose outcome +// is settled emits nothing, and so does a run that the remote gate turns off; +// the CI and opt-out checks come before the gate, so they are emitted either +// way. An install with no tools installed yet also emits an +// AitoolsInstallEvent with Trigger AUTO_FIRST_RUN. +type AitoolsAutoInstallEvent struct { + Outcome AitoolsAutoInstallOutcome `json:"outcome"` + HoldoutArm AitoolsHoldoutArm `json:"holdout_arm"` + + // Whether the run could prompt (cmdio.IsPromptSupported and no coding + // agent detected). A measurable false, so no omitempty. + Interactive bool `json:"interactive"` + + // Set for FAILED and RETRIES_EXHAUSTED. + ErrorCategory AitoolsErrorCategory `json:"error_category,omitempty"` +} diff --git a/libs/telemetry/protos/aitools_install.go b/libs/telemetry/protos/aitools_install.go index 406dde89a58..051076f6241 100644 --- a/libs/telemetry/protos/aitools_install.go +++ b/libs/telemetry/protos/aitools_install.go @@ -84,8 +84,21 @@ type AitoolsAgentResult struct { ErrorCategory AitoolsErrorCategory `json:"error_category"` } +// AitoolsInstallTrigger mirrors AitoolsInstallTrigger.Type in the +// databricks_cli lumberjack proto: what started an install other than the +// `databricks aitools install` command. The command's own installs leave it +// unset. +type AitoolsInstallTrigger string + +const ( + AitoolsInstallTriggerUnspecified AitoolsInstallTrigger = "TYPE_UNSPECIFIED" + // The install the CLI runs on its own on the first run of a new version. + AitoolsInstallTriggerAutoFirstRun AitoolsInstallTrigger = "AUTO_FIRST_RUN" +) + // AitoolsInstallEvent is emitted on every execution of the `databricks aitools -// install` command. +// install` command, and for the install the CLI runs on the first run of a new +// version (Trigger AUTO_FIRST_RUN). type AitoolsInstallEvent struct { // Agents the user targeted, deduplicated and sorted for stable output. Agents []AitoolsAgentType `json:"agents,omitempty"` @@ -105,4 +118,7 @@ type AitoolsInstallEvent struct { // AgentResults records the per-agent failure/skip categories, one entry per // non-successful agent. Empty when every targeted agent succeeded. AgentResults []AitoolsAgentResult `json:"agent_results,omitempty"` + + // Trigger is set when the install did not come from the command. + Trigger AitoolsInstallTrigger `json:"trigger,omitempty"` } diff --git a/libs/telemetry/protos/frontend_log.go b/libs/telemetry/protos/frontend_log.go index f3c37736cc1..8a7e5bbdb11 100644 --- a/libs/telemetry/protos/frontend_log.go +++ b/libs/telemetry/protos/frontend_log.go @@ -28,4 +28,5 @@ type DatabricksCliLog struct { AirRunEvent *AirRunEvent `json:"air_run_event,omitempty"` AitoolsHintEvent *AitoolsHintEvent `json:"aitools_hint_event,omitempty"` AitoolsCommandEvent *AitoolsCommandEvent `json:"aitools_command_event,omitempty"` + AitoolsAutoInstallEvent *AitoolsAutoInstallEvent `json:"aitools_auto_install_event,omitempty"` } diff --git a/libs/versioncheck/notice.go b/libs/versioncheck/notice.go index dc3a9d5c864..9c57136722b 100644 --- a/libs/versioncheck/notice.go +++ b/libs/versioncheck/notice.go @@ -263,7 +263,7 @@ func gatherConditions(ctx context.Context, cmd *cobra.Command) notifyConditions ci: ci, nonInteractive: nonInteractive, jsonOutput: isJSONOutput(cmd), - exemptCommand: isExemptCommand(cmd), + exemptCommand: IsExemptCommand(cmd), } } @@ -272,10 +272,10 @@ func isJSONOutput(cmd *cobra.Command) bool { return f != nil && f.Value.String() == "json" } -// isExemptCommand suppresses the notice for commands where it would be noise or +// IsExemptCommand reports whether cmd gets no notice, as it for commands where it would be noise or // would corrupt machine-readable output: the version commands themselves, shell // completion, and help. -func isExemptCommand(cmd *cobra.Command) bool { +func IsExemptCommand(cmd *cobra.Command) bool { for c := cmd; c != nil; c = c.Parent() { switch c.Name() { case "version", "completion", "help", cobra.ShellCompRequestCmd, cobra.ShellCompNoDescRequestCmd: diff --git a/libs/versioncheck/notice_test.go b/libs/versioncheck/notice_test.go index eadd50c707c..c8643e860c2 100644 --- a/libs/versioncheck/notice_test.go +++ b/libs/versioncheck/notice_test.go @@ -148,7 +148,7 @@ func TestIsExemptCommand(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - assert.Equal(t, tt.want, isExemptCommand(tt.cmd)) + assert.Equal(t, tt.want, IsExemptCommand(tt.cmd)) }) } } diff --git a/main.go b/main.go index 10c3686a223..0b4e8e48a0b 100644 --- a/main.go +++ b/main.go @@ -6,6 +6,8 @@ import ( "path/filepath" "github.com/databricks/cli/cmd" + "github.com/databricks/cli/cmd/aitools" + "github.com/databricks/cli/libs/aitools/autoinstall" // Registers a disk-cached HostMetadataResolver factory on the SDK so every // *config.Config the CLI constructs reuses the cached /.well-known lookup. @@ -23,6 +25,11 @@ func main() { os.Setenv("DATABRICKS_CLI_PATH", arg0) } + // cmd/root runs auto-install after a command but cannot import cmd/aitools, + // which imports it. It is wired here, not in a package init, for the same + // reason as DATABRICKS_CLI_PATH above. + autoinstall.Run = aitools.AutoInstall + ctx := context.Background() err := cmd.New(ctx).Execute(ctx, os.Args[1:]) if err != nil {