From c5eef9cc3d7d9e3c04ed3082a38b34f5f1821dd3 Mon Sep 17 00:00:00 2001 From: Anton Nekipelov <226657+anton-107@users.noreply.github.com> Date: Mon, 16 Nov 2026 19:22:06 +0000 Subject: [PATCH 1/7] aitools: store the auto-install state and opt-out, gate it in cli-compat First of three changes that install Databricks AI tools automatically on the first run of a new CLI version. This one stores the state and the opt-out and adds the remote gate; nothing installs yet. - ~/.databricks/aitools/auto-install.json holds last_version, attempts and opted_out. It is a dedicated file, not a libs/cache entry, so an opt-out survives upgrades and `databricks cache clear`. - `databricks aitools auto-install disable|enable` toggles the opt-out; enable warns when DATABRICKS_AITOOLS_AUTO_INSTALL=false is set. - A global-scope `databricks aitools uninstall` (including --skills and --agents) opts out. - cli-compat.json entries get an optional auto_install field, resolved by clicompat.ResolveAutoInstall. No entry sets it, and TestManifestLeavesAutoInstallOff keeps it off. Co-authored-by: Isaac --- .agents/skills/bump-cli-compat/SKILL.md | 2 +- .../cli/aitools-auto-install-opt-out.md | 1 + .../auto-install/enable-disable/out.test.toml | 1 + .../auto-install/enable-disable/output.txt | 57 ++++++++++ .../auto-install/enable-disable/script | 35 ++++++ .../cmd/aitools/auto-install/script.prepare | 5 + acceptance/cmd/aitools/auto-install/test.toml | 35 ++++++ cmd/aitools/aitools.go | 1 + cmd/aitools/autoinstall.go | 65 +++++++++++ cmd/aitools/uninstall.go | 14 +++ libs/aitools/autoinstall/state.go | 101 ++++++++++++++++++ libs/clicompat/clicompat.go | 17 +++ libs/clicompat/clicompat_test.go | 39 +++++++ 13 files changed, 372 insertions(+), 1 deletion(-) create mode 100644 .nextchanges/cli/aitools-auto-install-opt-out.md create mode 100644 acceptance/cmd/aitools/auto-install/enable-disable/out.test.toml create mode 100644 acceptance/cmd/aitools/auto-install/enable-disable/output.txt create mode 100644 acceptance/cmd/aitools/auto-install/enable-disable/script create mode 100644 acceptance/cmd/aitools/auto-install/script.prepare create mode 100644 acceptance/cmd/aitools/auto-install/test.toml create mode 100644 cmd/aitools/autoinstall.go create mode 100644 libs/aitools/autoinstall/state.go diff --git a/.agents/skills/bump-cli-compat/SKILL.md b/.agents/skills/bump-cli-compat/SKILL.md index 454c93e14a9..a5808c4ea8b 100644 --- a/.agents/skills/bump-cli-compat/SKILL.md +++ b/.agents/skills/bump-cli-compat/SKILL.md @@ -89,7 +89,7 @@ Write the updated `internal/build/cli-compat.json` and `internal/build/clicompat Ask the user for the **minimum CLI version** that supports the new features. -Add a new entry keyed to that CLI version with the new `appkit` and `"skills": "latest"`. Update `AgentSkillsFallbackVersion` in `internal/build/clicompat.go` to the tested skills version. If skills compatibility changed, replace `"latest"` in the affected older ranges with the last compatible release. Preserve unrelated AppKit and skills compatibility boundaries. +Add a new entry keyed to that CLI version with the new `appkit` and `"skills": "latest"`. If the previous highest entry has `"auto_install": true`, copy it to the new entry: it turns on the automatic installation of AI tools for its range, so a new entry without it turns the feature off for every newer CLI. Update `AgentSkillsFallbackVersion` in `internal/build/clicompat.go` to the tested skills version. If skills compatibility changed, replace `"latest"` in the affected older ranges with the last compatible release. Preserve unrelated AppKit and skills compatibility boundaries. Write the updated `internal/build/cli-compat.json` and `internal/build/clicompat.go`. diff --git a/.nextchanges/cli/aitools-auto-install-opt-out.md b/.nextchanges/cli/aitools-auto-install-opt-out.md new file mode 100644 index 00000000000..cd24a1a300d --- /dev/null +++ b/.nextchanges/cli/aitools-auto-install-opt-out.md @@ -0,0 +1 @@ +* Add `databricks aitools auto-install disable` and `enable` to turn off (or back on) the automatic installation of Databricks AI tools on CLI upgrades. A global-scope `databricks aitools uninstall` also turns it off. ([#7701](https://github.com/databricks/cli/pull/7701)) diff --git a/acceptance/cmd/aitools/auto-install/enable-disable/out.test.toml b/acceptance/cmd/aitools/auto-install/enable-disable/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/cmd/aitools/auto-install/enable-disable/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/cmd/aitools/auto-install/enable-disable/output.txt b/acceptance/cmd/aitools/auto-install/enable-disable/output.txt new file mode 100644 index 00000000000..c5b3db00b1f --- /dev/null +++ b/acceptance/cmd/aitools/auto-install/enable-disable/output.txt @@ -0,0 +1,57 @@ + +=== nothing stored yet + +>>> auto_install_state +(no auto-install state) + +=== disable stores the opt-out + +>>> [CLI] aitools auto-install disable +Automatic installation of Databricks AI tools is disabled. + +>>> auto_install_state +{ + "opted_out": true +} + +=== enable clears it + +>>> [CLI] aitools auto-install enable +Automatic installation of Databricks AI tools is enabled. + +>>> auto_install_state +{} + +=== enable says when the environment still turns it off + +>>> DATABRICKS_AITOOLS_AUTO_INSTALL=false [CLI] aitools auto-install enable +Automatic installation of Databricks AI tools is enabled. +DATABRICKS_AITOOLS_AUTO_INSTALL still disables it in this environment; unset it to let it run. + +=== disable replaces a state file it cannot read + +>>> [CLI] aitools auto-install disable +Automatic installation of Databricks AI tools is disabled. + +>>> auto_install_state +{ + "opted_out": true +} + +=== a global uninstall turns it off + +>>> [CLI] aitools uninstall --scope=global +Uninstalled 1 skill. + +>>> auto_install_state +{ + "opted_out": true +} + +=== a project uninstall leaves it on + +>>> [CLI] aitools uninstall --scope=project +Uninstalled 1 skill. + +>>> auto_install_state +{} diff --git a/acceptance/cmd/aitools/auto-install/enable-disable/script b/acceptance/cmd/aitools/auto-install/enable-disable/script new file mode 100644 index 00000000000..7b802bfb85a --- /dev/null +++ b/acceptance/cmd/aitools/auto-install/enable-disable/script @@ -0,0 +1,35 @@ +sethome home +mkdir -p "${USERPROFILE:-$HOME}/.cursor" + +title "nothing stored yet" +trace auto_install_state + +title "disable stores the opt-out" +trace $CLI aitools auto-install disable +trace auto_install_state + +title "enable clears it" +trace $CLI aitools auto-install enable +trace auto_install_state + +title "enable says when the environment still turns it off" +trace DATABRICKS_AITOOLS_AUTO_INSTALL=false $CLI aitools auto-install enable + +title "disable replaces a state file it cannot read" +echo "not json" > "${USERPROFILE:-$HOME}/.databricks/aitools/auto-install.json" +trace $CLI aitools auto-install disable +trace auto_install_state +$CLI aitools auto-install enable > /dev/null 2>&1 + +title "a global uninstall turns it off" +$CLI aitools install --skills-only --scope=global --agents=cursor &> LOG.install-global +trace $CLI aitools uninstall --scope=global +trace auto_install_state +$CLI aitools auto-install enable > /dev/null 2>&1 + +title "a project uninstall leaves it on" +mkdir -p "${USERPROFILE:-$HOME}/project" +cd "${USERPROFILE:-$HOME}/project" +$CLI aitools install --skills-only --scope=project --agents=claude-code &> LOG.install-project +trace $CLI aitools uninstall --scope=project +trace auto_install_state diff --git a/acceptance/cmd/aitools/auto-install/script.prepare b/acceptance/cmd/aitools/auto-install/script.prepare new file mode 100644 index 00000000000..8f64b39a860 --- /dev/null +++ b/acceptance/cmd/aitools/auto-install/script.prepare @@ -0,0 +1,5 @@ +# auto_install_state prints the persisted auto-install state. +auto_install_state() { + local path="${USERPROFILE:-$HOME}/.databricks/aitools/auto-install.json" + if [ -f "$path" ]; then cat "$path"; else echo "(no auto-install state)"; fi +} diff --git a/acceptance/cmd/aitools/auto-install/test.toml b/acceptance/cmd/aitools/auto-install/test.toml new file mode 100644 index 00000000000..9fef185795b --- /dev/null +++ b/acceptance/cmd/aitools/auto-install/test.toml @@ -0,0 +1,35 @@ +# Automatic installation of Databricks AI tools on the first run of a new CLI +# version. The mock server replaces raw.githubusercontent.com (skills manifest +# and files). +Env.DATABRICKS_SKILLS_BASE_URL = "$DATABRICKS_HOST" +Env.DATABRICKS_SKILLS_REF = "test-ref" + +Ignore = [ + "home", +] + +[[Server]] +Pattern = "GET /test-ref/manifest.json" +Response.Body = ''' +{ + "version": "2", + "updated_at": "2026-01-01T00:00:00Z", + "skills": { + "databricks-test": { + "version": "1.0.0", + "description": "Test skill", + "files": ["SKILL.md"], + "repo_dir": "skills" + } + } +} +''' + +[[Server]] +Pattern = "GET /test-ref/skills/databricks-test/SKILL.md" +Response.Body = '''--- +name: databricks-test +--- + +# Test skill +''' diff --git a/cmd/aitools/aitools.go b/cmd/aitools/aitools.go index 503bfc634ec..2805d891ef5 100644 --- a/cmd/aitools/aitools.go +++ b/cmd/aitools/aitools.go @@ -31,4 +31,5 @@ https://github.com/databricks/databricks-agent-skills` cmd.AddCommand(NewUninstallCmd()) cmd.AddCommand(NewListCmd()) cmd.AddCommand(NewVersionCmd()) + cmd.AddCommand(newAutoInstallCmd()) } diff --git a/cmd/aitools/autoinstall.go b/cmd/aitools/autoinstall.go new file mode 100644 index 00000000000..7dd77896490 --- /dev/null +++ b/cmd/aitools/autoinstall.go @@ -0,0 +1,65 @@ +package aitools + +import ( + "context" + "fmt" + + "github.com/databricks/cli/libs/aitools/autoinstall" + "github.com/databricks/cli/libs/cmdio" + "github.com/spf13/cobra" +) + +func newAutoInstallCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "auto-install", + Short: "Turn the automatic installation of Databricks AI tools on or off", + Long: `Turn the automatic installation of Databricks AI tools on or off. + +On the first run of a new CLI version, the CLI can install the Databricks AI +tools (skills and plugins) for the coding agents already on this machine, as +'databricks aitools install --scope=global' does, or update an existing +installation, as 'databricks aitools update' does. It never runs in CI and only +touches the global scope. + +It stays off after 'databricks aitools auto-install disable' or a global-scope +'databricks aitools uninstall', until 'databricks aitools auto-install enable'. +Setting ` + autoinstall.EnvVar + `=false turns it off in that environment.`, + } + cmd.AddCommand(newAutoInstallToggleCmd(true)) + cmd.AddCommand(newAutoInstallToggleCmd(false)) + return cmd +} + +// newAutoInstallToggleCmd returns `auto-install enable` or `auto-install disable`. +func newAutoInstallToggleCmd(enable bool) *cobra.Command { + use, short := "disable", "Turn off the automatic installation of Databricks AI tools" + if enable { + use, short = "enable", "Turn the automatic installation of Databricks AI tools back on" + } + return &cobra.Command{ + Use: use, + Short: short, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + if err := autoinstall.SetOptedOut(ctx, !enable); err != nil { + return fmt.Errorf("failed to save the auto-install setting: %w", err) + } + printAutoInstallSetting(ctx, enable) + return nil + }, + } +} + +// printAutoInstallSetting confirms the stored setting. An enable that the +// environment still overrides says so, rather than looking like it worked. +func printAutoInstallSetting(ctx context.Context, enabled bool) { + if !enabled { + cmdio.LogString(ctx, "Automatic installation of Databricks AI tools is disabled.") + return + } + cmdio.LogString(ctx, "Automatic installation of Databricks AI tools is enabled.") + if autoinstall.EnvOptedOut(ctx) { + cmdio.LogString(ctx, cmdio.Yellow(ctx, autoinstall.EnvVar+" still disables it in this environment; unset it to let it run.")) + } +} diff --git a/cmd/aitools/uninstall.go b/cmd/aitools/uninstall.go index 787156ff519..5a12d73a8d9 100644 --- a/cmd/aitools/uninstall.go +++ b/cmd/aitools/uninstall.go @@ -7,8 +7,10 @@ import ( "slices" "strings" + "github.com/databricks/cli/libs/aitools/autoinstall" "github.com/databricks/cli/libs/aitools/installer" "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/telemetry/protos" "github.com/spf13/cobra" ) @@ -27,6 +29,10 @@ func NewUninstallCmd() *cobra.Command { By default, removes all skills. Use --skills to remove specific skills only. +A global-scope uninstall also turns off the automatic installation of +Databricks AI tools on CLI upgrades; 'databricks aitools auto-install enable' +turns it back on. + The command exits with code 1 when every removal it attempts fails.`, Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { @@ -103,6 +109,14 @@ The command exits with code 1 when every removal it attempts fails.`, if state != nil && skillsInstalled(state, opts.Skills) { waitForTelemetryAuth(ctx) } + // Auto-install only ever installs in global scope, so only a global + // uninstall could be undone by it on the next CLI upgrade. A partial + // one counts too: the update it runs re-adds removed skills. + if scope == installer.ScopeGlobal { + if err := autoinstall.SetOptedOut(ctx, true); err != nil { + log.Warnf(ctx, "Could not turn off the automatic installation of Databricks AI tools: %v", err) + } + } result, err := uninstallSkillsFn(ctx, opts) if err != nil { return err diff --git a/libs/aitools/autoinstall/state.go b/libs/aitools/autoinstall/state.go new file mode 100644 index 00000000000..5227b14ef53 --- /dev/null +++ b/libs/aitools/autoinstall/state.go @@ -0,0 +1,101 @@ +// Package autoinstall holds what the CLI remembers about installing Databricks +// AI tools automatically on the first run of a new CLI version. +package autoinstall + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + + "github.com/databricks/cli/libs/atomicfile" + "github.com/databricks/cli/libs/env" + "github.com/databricks/cli/libs/log" +) + +// stateFileName is the state file under ~/.databricks/aitools. It is not a +// libs/cache entry: cache entries are namespaced by CLI version and expire, +// and this state must outlive upgrades and `databricks cache clear`. +const stateFileName = "auto-install.json" + +// State is what auto-install remembers across CLI versions. +type State struct { + // LastVersion is the CLI version auto-install last ran for. + LastVersion string `json:"last_version,omitempty"` + + // Attempts counts the failed attempts for LastVersion. It is 0 once a run + // for LastVersion succeeded or was skipped for good. + Attempts int `json:"attempts,omitempty"` + + // OptedOut turns auto-install off for every later version. It is set by + // `databricks aitools auto-install disable` and by a global-scope + // `databricks aitools uninstall`. + OptedOut bool `json:"opted_out,omitempty"` +} + +// statePath returns ~/.databricks/aitools/auto-install.json. +func statePath(ctx context.Context) (string, error) { + home, err := env.UserHomeDir(ctx) + if err != nil { + return "", err + } + return filepath.Join(home, ".databricks", "aitools", stateFileName), nil +} + +// Load reads the state. A missing file is the zero State. +func Load(ctx context.Context) (State, error) { + path, err := statePath(ctx) + if err != nil { + return State{}, err + } + data, err := os.ReadFile(path) + if errors.Is(err, fs.ErrNotExist) { + return State{}, nil + } + if err != nil { + return State{}, err + } + var s State + if err := json.Unmarshal(data, &s); err != nil { + return State{}, fmt.Errorf("parse %s: %w", filepath.ToSlash(path), err) + } + return s, nil +} + +// Save writes the state atomically. +func Save(ctx context.Context, s State) error { + path, err := statePath(ctx) + if err != nil { + return err + } + data, err := json.MarshalIndent(s, "", " ") + if err != nil { + return err + } + return atomicfile.Write(path, append(data, '\n'), 0o600, atomicfile.MkDir(0o755)) +} + +// SetOptedOut turns auto-install off (true) or back on (false), keeping the +// rest of the state. A state file that cannot be read is replaced: the user's +// explicit choice matters more than a broken record. +func SetOptedOut(ctx context.Context, optedOut bool) error { + s, err := Load(ctx) + if err != nil { + log.Debugf(ctx, "Replacing unreadable auto-install state: %v", err) + s = State{} + } + s.OptedOut = optedOut + return Save(ctx, s) +} + +// EnvVar turns auto-install off when set to false (or 0, no, off). +const EnvVar = "DATABRICKS_AITOOLS_AUTO_INSTALL" + +// EnvOptedOut reports whether EnvVar turns auto-install off. +func EnvOptedOut(ctx context.Context) bool { + enabled, ok := env.GetBool(ctx, EnvVar) + return ok && !enabled +} diff --git a/libs/clicompat/clicompat.go b/libs/clicompat/clicompat.go index 446c8e4755c..7d9f6f746fa 100644 --- a/libs/clicompat/clicompat.go +++ b/libs/clicompat/clicompat.go @@ -56,6 +56,13 @@ const ( type Entry struct { AppKit string `json:"appkit"` AgentSkills string `json:"skills"` + + // AutoInstall turns on the automatic installation of Databricks AI tools + // on the first run of a new CLI version in this range. Off when absent, so + // the feature ships off and can be turned on (or off again) remotely; a + // change reaches a machine on its first run of a version after the change. + // CLIs that predate the field ignore it. + AutoInstall bool `json:"auto_install,omitempty"` } // Manifest is the compatibility manifest: a map of CLI version strings to entries. @@ -287,6 +294,16 @@ func ResolveAgentSkillsVersion(ctx context.Context) (string, error) { return entry.AgentSkills, nil } +// ResolveAutoInstall reports whether the manifest entry for cliVersion turns on +// the automatic installation of Databricks AI tools. +func ResolveAutoInstall(ctx context.Context, cliVersion string) (bool, error) { + entry, err := resolveEntry(ctx, cliVersion) + if err != nil { + return false, err + } + return entry.AutoInstall, nil +} + // --- Local manifest cache --- // manifestLocalPath returns the path to the locally cached manifest file. diff --git a/libs/clicompat/clicompat_test.go b/libs/clicompat/clicompat_test.go index 644df86efdf..1fda6a53979 100644 --- a/libs/clicompat/clicompat_test.go +++ b/libs/clicompat/clicompat_test.go @@ -324,6 +324,45 @@ func TestResolveAgentSkillsVersion(t *testing.T) { } } +func TestResolveAutoInstall(t *testing.T) { + manifest := `{ + "1.20.0": {"appkit": "0.84.0", "skills": "0.2.30"}, + "1.30.0": {"appkit": "0.84.0", "skills": "0.2.30", "auto_install": true} + }` + tests := []struct { + cliVersion string + want bool + }{ + {cliVersion: "1.25.0", want: false}, + {cliVersion: "1.30.0", want: true}, + {cliVersion: "1.31.2", want: true}, + } + for _, tc := range tests { + t.Run(tc.cliVersion, func(t *testing.T) { + ctx := testContext(t) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(manifest)) + })) + defer srv.Close() + redirectToServer(t, srv) + + got, err := ResolveAutoInstall(ctx, tc.cliVersion) + require.NoError(t, err) + assert.Equal(t, tc.want, got) + }) + } +} + +// The remote manifest is this same file on main, so turning auto_install on +// anywhere is a launch decision: change this test in the PR that makes it. +func TestManifestLeavesAutoInstallOff(t *testing.T) { + m, err := parseManifest(build.CLICompatManifestJSON) + require.NoError(t, err) + for version, entry := range m { + assert.False(t, entry.AutoInstall, "auto_install is on for %s", version) + } +} + // --- ResolveEmbeddedAppKitVersion --- func TestResolveEmbeddedAppKitVersion(t *testing.T) { From e5662aded5d467f3bdd83abd10f6fbffcaf23d8f Mon Sep 17 00:00:00 2001 From: Anton Nekipelov <226657+anton-107@users.noreply.github.com> Date: Mon, 16 Nov 2026 19:24:24 +0000 Subject: [PATCH 2/7] aitools: install AI tools after the first command of a new CLI version After a command succeeds, the root PersistentPostRunE installs (or updates) Databricks AI tools for the coding agents on the machine, once per new CLI version. It ships off: no cli-compat entry sets the gate. - autoinstall.Decide is a pure decision over the command, version, state, CI, opt-out, gate and holdout; skipped commands and development builds return before any state read. - A fresh install runs as `aitools install --scope=global` without prompts for agents whose config dir exists; an existing global install runs `aitools update --scope=global`, keeping edited skills. The run has a 10s budget, discards installer output and prints one notice line. - State is updated under OS file locks that are only tried, never waited on beyond 1s. Attempts are retried up to 10 times per version. - Agent CLIs run in their own process group so a timeout stops their descendants. Windows cannot do that, so auto-install skips plugin installs there. - Answering "no" at the `apps init` skills prompt opts out and is passed to the CLI children `apps init` starts. - Telemetry: aitools_install_event.trigger = AUTO_FIRST_RUN and a new aitools_auto_install_event. - Refactor: `aitools update`'s RunE body moves to runUpdate, versioncheck.InCI is exported, and the print_telemetry acceptance helper moves up a directory. Co-authored-by: Isaac --- .../auto-install/first-run/out.test.toml | 1 + .../aitools/auto-install/first-run/output.txt | 39 ++ .../cmd/aitools/auto-install/first-run/script | 21 + .../auto-install/retries/out.test.toml | 1 + .../aitools/auto-install/retries/output.txt | 43 ++ .../cmd/aitools/auto-install/retries/script | 22 + .../cmd/aitools/auto-install/script.prepare | 60 +++ .../auto-install/skipped/out.test.toml | 1 + .../aitools/auto-install/skipped/output.txt | 96 +++++ .../cmd/aitools/auto-install/skipped/script | 58 +++ acceptance/cmd/aitools/auto-install/test.toml | 10 +- .../auto-install/upgrade/out.test.toml | 1 + .../aitools/auto-install/upgrade/output.txt | 66 +++ .../cmd/aitools/auto-install/upgrade/script | 36 ++ .../aitools/auto-install/upgrade/test.toml | 73 ++++ acceptance/cmd/aitools/script.prepare | 13 + .../cmd/aitools/telemetry/script.prepare | 14 - cmd/aitools/autoinstall.go | 5 +- cmd/aitools/autoinstall_plan_test.go | 88 ++++ cmd/aitools/autoinstall_run.go | 379 +++++++++++++++++ cmd/aitools/telemetry.go | 2 + cmd/aitools/update.go | 380 +++++++++--------- cmd/apps/init.go | 3 + cmd/root/autoinstall_test.go | 62 +++ cmd/root/root.go | 7 + libs/aitools/agents/detect.go | 2 +- libs/aitools/agents/recommend.go | 9 +- libs/aitools/agents/recommend_test.go | 6 + libs/aitools/autoinstall/decide.go | 139 +++++++ libs/aitools/autoinstall/decide_test.go | 83 ++++ libs/aitools/autoinstall/hook.go | 10 + libs/aitools/autoinstall/lock_unix.go | 21 + libs/aitools/autoinstall/lock_windows.go | 22 + libs/aitools/autoinstall/run.go | 155 +++++++ libs/aitools/autoinstall/run_test.go | 108 +++++ libs/aitools/autoinstall/state.go | 83 +++- libs/aitools/installer/plugin.go | 4 +- libs/aitools/installer/plugin_unix_test.go | 23 ++ libs/aitools/installer/update.go | 12 + libs/aitools/installer/update_test.go | 36 ++ libs/telemetry/protos/README.md | 12 + libs/telemetry/protos/aitools_events.go | 56 +++ libs/telemetry/protos/aitools_install.go | 18 +- libs/telemetry/protos/frontend_log.go | 1 + libs/versioncheck/notice.go | 8 +- main.go | 7 + 46 files changed, 2084 insertions(+), 212 deletions(-) create mode 100644 acceptance/cmd/aitools/auto-install/first-run/out.test.toml create mode 100644 acceptance/cmd/aitools/auto-install/first-run/output.txt create mode 100644 acceptance/cmd/aitools/auto-install/first-run/script create mode 100644 acceptance/cmd/aitools/auto-install/retries/out.test.toml create mode 100644 acceptance/cmd/aitools/auto-install/retries/output.txt create mode 100644 acceptance/cmd/aitools/auto-install/retries/script create mode 100644 acceptance/cmd/aitools/auto-install/skipped/out.test.toml create mode 100644 acceptance/cmd/aitools/auto-install/skipped/output.txt create mode 100644 acceptance/cmd/aitools/auto-install/skipped/script create mode 100644 acceptance/cmd/aitools/auto-install/upgrade/out.test.toml create mode 100644 acceptance/cmd/aitools/auto-install/upgrade/output.txt create mode 100644 acceptance/cmd/aitools/auto-install/upgrade/script create mode 100644 acceptance/cmd/aitools/auto-install/upgrade/test.toml create mode 100644 acceptance/cmd/aitools/script.prepare create mode 100644 cmd/aitools/autoinstall_plan_test.go create mode 100644 cmd/aitools/autoinstall_run.go create mode 100644 cmd/root/autoinstall_test.go create mode 100644 libs/aitools/autoinstall/decide.go create mode 100644 libs/aitools/autoinstall/decide_test.go create mode 100644 libs/aitools/autoinstall/hook.go create mode 100644 libs/aitools/autoinstall/lock_unix.go create mode 100644 libs/aitools/autoinstall/lock_windows.go create mode 100644 libs/aitools/autoinstall/run.go create mode 100644 libs/aitools/autoinstall/run_test.go create mode 100644 libs/aitools/installer/plugin_unix_test.go diff --git a/acceptance/cmd/aitools/auto-install/first-run/out.test.toml b/acceptance/cmd/aitools/auto-install/first-run/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/cmd/aitools/auto-install/first-run/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/cmd/aitools/auto-install/first-run/output.txt b/acceptance/cmd/aitools/auto-install/first-run/output.txt new file mode 100644 index 00000000000..3a85c3b02e8 --- /dev/null +++ b/acceptance/cmd/aitools/auto-install/first-run/output.txt @@ -0,0 +1,39 @@ + +=== first run of a new version: installs for the detected agents, one stderr line + +>>> [CLI] jobs list +Installed Databricks AI tools for Claude Code, Cursor. Disable: databricks aitools auto-install disable + +>>> print_telemetry +{"command":"jobs_list","exit_code":0,"aitools_install_event":{"agents":["CLAUDE_CODE","CURSOR"],"scope":"GLOBAL","error_category":"TYPE_UNSPECIFIED","agent_results":[{"agent":"CLAUDE_CODE","error_category":"CLI_NOT_ON_PATH"}],"trigger":"AUTO_FIRST_RUN"}} +{"command":"jobs_list","exit_code":0,"aitools_auto_install_event":{"outcome":"INSTALLED","holdout_arm":"TREATMENT","interactive":false}} + +>>> cursor_skills +databricks-test + +>>> auto_install_state +{ + "last_version": "1.30.0", + "id": "treatment-machine" +} + +=== second run of the same version: nothing + +>>> [CLI] jobs list + +>>> print_telemetry + +=== uninstall turns it off: the next version installs nothing + +>>> [CLI] aitools uninstall +Uninstalled 1 skill. + +>>> DATABRICKS_AITOOLS_AUTO_INSTALL_CLI_VERSION=1.31.0 [CLI] jobs list + +>>> print_telemetry +{"command":"aitools_uninstall","exit_code":0,"aitools_command_event":{"command":"UNINSTALL","stage":"STARTED","interactive":false}} +{"command":"aitools_uninstall","exit_code":0,"aitools_command_event":{"command":"UNINSTALL","stage":"FINISHED","outcome":"SUCCESS","interactive":false,"scope":"GLOBAL"}} +{"command":"jobs_list","exit_code":0,"aitools_auto_install_event":{"outcome":"SKIPPED_OPTOUT","holdout_arm":"TREATMENT","interactive":false}} + +>>> cursor_skills +(no skills) diff --git a/acceptance/cmd/aitools/auto-install/first-run/script b/acceptance/cmd/aitools/auto-install/first-run/script new file mode 100644 index 00000000000..383f46fc466 --- /dev/null +++ b/acceptance/cmd/aitools/auto-install/first-run/script @@ -0,0 +1,21 @@ +use_home home +# Claude Code's config dir exists but its CLI is not on PATH (a desktop or IDE +# install), so it gets skill files, as `aitools install` gives it. +"$mkdir_bin" -p "${USERPROFILE:-$HOME}/.claude" +gate true + +title "first run of a new version: installs for the detected agents, one stderr line" +trace $CLI jobs list +trace print_telemetry +trace cursor_skills +trace auto_install_state + +title "second run of the same version: nothing" +trace $CLI jobs list +trace print_telemetry + +title "uninstall turns it off: the next version installs nothing" +trace $CLI aitools uninstall +trace DATABRICKS_AITOOLS_AUTO_INSTALL_CLI_VERSION=1.31.0 $CLI jobs list +trace print_telemetry +trace cursor_skills diff --git a/acceptance/cmd/aitools/auto-install/retries/out.test.toml b/acceptance/cmd/aitools/auto-install/retries/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/cmd/aitools/auto-install/retries/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/cmd/aitools/auto-install/retries/output.txt b/acceptance/cmd/aitools/auto-install/retries/output.txt new file mode 100644 index 00000000000..5e2d4d073bc --- /dev/null +++ b/acceptance/cmd/aitools/auto-install/retries/output.txt @@ -0,0 +1,43 @@ + +=== a failed attempt is silent and retried on the next run, up to 10 attempts + +>>> print_telemetry +9 {"command":"jobs_list","exit_code":0,"aitools_auto_install_event":{"outcome":"FAILED","holdout_arm":"TREATMENT","interactive":false,"error_category":"SKILLS_FETCH_FAILED"}} +9 {"command":"jobs_list","exit_code":0,"aitools_install_event":{"agents":["CURSOR"],"scope":"GLOBAL","error_category":"SKILLS_FETCH_FAILED","trigger":"AUTO_FIRST_RUN"}} + +>>> auto_install_state +{ + "last_version": "1.30.0", + "attempts": 9, + "id": "treatment-machine" +} + +>>> DATABRICKS_SKILLS_REF=broken-ref [CLI] jobs list + +>>> print_telemetry +{"command":"jobs_list","exit_code":0,"aitools_install_event":{"agents":["CURSOR"],"scope":"GLOBAL","error_category":"SKILLS_FETCH_FAILED","trigger":"AUTO_FIRST_RUN"}} +{"command":"jobs_list","exit_code":0,"aitools_auto_install_event":{"outcome":"RETRIES_EXHAUSTED","holdout_arm":"TREATMENT","interactive":false,"error_category":"SKILLS_FETCH_FAILED"}} + +=== after 10 attempts the version gets no more + +>>> [CLI] jobs list + +>>> print_telemetry + +>>> cursor_skills +(no skills) + +=== a new version starts over + +>>> DATABRICKS_AITOOLS_AUTO_INSTALL_CLI_VERSION=1.31.0 [CLI] jobs list +Installed Databricks AI tools for Cursor. Disable: databricks aitools auto-install disable + +>>> print_telemetry +{"command":"jobs_list","exit_code":0,"aitools_install_event":{"agents":["CURSOR"],"scope":"GLOBAL","error_category":"TYPE_UNSPECIFIED","trigger":"AUTO_FIRST_RUN"}} +{"command":"jobs_list","exit_code":0,"aitools_auto_install_event":{"outcome":"INSTALLED","holdout_arm":"TREATMENT","interactive":false}} + +>>> auto_install_state +{ + "last_version": "1.31.0", + "id": "treatment-machine" +} diff --git a/acceptance/cmd/aitools/auto-install/retries/script b/acceptance/cmd/aitools/auto-install/retries/script new file mode 100644 index 00000000000..e987c9f90c4 --- /dev/null +++ b/acceptance/cmd/aitools/auto-install/retries/script @@ -0,0 +1,22 @@ +use_home home +gate true + +title "a failed attempt is silent and retried on the next run, up to 10 attempts" +for i in 1 2 3 4 5 6 7 8 9; do + DATABRICKS_SKILLS_REF=broken-ref $CLI jobs list +done +# One line per distinct event with its count (uniq -c pads differently by platform). +trace print_telemetry | sort | uniq -c | sed 's/^ *//' +trace auto_install_state +trace DATABRICKS_SKILLS_REF=broken-ref $CLI jobs list +trace print_telemetry + +title "after 10 attempts the version gets no more" +trace $CLI jobs list +trace print_telemetry +trace cursor_skills + +title "a new version starts over" +trace DATABRICKS_AITOOLS_AUTO_INSTALL_CLI_VERSION=1.31.0 $CLI jobs list +trace print_telemetry +trace auto_install_state diff --git a/acceptance/cmd/aitools/auto-install/script.prepare b/acceptance/cmd/aitools/auto-install/script.prepare index 8f64b39a860..8d8617052af 100644 --- a/acceptance/cmd/aitools/auto-install/script.prepare +++ b/acceptance/cmd/aitools/auto-install/script.prepare @@ -3,3 +3,63 @@ auto_install_state() { local path="${USERPROFILE:-$HOME}/.databricks/aitools/auto-install.json" if [ -f "$path" ]; then cat "$path"; else echo "(no auto-install state)"; fi } + +# The host's own CI flag or opt-out would change every case. +unset CI DATABRICKS_AITOOLS_AUTO_INSTALL +# Telemetry is printed by print_telemetry. +unset DATABRICKS_CLI_DISABLE_TELEMETRY DATABRICKS_CLI_TELEMETRY_TIMEOUT + +# Test binaries are development builds, which never auto-install. Each case +# names the released version it simulates. +export DATABRICKS_AITOOLS_AUTO_INSTALL_CLI_VERSION=1.30.0 + +# use_home drops the PATH entries below, so resolve these first. +mkdir_bin=$(command -v mkdir) +rm_bin=$(command -v rm) +ls_bin=$(command -v ls) + +# use_home HOME [ID] isolates HOME with Cursor's config dir present, and stores +# the machine id ID (default: treatment-machine, whose holdout bucket is in the +# treatment arm; holdout-machine is in the holdout), so the arm does not +# depend on a random id. +# +# Auto-install acts only on agents whose config directory exists, and installs +# plugins through agent CLIs found on PATH. Keep both inside the test: point +# every agent config dir at the test HOME, and drop every PATH entry holding an +# agent CLI. Windows also finds a command under each PATHEXT extension. +use_home() { + sethome "$1" + unset CLAUDE_CONFIG_DIR PI_CODING_AGENT_DIR GEMINI_CLI_HOME GOOSE_PATH_ROOT + export XDG_CONFIG_HOME="${USERPROFILE:-$HOME}/.config" + export APPDATA="${USERPROFILE:-$HOME}/AppData/Roaming" + local exts="" filtered_path="" dir bin ext keep + if [ -n "${PATHEXT:-}" ]; then exts=$(printf '%s' "$PATHEXT" | tr ';' ':'); fi + local IFS=: + for dir in $PATH; do + keep=1 + for bin in claude cursor-agent codex opencode copilot pi gemini goose kiro; do + if [ -x "$dir/$bin" ]; then keep=; fi + for ext in $exts; do + if [ -f "$dir/$bin$ext" ]; then keep=; fi + done + done + if [ -n "$keep" ]; then filtered_path="$filtered_path:$dir"; fi + done + export PATH="${filtered_path#:}" + "$mkdir_bin" -p "${USERPROFILE:-$HOME}/.cursor" "${USERPROFILE:-$HOME}/.databricks/aitools" + echo "{\"id\": \"${2:-treatment-machine}\"}" > "${USERPROFILE:-$HOME}/.databricks/aitools/auto-install.json" +} + +# gate true|false writes a compatibility manifest into the CLI cache with that +# auto_install flag. A fresh cache file replaces the remote manifest. +gate() { + "$mkdir_bin" -p "$DATABRICKS_CACHE_DIR" + echo "{\"0.0.0\": {\"appkit\": \"0.84.0\", \"skills\": \"latest\", \"auto_install\": $1}}" > "$DATABRICKS_CACHE_DIR/compat-manifest.json" +} + +# cursor_skills lists the skills installed for Cursor. +cursor_skills() { + local skills + skills=$("$ls_bin" "${USERPROFILE:-$HOME}/.cursor/skills" 2>/dev/null || true) + echo "${skills:-(no skills)}" +} diff --git a/acceptance/cmd/aitools/auto-install/skipped/out.test.toml b/acceptance/cmd/aitools/auto-install/skipped/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/cmd/aitools/auto-install/skipped/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/cmd/aitools/auto-install/skipped/output.txt b/acceptance/cmd/aitools/auto-install/skipped/output.txt new file mode 100644 index 00000000000..55321b2a457 --- /dev/null +++ b/acceptance/cmd/aitools/auto-install/skipped/output.txt @@ -0,0 +1,96 @@ + +=== CI: not installed, reported once for the version + +>>> CI=true [CLI] jobs list + +>>> print_telemetry +{"command":"jobs_list","exit_code":0,"aitools_auto_install_event":{"outcome":"SKIPPED_CI","holdout_arm":"TREATMENT","interactive":false}} + +>>> CI=true [CLI] jobs list + +>>> print_telemetry + +>>> cursor_skills +(no skills) + +=== DATABRICKS_AITOOLS_AUTO_INSTALL=false: not installed + +>>> DATABRICKS_AITOOLS_AUTO_INSTALL=false [CLI] jobs list + +>>> print_telemetry +{"command":"jobs_list","exit_code":0,"aitools_auto_install_event":{"outcome":"SKIPPED_OPTOUT","holdout_arm":"TREATMENT","interactive":false}} + +=== auto-install disable: not installed + +>>> [CLI] aitools auto-install disable +Automatic installation of Databricks AI tools is disabled. + +>>> [CLI] jobs list + +>>> print_telemetry +{"command":"jobs_list","exit_code":0,"aitools_auto_install_event":{"outcome":"SKIPPED_OPTOUT","holdout_arm":"TREATMENT","interactive":false}} + +>>> cursor_skills +(no skills) + +>>> [CLI] aitools auto-install enable +Automatic installation of Databricks AI tools is enabled. + +=== commands that never trigger it + +>>> [CLI] version +Databricks CLI v[CLI_VERSION] + +>>> [CLI] aitools auto-install enable +Automatic installation of Databricks AI tools is enabled. + +>>> print_telemetry + +>>> DATABRICKS_CACHE_ENABLED=false [CLI] jobs list + +>>> print_telemetry + +>>> auto_install_state +{ + "last_version": "1.32.0", + "id": "treatment-machine" +} + +>>> cursor_skills +(no skills) + +=== the gate is off: nothing printed or reported, and the version is settled + +>>> [CLI] jobs list + +>>> print_telemetry + +>>> auto_install_state +{ + "last_version": "1.33.0", + "id": "treatment-machine" +} + +>>> [CLI] jobs list + +>>> print_telemetry + +>>> cursor_skills +(no skills) + +=== a machine in the holdout: not installed + +>>> [CLI] jobs list + +>>> print_telemetry +{"command":"jobs_list","exit_code":0,"aitools_auto_install_event":{"outcome":"SKIPPED_HOLDOUT","holdout_arm":"HOLDOUT","interactive":false}} + +>>> cursor_skills +(no skills) + +=== no agent config directory: nothing to install + +>>> [CLI] jobs list + +>>> print_telemetry +{"command":"jobs_list","exit_code":0,"aitools_auto_install_event":{"outcome":"SKIPPED_NO_AGENTS","holdout_arm":"TREATMENT","interactive":false}} diff --git a/acceptance/cmd/aitools/auto-install/skipped/script b/acceptance/cmd/aitools/auto-install/skipped/script new file mode 100644 index 00000000000..0e02ca9153d --- /dev/null +++ b/acceptance/cmd/aitools/auto-install/skipped/script @@ -0,0 +1,58 @@ +use_home home +gate true + +title "CI: not installed, reported once for the version" +trace CI=true $CLI jobs list +trace print_telemetry +trace CI=true $CLI jobs list +trace print_telemetry +trace cursor_skills + +export DATABRICKS_AITOOLS_AUTO_INSTALL_CLI_VERSION=1.31.0 +title "DATABRICKS_AITOOLS_AUTO_INSTALL=false: not installed" +trace DATABRICKS_AITOOLS_AUTO_INSTALL=false $CLI jobs list +trace print_telemetry + +export DATABRICKS_AITOOLS_AUTO_INSTALL_CLI_VERSION=1.32.0 +title "auto-install disable: not installed" +trace $CLI aitools auto-install disable +trace $CLI jobs list +trace print_telemetry +trace cursor_skills +trace $CLI aitools auto-install enable + +export DATABRICKS_AITOOLS_AUTO_INSTALL_CLI_VERSION=1.33.0 +title "commands that never trigger it" +trace $CLI version +trace $CLI aitools auto-install enable +$CLI completion bash > /dev/null 2>&1 +$CLI __complete jobs > /dev/null 2>&1 +$CLI help jobs > /dev/null 2>&1 +trace print_telemetry +trace DATABRICKS_CACHE_ENABLED=false $CLI jobs list +trace print_telemetry +trace auto_install_state +trace cursor_skills + +title "the gate is off: nothing printed or reported, and the version is settled" +gate false +trace $CLI jobs list +trace print_telemetry +trace auto_install_state +# Turning the gate on reaches this machine on its next new version. +gate true +trace $CLI jobs list +trace print_telemetry +trace cursor_skills + +title "a machine in the holdout: not installed" +use_home holdout-home holdout-machine +trace $CLI jobs list +trace print_telemetry +trace cursor_skills + +title "no agent config directory: nothing to install" +use_home no-agents-home +"$rm_bin" -r "${USERPROFILE:-$HOME}/.cursor" +trace $CLI jobs list +trace print_telemetry diff --git a/acceptance/cmd/aitools/auto-install/test.toml b/acceptance/cmd/aitools/auto-install/test.toml index 9fef185795b..fef284271cd 100644 --- a/acceptance/cmd/aitools/auto-install/test.toml +++ b/acceptance/cmd/aitools/auto-install/test.toml @@ -1,11 +1,15 @@ # Automatic installation of Databricks AI tools on the first run of a new CLI # version. The mock server replaces raw.githubusercontent.com (skills manifest -# and files). +# and files) and the GitHub API (latest skills release). Env.DATABRICKS_SKILLS_BASE_URL = "$DATABRICKS_HOST" +Env.DATABRICKS_CLI_GITHUB_API_URL = "$DATABRICKS_HOST" Env.DATABRICKS_SKILLS_REF = "test-ref" +RecordRequests = true Ignore = [ "home", + "holdout-home", + "no-agents-home", ] [[Server]] @@ -33,3 +37,7 @@ name: databricks-test # Test skill ''' + +[[Server]] +Pattern = "GET /broken-ref/manifest.json" +Response.StatusCode = 500 diff --git a/acceptance/cmd/aitools/auto-install/upgrade/out.test.toml b/acceptance/cmd/aitools/auto-install/upgrade/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/cmd/aitools/auto-install/upgrade/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/cmd/aitools/auto-install/upgrade/output.txt b/acceptance/cmd/aitools/auto-install/upgrade/output.txt new file mode 100644 index 00000000000..4eaed0398a9 --- /dev/null +++ b/acceptance/cmd/aitools/auto-install/upgrade/output.txt @@ -0,0 +1,66 @@ + +=== first run of 1.30.0 installs skills release 0.3.0 + +>>> [CLI] jobs list +Installed Databricks AI tools for Cursor. Disable: databricks aitools auto-install disable + +>>> print_telemetry +{"command":"jobs_list","exit_code":0,"aitools_install_event":{"agents":["CURSOR"],"scope":"GLOBAL","error_category":"TYPE_UNSPECIFIED","trigger":"AUTO_FIRST_RUN"}} +{"command":"jobs_list","exit_code":0,"aitools_auto_install_event":{"outcome":"INSTALLED","holdout_arm":"TREATMENT","interactive":false}} + +>>> cursor_skills +databricks-dropped +databricks-edited +databricks-kept + +=== upgrade to 1.31.0 with release 0.4.0: updates like aitools update, keeping the edited skills + +>>> DATABRICKS_AITOOLS_AUTO_INSTALL_CLI_VERSION=1.31.0 DATABRICKS_CLI_GITHUB_API_URL=[DATABRICKS_URL]/newer [CLI] jobs list +Updated Databricks AI tools for Cursor. Disable: databricks aitools auto-install disable + +>>> print_telemetry +{"command":"jobs_list","exit_code":0,"aitools_auto_install_event":{"outcome":"INSTALLED","holdout_arm":"TREATMENT","interactive":false}} + +>>> cursor_skills +databricks-dropped +databricks-edited +databricks-kept + +>>> skill_file databricks-kept +kept 2.0.0 + +>>> skill_file databricks-dropped +dropped 1.0.0 +my notes + +>>> skill_file databricks-edited +edited 1.0.0 +my notes + +=== upgrade to 1.32.0 while the latest release is 0.2.0: no downgrade, nothing printed, not retried + +>>> DATABRICKS_AITOOLS_AUTO_INSTALL_CLI_VERSION=1.32.0 DATABRICKS_CLI_GITHUB_API_URL=[DATABRICKS_URL]/older [CLI] jobs list + +>>> print_telemetry +{"command":"jobs_list","exit_code":0,"aitools_auto_install_event":{"outcome":"INSTALLED","holdout_arm":"TREATMENT","interactive":false}} + +>>> skill_file databricks-kept +kept 2.0.0 + +>>> auto_install_state +{ + "last_version": "1.32.0", + "id": "treatment-machine" +} + +=== an older CLI on the same machine (e.g. one an IDE ships) does nothing + +>>> DATABRICKS_AITOOLS_AUTO_INSTALL_CLI_VERSION=1.31.0 [CLI] jobs list + +>>> print_telemetry + +>>> auto_install_state +{ + "last_version": "1.32.0", + "id": "treatment-machine" +} diff --git a/acceptance/cmd/aitools/auto-install/upgrade/script b/acceptance/cmd/aitools/auto-install/upgrade/script new file mode 100644 index 00000000000..0be68b26379 --- /dev/null +++ b/acceptance/cmd/aitools/auto-install/upgrade/script @@ -0,0 +1,36 @@ +use_home home +# Track the latest skills release, as the compatibility manifest does. +unset DATABRICKS_SKILLS_REF +gate true +skills_dir="${USERPROFILE:-$HOME}/.databricks/aitools/skills" +skill_file() { + cat "$skills_dir/$1/SKILL.md" +} + +title "first run of 1.30.0 installs skills release 0.3.0" +trace $CLI jobs list +trace print_telemetry +trace cursor_skills + +# The user edits a skill that the next release drops, and one that it updates. +echo "my notes" >> "$skills_dir/databricks-dropped/SKILL.md" +echo "my notes" >> "$skills_dir/databricks-edited/SKILL.md" + +title "upgrade to 1.31.0 with release 0.4.0: updates like aitools update, keeping the edited skills" +trace DATABRICKS_AITOOLS_AUTO_INSTALL_CLI_VERSION=1.31.0 DATABRICKS_CLI_GITHUB_API_URL=$DATABRICKS_HOST/newer $CLI jobs list +trace print_telemetry +trace cursor_skills +trace skill_file databricks-kept +trace skill_file databricks-dropped +trace skill_file databricks-edited + +title "upgrade to 1.32.0 while the latest release is 0.2.0: no downgrade, nothing printed, not retried" +trace DATABRICKS_AITOOLS_AUTO_INSTALL_CLI_VERSION=1.32.0 DATABRICKS_CLI_GITHUB_API_URL=$DATABRICKS_HOST/older $CLI jobs list +trace print_telemetry +trace skill_file databricks-kept +trace auto_install_state + +title "an older CLI on the same machine (e.g. one an IDE ships) does nothing" +trace DATABRICKS_AITOOLS_AUTO_INSTALL_CLI_VERSION=1.31.0 $CLI jobs list +trace print_telemetry +trace auto_install_state diff --git a/acceptance/cmd/aitools/auto-install/upgrade/test.toml b/acceptance/cmd/aitools/auto-install/upgrade/test.toml new file mode 100644 index 00000000000..1511d4aaad2 --- /dev/null +++ b/acceptance/cmd/aitools/auto-install/upgrade/test.toml @@ -0,0 +1,73 @@ +# Skills releases: the latest release is v0.3.0, under /newer it is v0.4.0 and +# under /older it is v0.2.0 (DATABRICKS_CLI_GITHUB_API_URL picks one). + +[[Server]] +Pattern = "GET /repos/databricks/databricks-agent-skills/releases/latest" +Response.Body = '{"tag_name": "v0.3.0"}' + +[[Server]] +Pattern = "GET /newer/repos/databricks/databricks-agent-skills/releases/latest" +Response.Body = '{"tag_name": "v0.4.0"}' + +[[Server]] +Pattern = "GET /older/repos/databricks/databricks-agent-skills/releases/latest" +Response.Body = '{"tag_name": "v0.2.0"}' + +[[Server]] +Pattern = "GET /v0.3.0/manifest.json" +Response.Body = ''' +{ + "version": "2", + "updated_at": "2026-01-01T00:00:00Z", + "skills": { + "databricks-kept": {"version": "1.0.0", "description": "Kept", "files": ["SKILL.md"], "repo_dir": "skills"}, + "databricks-dropped": {"version": "1.0.0", "description": "Dropped", "files": ["SKILL.md"], "repo_dir": "skills"}, + "databricks-edited": {"version": "1.0.0", "description": "Edited", "files": ["SKILL.md"], "repo_dir": "skills"} + } +} +''' + +[[Server]] +Pattern = "GET /v0.3.0/skills/databricks-kept/SKILL.md" +Response.Body = "kept 1.0.0\n" + +[[Server]] +Pattern = "GET /v0.3.0/skills/databricks-dropped/SKILL.md" +Response.Body = "dropped 1.0.0\n" + +[[Server]] +Pattern = "GET /v0.3.0/skills/databricks-edited/SKILL.md" +Response.Body = "edited 1.0.0\n" + +[[Server]] +Pattern = "GET /v0.4.0/manifest.json" +Response.Body = ''' +{ + "version": "2", + "updated_at": "2026-02-01T00:00:00Z", + "skills": { + "databricks-kept": {"version": "2.0.0", "description": "Kept", "files": ["SKILL.md"], "repo_dir": "skills"}, + "databricks-edited": {"version": "2.0.0", "description": "Edited", "files": ["SKILL.md"], "repo_dir": "skills"} + } +} +''' + +[[Server]] +Pattern = "GET /v0.4.0/skills/databricks-kept/SKILL.md" +Response.Body = "kept 2.0.0\n" + +[[Server]] +Pattern = "GET /v0.4.0/skills/databricks-edited/SKILL.md" +Response.Body = "edited 2.0.0\n" + +[[Server]] +Pattern = "GET /v0.2.0/manifest.json" +Response.Body = ''' +{ + "version": "2", + "updated_at": "2025-12-01T00:00:00Z", + "skills": { + "databricks-kept": {"version": "0.5.0", "description": "Kept", "files": ["SKILL.md"], "repo_dir": "skills"} + } +} +''' diff --git a/acceptance/cmd/aitools/script.prepare b/acceptance/cmd/aitools/script.prepare new file mode 100644 index 00000000000..48a1478966f --- /dev/null +++ b/acceptance/cmd/aitools/script.prepare @@ -0,0 +1,13 @@ +# print_telemetry prints each telemetry event uploaded since the last call, one +# per line: the command it ran under, its exit code, and the event without the +# rest of the execution context (timings and versions vary). print_requests.py +# clears the recorded requests, so the next call sees only the next command's +# upload. It fails when no request was recorded, hence the file check. +print_telemetry() { + if [ -f "$OUT_REQUESTS" ]; then + print_requests.py --oneline //telemetry-ext | jq -c '.body.protoLogs[] | fromjson + | .entry.databricks_cli_log + | {command: .execution_context.command, exit_code: .execution_context.exit_code} + + (del(.execution_context))' + fi +} diff --git a/acceptance/cmd/aitools/telemetry/script.prepare b/acceptance/cmd/aitools/telemetry/script.prepare index f55e1cef710..9d3b1d18c4a 100644 --- a/acceptance/cmd/aitools/telemetry/script.prepare +++ b/acceptance/cmd/aitools/telemetry/script.prepare @@ -1,17 +1,3 @@ # A host that disables telemetry or sets a zero upload timeout would upload # nothing, so these cases would record no events. unset DATABRICKS_CLI_DISABLE_TELEMETRY DATABRICKS_CLI_TELEMETRY_TIMEOUT - -# print_telemetry prints each telemetry event uploaded since the last call, one -# per line: the command it ran under, its exit code, and the event without the -# rest of the execution context (timings and versions vary). print_requests.py -# clears the recorded requests, so the next call sees only the next command's -# upload. It fails when no request was recorded, hence the file check. -print_telemetry() { - if [ -f "$OUT_REQUESTS" ]; then - print_requests.py --oneline //telemetry-ext | jq -c '.body.protoLogs[] | fromjson - | .entry.databricks_cli_log - | {command: .execution_context.command, exit_code: .execution_context.exit_code} - + (del(.execution_context))' - fi -} diff --git a/cmd/aitools/autoinstall.go b/cmd/aitools/autoinstall.go index 7dd77896490..e650ec1db8e 100644 --- a/cmd/aitools/autoinstall.go +++ b/cmd/aitools/autoinstall.go @@ -21,8 +21,9 @@ tools (skills and plugins) for the coding agents already on this machine, as installation, as 'databricks aitools update' does. It never runs in CI and only touches the global scope. -It stays off after 'databricks aitools auto-install disable' or a global-scope -'databricks aitools uninstall', until 'databricks aitools auto-install enable'. +It stays off after 'databricks aitools auto-install disable', a global-scope +'databricks aitools uninstall', or a "no" to the skills prompt of +'databricks apps init', until 'databricks aitools auto-install enable'. Setting ` + autoinstall.EnvVar + `=false turns it off in that environment.`, } cmd.AddCommand(newAutoInstallToggleCmd(true)) diff --git a/cmd/aitools/autoinstall_plan_test.go b/cmd/aitools/autoinstall_plan_test.go new file mode 100644 index 00000000000..2fa0f4934b2 --- /dev/null +++ b/cmd/aitools/autoinstall_plan_test.go @@ -0,0 +1,88 @@ +package aitools + +import ( + "os" + "path/filepath" + "testing" + + "github.com/databricks/cli/libs/aitools/installer" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// autoInstallHome isolates HOME with Claude Code (its CLI on PATH) and Cursor +// detected. +func autoInstallHome(t *testing.T) string { + t.Helper() + tmp := t.TempDir() + isolateAgents(t, "HOME", tmp, "USERPROFILE", tmp, "XDG_CONFIG_HOME", filepath.Join(tmp, ".config")) + require.NoError(t, os.MkdirAll(filepath.Join(tmp, ".claude"), 0o755)) + require.NoError(t, os.MkdirAll(filepath.Join(tmp, ".cursor"), 0o755)) + fakeBinsOnPath(t, "claude") + return tmp +} + +func setAgentCLIsStoppable(t *testing.T, v bool) { + t.Helper() + orig := agentCLIsStoppable + t.Cleanup(func() { agentCLIsStoppable = orig }) + agentCLIsStoppable = v +} + +func planAgentNames(plan []agentPlanItem) []string { + var names []string + for _, it := range plan { + names = append(names, it.agent.DisplayName) + } + return names +} + +func TestPlanToolsRunsNoAgentCLIWhereTimeoutsCannotStopThem(t *testing.T) { + autoInstallHome(t) + + setAgentCLIsStoppable(t, true) + tools, err := planTools(t.Context()) + require.NoError(t, err) + assert.Equal(t, []string{"Claude Code", "Cursor"}, planAgentNames(tools.plan)) + + // The plugin install would run Claude Code's CLI. + setAgentCLIsStoppable(t, false) + tools, err = planTools(t.Context()) + require.NoError(t, err) + assert.Equal(t, []string{"Cursor"}, planAgentNames(tools.plan)) + assert.Empty(t, tools.keep) + + // Updating the recorded plugins would run agent CLIs too. + saveGlobalState(t, map[string]string{"databricks-sql": "0.1.0"}) + tools, err = planTools(t.Context()) + require.NoError(t, err) + assert.Nil(t, tools.existing) + assert.NotEmpty(t, tools.keep) + + setAgentCLIsStoppable(t, true) + tools, err = planTools(t.Context()) + require.NoError(t, err) + assert.NotNil(t, tools.existing) + assert.Empty(t, tools.keep) +} + +func TestPlanToolsKeepsSkillsWithoutInstallState(t *testing.T) { + autoInstallHome(t) + dir, err := installer.GlobalSkillsDir(t.Context()) + require.NoError(t, err) + // No checksums record what the CLI wrote, so edits cannot be told apart. + require.NoError(t, os.MkdirAll(filepath.Join(dir, "databricks-sql"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "databricks-sql", "SKILL.md"), []byte("user edit"), 0o644)) + + tools, err := planTools(t.Context()) + require.NoError(t, err) + assert.NotEmpty(t, tools.keep) + assert.Empty(t, tools.plan) + + result, err := tools.install(t.Context()) + require.NoError(t, err) + assert.Empty(t, result.agents) + data, err := os.ReadFile(filepath.Join(dir, "databricks-sql", "SKILL.md")) + require.NoError(t, err) + assert.Equal(t, "user edit", string(data)) +} diff --git a/cmd/aitools/autoinstall_run.go b/cmd/aitools/autoinstall_run.go new file mode 100644 index 00000000000..524e8457e10 --- /dev/null +++ b/cmd/aitools/autoinstall_run.go @@ -0,0 +1,379 @@ +package aitools + +import ( + "context" + "errors" + "fmt" + "io" + "log/slog" + "maps" + "runtime" + "slices" + "strings" + "time" + + "github.com/databricks/cli/libs/aitools/agents" + "github.com/databricks/cli/libs/aitools/autoinstall" + "github.com/databricks/cli/libs/aitools/installer" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/flags" + "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/telemetry" + "github.com/databricks/cli/libs/telemetry/protos" + "github.com/databricks/cli/libs/versioncheck" + "github.com/spf13/cobra" +) + +const ( + // gateTimeout bounds the gate lookup, one compatibility manifest fetch. + // A lookup that runs out reads as off: every machine pays it once per + // version while the gate is off, so it is kept short. + gateTimeout = 3 * time.Second + + // installTimeout bounds the install or update (its downloads and the + // agent CLIs it runs). One that runs out of time counts as a failed + // attempt and is retried on a later run. + installTimeout = 10 * time.Second +) + +// autoInstallDisable is the opt-out command named in the notice. +const autoInstallDisable = "databricks aitools auto-install disable" + +// AutoInstall installs Databricks AI tools for the coding agents on this +// machine, or updates an existing installation, on the first run of a new CLI +// version. The root command runs it after a command succeeded. It never fails +// the command and never writes to stdout: it prints at most one line to +// stderr, and why it skipped or failed only to the debug log. +func AutoInstall(cmd *cobra.Command) { + ctx := cmd.Context() + defer func() { + if r := recover(); r != nil { + log.Debugf(ctx, "auto-install: panic: %v", r) + } + }() + autoInstall(ctx, commandPath(cmd)) +} + +// commandPath returns the names of cmd and its parents below the root. +func commandPath(cmd *cobra.Command) []string { + var path []string + for c := cmd; c.HasParent(); c = c.Parent() { + path = append([]string{c.Name()}, path...) + } + return path +} + +func autoInstall(ctx context.Context, path []string) { + version := autoinstall.Version(ctx) + in := autoinstall.Inputs{ + Command: path, + CacheDisabled: autoinstall.CacheDisabled(ctx), + Version: version, + CI: versioncheck.InCI(ctx), + // A "no" earlier in this process counts even if storing it failed. + EnvOptOut: autoinstall.EnvOptedOut(ctx) || autoinstall.Declined(), + } + // Commands and builds that never auto-install skip before the state is + // read, and without a log line: they are most runs. + if reason := autoinstall.Decide(in); reason == autoinstall.ReasonSkippedCommand || reason == autoinstall.ReasonDevelopmentBuild { + return + } + var err error + in.State, err = autoinstall.Load(ctx) + if err != nil { + log.Debugf(ctx, "auto-install: skipped: %v", err) + return + } + if reason := autoinstall.Decide(in); !reason.Due() { + log.Debugf(ctx, "auto-install: skipped: %s", reason) + return + } + + unlock, ok := autoinstall.TryLock(ctx) + if !ok { + log.Debugf(ctx, "auto-install: skipped: another CLI process is running it") + return + } + defer unlock() + // Another process may have finished a run since the state was read. + in.State, err = autoinstall.Load(ctx) + if err != nil { + log.Debugf(ctx, "auto-install: skipped: %v", err) + return + } + id := in.State.ID + if id == "" { + id = autoinstall.NewID() + } + in.Bucket = autoinstall.Bucket(id) + + // The gate costs a manifest lookup, so it is resolved only when it is + // what decides the run. + reason := autoinstall.Decide(in) + if reason == autoinstall.ReasonGateOff { + gateCtx, cancel := context.WithTimeout(ctx, gateTimeout) + in.GateOn = autoinstall.GateOn(gateCtx, version) + cancel() + reason = autoinstall.Decide(in) + } + run := autoInstallRun{ + version: version, + id: id, + arm: holdoutArm(in.Bucket), + interactive: installPromptsSupported(ctx), + } + if !reason.Due() { + // Another process settled the version while this one waited. + log.Debugf(ctx, "auto-install: skipped: %s", reason) + return + } + if reason == autoinstall.ReasonGateOff { + // The feature is not launched, so the skip is not reported. + log.Debugf(ctx, "auto-install: skipped: %s", reason) + if err := autoinstall.Record(ctx, version, 0, id); err != nil { + log.Debugf(ctx, "auto-install: could not save the state: %v", err) + } + return + } + if outcome, ok := skipOutcomes[reason]; ok { + run.settle(ctx, outcome) + return + } + run.attempt(ctx, in.State) +} + +// attempt installs or updates the tools and reports the outcome. prior is the +// state the run was decided on. +func (r autoInstallRun) attempt(ctx context.Context, prior autoinstall.State) { + attempt := 1 + if prior.LastVersion == r.version { + attempt = prior.Attempts + 1 + } + qctx := quietContext(ctx) + tools, err := planTools(qctx) + // Count the attempt before making it: an attempt that never finishes (the + // process is killed) still counts, and a state that cannot be saved stops + // auto-install instead of retrying it on every run. + if recErr := autoinstall.Record(ctx, r.version, attempt, r.id); recErr != nil { + log.Debugf(ctx, "auto-install: skipped: could not save the state: %v", recErr) + return + } + var result installResult + if err == nil { + runCtx, cancel := context.WithTimeout(qctx, installTimeout) + result, err = tools.install(runCtx) + cancel() + } + if _, refused := errors.AsType[*installer.RefusedError](err); refused { + // The installed skills are newer than the release this CLI would + // install (the downgrade guard). They are installed, and retrying + // changes nothing until a newer release, which a later version gets. + log.Debugf(ctx, "auto-install: kept the installed tools: %v", err) + err = nil + } + switch { + case err != nil: + outcome := protos.AitoolsAutoInstallOutcomeFailed + if attempt >= autoinstall.MaxAttempts { + outcome = protos.AitoolsAutoInstallOutcomeRetriesExhausted + } + log.Debugf(ctx, "auto-install: attempt %d of %d failed: %v", attempt, autoinstall.MaxAttempts, err) + r.report(ctx, outcome, classifyInstallError(err)) + case result.noAgents: + r.settle(ctx, protos.AitoolsAutoInstallOutcomeSkippedNoAgents) + default: + r.settle(ctx, protos.AitoolsAutoInstallOutcomeInstalled) + if len(result.agents) > 0 { + verb := "Installed" + if result.updated { + verb = "Updated" + } + cmdio.LogString(ctx, fmt.Sprintf("%s Databricks AI tools for %s. Disable: %s", verb, strings.Join(result.agents, ", "), autoInstallDisable)) + } + } +} + +// skipOutcomes are the reported outcomes of the skips that settle a version. +var skipOutcomes = map[autoinstall.Reason]protos.AitoolsAutoInstallOutcome{ + autoinstall.ReasonSkippedCI: protos.AitoolsAutoInstallOutcomeSkippedCI, + autoinstall.ReasonSkippedOptOut: protos.AitoolsAutoInstallOutcomeSkippedOptOut, + autoinstall.ReasonSkippedHoldout: protos.AitoolsAutoInstallOutcomeSkippedHoldout, +} + +// autoInstallRun is what the outcome of one auto-install run is reported with. +type autoInstallRun struct { + version string + id string + arm protos.AitoolsHoldoutArm + interactive bool +} + +// settle records the version as done and reports outcome. +func (r autoInstallRun) settle(ctx context.Context, outcome protos.AitoolsAutoInstallOutcome) { + log.Debugf(ctx, "auto-install: %s", strings.ToLower(string(outcome))) + if err := autoinstall.Record(ctx, r.version, 0, r.id); err != nil { + log.Debugf(ctx, "auto-install: could not save the state: %v", err) + } + r.report(ctx, outcome, protos.AitoolsErrorCategoryUnspecified) +} + +// report buffers the auto-install event; cmd/root uploads it at exit when the +// command resolved auth. +func (r autoInstallRun) report(ctx context.Context, outcome protos.AitoolsAutoInstallOutcome, category protos.AitoolsErrorCategory) { + if category == protos.AitoolsErrorCategoryUnspecified { + category = "" + } + telemetry.Log(ctx, protos.DatabricksCliLog{ + AitoolsAutoInstallEvent: &protos.AitoolsAutoInstallEvent{ + Outcome: outcome, + HoldoutArm: r.arm, + Interactive: r.interactive, + ErrorCategory: category, + }, + }) +} + +func holdoutArm(bucket int) protos.AitoolsHoldoutArm { + if autoinstall.InHoldout(bucket) { + return protos.AitoolsHoldoutArmHoldout + } + return protos.AitoolsHoldoutArmTreatment +} + +// installResult is what one auto-install attempt did. +type installResult struct { + // noAgents is set when no supported agent has a config directory. + noAgents bool + // updated is set when an existing installation was updated. + updated bool + // agents names the agents that got new or updated tools. + agents []string +} + +// toolsPlan is what an auto-install attempt does: update the existing global +// installation, or install for the agents in plan. +type toolsPlan struct { + globalDir string + existing *installer.InstallState + plan []agentPlanItem + // keep, when set, is why the installed tools are left as they are. + keep string +} + +// planTools plans what `databricks aitools install --scope=global` runs +// without prompts, for the agents whose config directory exists. An existing +// global installation is updated as `databricks aitools update --scope=global` +// does instead, which keeps its downgrade guard, and keeps the skills whose +// files the user modified at their version. Planning reads only local state. +func planTools(ctx context.Context) (toolsPlan, error) { + globalDir, err := installer.GlobalSkillsDir(ctx) + if err != nil { + return toolsPlan{}, err + } + existing, err := installer.LoadState(globalDir) + if err != nil { + return toolsPlan{}, err + } + if existing != nil { + if !agentCLIsStoppable { + // The update runs agent CLIs to update recorded plugins and to + // migrate raw skills to plugins. + candidates, err := pluginMigrationCandidatesForScope(ctx, installer.ScopeGlobal, existing) + if err != nil { + return toolsPlan{}, err + } + if len(existing.Plugins) > 0 || len(candidates) > 0 { + return toolsPlan{keep: "the update would run agent CLIs"}, nil + } + } + return toolsPlan{globalDir: globalDir, existing: existing}, nil + } + // Skills without install state have no checksums to tell the user's edits + // from ours, and an install would replace them. + if installer.HasUntrackedSkills(ctx, installer.ScopeGlobal) { + return toolsPlan{keep: "skills without install state are on disk"}, nil + } + plan := buildPlan(ctx, agents.DetectInstalled(ctx), installer.ScopeGlobal, false, false) + if !agentCLIsStoppable { + plan = slices.DeleteFunc(plan, func(it agentPlanItem) bool { + return it.delivery == deliveryPlugin && !it.cliMissing + }) + } + return toolsPlan{globalDir: globalDir, plan: plan}, nil +} + +// agentCLIsStoppable reports whether a timeout stops an agent CLI together +// with its descendants. On Windows it stops only the CLI, and a descendant +// left running could still be installing when the next run takes the lock, +// so auto-install there runs no agent CLI. +var agentCLIsStoppable = runtime.GOOS != "windows" + +// install carries out the plan. +func (t toolsPlan) install(ctx context.Context) (installResult, error) { + if t.keep != "" { + log.Debugf(ctx, "auto-install: kept the installed tools: %s", t.keep) + return installResult{}, nil + } + if t.existing != nil { + return updateTools(ctx, t.globalDir, t.existing) + } + if len(t.plan) == 0 { + return installResult{noAgents: true}, nil + } + opts := installer.InstallOptions{Scope: installer.ScopeGlobal} + outcomes, err := executePlan(ctx, &installer.GitHubManifestSource{}, t.plan, opts, installInvocation{scope: installer.ScopeGlobal}, true) + topErr := topLevelFailure(err) + logInstallEvent(ctx, t.plan, installOpts{Scope: opts.Scope, Trigger: protos.AitoolsInstallTriggerAutoFirstRun}, classifyInstallError(topErr), outcomes) + if err != nil && topErr == nil { + // Reported per agent: take the category from the outcomes, as install does. + err = withCategory(err, firstFailureCategory(outcomes)) + } + var names []string + for _, o := range outcomes { + if o.status == outcomeInstalled { + names = append(names, o.agent.DisplayName) + } + } + return installResult{agents: names}, err +} + +// updateTools runs the update command's body for the global scope. Its funnel +// events and its wait for telemetry auth belong to the command, so they are +// detached: the auto-install event reports the run. +func updateTools(ctx context.Context, globalDir string, before *installer.InstallState) (installResult, error) { + uctx := telemetry.WithNewLogger(ctx) + uctx = context.WithValue(uctx, funnelKey{}, &commandFunnel{command: protos.AitoolsCommandUpdate}) + uctx = context.WithValue(uctx, telemetryAuthKey{}, &telemetryAuth{}) + if err := runUpdate(uctx, updateFlags{scope: installer.ScopeGlobal, keepModified: true}); err != nil { + return installResult{}, err + } + after, err := installer.LoadState(globalDir) + if err != nil { + return installResult{}, err + } + result := installResult{updated: true} + if after != nil && toolsChanged(before, after) { + for _, e := range buildAgentEntries(ctx, map[string]*installer.InstallState{installer.ScopeGlobal: after}) { + if _, ok := e.Installed[installer.ScopeGlobal]; ok { + result.agents = append(result.agents, e.DisplayName) + } + } + } + return result, nil +} + +// toolsChanged reports whether an update changed the installed skills or +// plugin versions. +func toolsChanged(before, after *installer.InstallState) bool { + return before.Release != after.Release || + !maps.Equal(before.Skills, after.Skills) || + !maps.EqualFunc(before.Plugins, after.Plugins, func(a, b installer.PluginRecord) bool { return a.Version == b.Version }) +} + +// quietContext discards what the installer prints and logs its warnings at +// debug level, so auto-install shows only its notice and its failures appear +// only in the debug log. +func quietContext(ctx context.Context) context.Context { + ctx = cmdio.InContext(ctx, cmdio.NewIO(ctx, flags.OutputText, nil, io.Discard, io.Discard, "", "")) + return log.NewContext(ctx, slog.New(debugOnlyHandler{log.GetLogger(ctx).Handler()})) +} diff --git a/cmd/aitools/telemetry.go b/cmd/aitools/telemetry.go index 4e451feb6eb..25d9be40308 100644 --- a/cmd/aitools/telemetry.go +++ b/cmd/aitools/telemetry.go @@ -190,6 +190,7 @@ func (h debugOnlyHandler) WithGroup(name string) slog.Handler { type installOpts struct { Scope string Experimental bool + Trigger protos.AitoolsInstallTrigger } // logInstallEvent buffers an install event; cmd/root uploads it at exit. @@ -205,6 +206,7 @@ func logInstallEvent(ctx context.Context, plan []agentPlanItem, opts installOpts Experimental: opts.Experimental, ErrorCategory: errCategory, AgentResults: agentResultsField(outcomes), + Trigger: opts.Trigger, }, }) } diff --git a/cmd/aitools/update.go b/cmd/aitools/update.go index bdddafbc56c..9c6269d35c3 100644 --- a/cmd/aitools/update.go +++ b/cmd/aitools/update.go @@ -42,9 +42,7 @@ type pluginMigrationResult struct { } func NewUpdateCmd() *cobra.Command { - var check, force, noNew, noPrune bool - var skillsFlag, scopeFlag string - var projectFlag, globalFlag bool + var f updateFlags cmd := &cobra.Command{ Use: "update", @@ -58,212 +56,228 @@ preview what would change without downloading. The command exits with code 1 when every update or check it attempts fails.`, Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { - ctx := cmd.Context() + return runUpdate(cmd.Context(), f) + }, + } - globalDir, err := installer.GlobalSkillsDir(ctx) - if err != nil { - return err - } - projectDir, err := installer.ProjectSkillsDir(ctx) + cmd.Flags().BoolVar(&f.check, "check", false, "Show what would be updated without downloading") + cmd.Flags().BoolVar(&f.force, "force", false, "Re-download even if versions match") + cmd.Flags().BoolVar(&f.noNew, "no-new", false, "Don't auto-install new skills from manifest") + cmd.Flags().BoolVar(&f.noPrune, "no-prune", false, "Keep skills that vanished from the manifest instead of removing them") + cmd.Flags().StringVar(&f.skills, "skills", "", "Specific skills to update (comma-separated)") + cmd.Flags().StringVar(&f.scope, "scope", "", "Update scope: project, global, or both") + cmd.Flags().BoolVar(&f.project, "project", false, "Update project-scoped skills") + cmd.Flags().BoolVar(&f.global, "global", false, "Update globally-scoped skills") + markScopeBoolsDeprecated(cmd) + return instrumentCommand(cmd, protos.AitoolsCommandUpdate) +} + +// updateFlags are the flags of `databricks aitools update`. +type updateFlags struct { + check, force, noNew, noPrune bool + skills, scope string + project, global bool + // keepModified is not a flag: auto-install sets it (see + // installer.UpdateOptions.KeepModified). + keepModified bool +} + +// runUpdate is the body of `databricks aitools update`. Auto-install runs it +// too, to update an existing installation. +func runUpdate(ctx context.Context, f updateFlags) error { + globalDir, err := installer.GlobalSkillsDir(ctx) + if err != nil { + return err + } + projectDir, err := installer.ProjectSkillsDir(ctx) + if err != nil { + return err + } + + projectFlag, globalFlag, err := parseScopeFlag(f.scope, f.project, f.global, true) + if err != nil { + return withCategory(err, protos.AitoolsErrorCategoryInvalidFlags) + } + // Record a scope flag before resolving, so a scope error still reports it. + funnel := funnelFrom(ctx) + funnel.setScope(explicitScope(projectFlag, globalFlag)) + + scopes, err := resolveScopeForUpdate(ctx, projectFlag, globalFlag, globalDir, projectDir) + if err != nil { + return err + } + // The scope enum has no "both"; a run over both scopes reports none. + if len(scopes) == 1 { + funnel.setScope(scopes[0]) + } + + installed := agents.DetectInstalled(ctx) + src := &installer.GitHubManifestSource{} + skills := splitAndTrim(f.skills) + + // The plugin version recorded on update tracks the skills release. + // Tolerate a resolution error (skip plugin updates, still do skills). + ref, _, refErr := updateGetSkillsRefFn(ctx) + + // A failed or skipped plugin update, or a check whose manifest could + // not be fetched, is only a warning, unless every attempt failed and + // nothing else ran. A plugin record for an agent this CLI does not know + // is not an attempt: no command can act on it. + // firstErr is the first failure of any kind; firstAttemptErr the first + // one of an attempt (a record of an unknown agent is not one). + var firstErr, firstAttemptErr error + updatedAnything := false + + for _, scope := range scopes { + if len(scopes) > 1 { + cmdio.LogString(ctx, fmt.Sprintf("Updating %s skills...", scope)) + } + + dir := globalDir + if scope == installer.ScopeProject { + dir = projectDir + } + state, err := installer.LoadState(dir) + if err != nil { + return err + } + // A scope with something installed fetches releases and runs agent + // CLIs, so the run waits for the telemetry auth. Without state the + // run only reports that nothing (or a legacy install) is there. + if state != nil { + waitForTelemetryAuth(ctx) + } + + if f.check && refErr == nil && state != nil && len(state.Plugins) > 0 { + printPluginCheckResults(ctx, state, installer.DisplaySkillsVersion(ref)) + updatedAnything = true + } + + migrationCandidates, err := pluginMigrationCandidatesForScope(ctx, scope, state) + if err != nil { + return err + } + if f.check { + printPluginMigrationCheckResults(ctx, migrationCandidates) + } + + // Without a ref the recorded plugins are skipped (with --check, + // not checked). + if refErr != nil && state != nil && hasKnownPluginRecord(state) { + firstAttemptErr = cmp.Or(firstAttemptErr, refErr) + firstErr = cmp.Or(firstErr, refErr) + } + + // Update plugin agents through their own CLI (check mode skips this). + if !f.check && refErr == nil { + pluginUpdates, err := updatePluginsFn(ctx, scope, ref) if err != nil { return err } - - projectFlag, globalFlag, err := parseScopeFlag(scopeFlag, projectFlag, globalFlag, true) - if err != nil { - return withCategory(err, protos.AitoolsErrorCategoryInvalidFlags) + for _, pu := range pluginUpdates { + if pu.Err != nil { + if !errors.Is(pu.Err, installer.ErrUnknownAgent) { + firstAttemptErr = cmp.Or(firstAttemptErr, pu.Err) + } + firstErr = cmp.Or(firstErr, pu.Err) + continue + } + updatedAnything = true + cmdio.LogString(ctx, fmt.Sprintf(" %s databricks plugin %s", pu.Agent, versionToken(pu.Version))) } - // Record a scope flag before resolving, so a scope error still reports it. - funnel := funnelFrom(ctx) - funnel.setScope(explicitScope(projectFlag, globalFlag)) - scopes, err := resolveScopeForUpdate(ctx, projectFlag, globalFlag, globalDir, projectDir) + migrated, err := migrateLegacyRawSkillsToPlugins(ctx, scope, ref, migrationCandidates) if err != nil { return err } - // The scope enum has no "both"; a run over both scopes reports none. - if len(scopes) == 1 { - funnel.setScope(scopes[0]) - } - - installed := agents.DetectInstalled(ctx) - src := &installer.GitHubManifestSource{} - skills := splitAndTrim(skillsFlag) - - // The plugin version recorded on update tracks the skills release. - // Tolerate a resolution error (skip plugin updates, still do skills). - ref, _, refErr := updateGetSkillsRefFn(ctx) - - // A failed or skipped plugin update, or a check whose manifest could - // not be fetched, is only a warning, unless every attempt failed and - // nothing else ran. A plugin record for an agent this CLI does not know - // is not an attempt: no command can act on it. - // firstErr is the first failure of any kind; firstAttemptErr the first - // one of an attempt (a record of an unknown agent is not one). - var firstErr, firstAttemptErr error - updatedAnything := false - - for _, scope := range scopes { - if len(scopes) > 1 { - cmdio.LogString(ctx, fmt.Sprintf("Updating %s skills...", scope)) - } - - dir := globalDir - if scope == installer.ScopeProject { - dir = projectDir - } - state, err := installer.LoadState(dir) + if migrated { + updatedAnything = true + state, err = installer.LoadState(dir) if err != nil { return err } - // A scope with something installed fetches releases and runs agent - // CLIs, so the run waits for the telemetry auth. Without state the - // run only reports that nothing (or a legacy install) is there. - if state != nil { - waitForTelemetryAuth(ctx) - } + } + } - if check && refErr == nil && state != nil && len(state.Plugins) > 0 { - printPluginCheckResults(ctx, state, installer.DisplaySkillsVersion(ref)) - updatedAnything = true - } + // Reconcile file skills for non-plugin agents. Skip entirely for a + // pure-plugin install (no file skills to maintain); keep calling for + // no-state installs so the "no skills or plugins installed" guidance still fires. + if state == nil || len(state.Skills) > 0 { + opts := installer.UpdateOptions{ + Check: f.check, + Force: f.force, + NoNew: f.noNew, + NoPrune: f.noPrune, + Scope: scope, + + KeepModified: f.keepModified, + } + opts.Skills = skills - migrationCandidates, err := pluginMigrationCandidatesForScope(ctx, scope, state) + targetAgents := installed + if scope == installer.ScopeProject { + cwd, err := os.Getwd() if err != nil { return err } - if check { - printPluginMigrationCheckResults(ctx, migrationCandidates) - } - - // Without a ref the recorded plugins are skipped (with --check, - // not checked). - if refErr != nil && state != nil && hasKnownPluginRecord(state) { - firstAttemptErr = cmp.Or(firstAttemptErr, refErr) - firstErr = cmp.Or(firstErr, refErr) - } - - // Update plugin agents through their own CLI (check mode skips this). - if !check && refErr == nil { - pluginUpdates, err := updatePluginsFn(ctx, scope, ref) - if err != nil { - return err - } - for _, pu := range pluginUpdates { - if pu.Err != nil { - if !errors.Is(pu.Err, installer.ErrUnknownAgent) { - firstAttemptErr = cmp.Or(firstAttemptErr, pu.Err) - } - firstErr = cmp.Or(firstErr, pu.Err) - continue - } - updatedAnything = true - cmdio.LogString(ctx, fmt.Sprintf(" %s databricks plugin %s", pu.Agent, versionToken(pu.Version))) - } + targetAgents = mergeAgents(installed, agents.DetectProjectInstalled(cwd)) + } - migrated, err := migrateLegacyRawSkillsToPlugins(ctx, scope, ref, migrationCandidates) - if err != nil { - return err - } - if migrated { - updatedAnything = true - state, err = installer.LoadState(dir) - if err != nil { - return err - } + fileAgents, manifestPlugins := excludePluginAgents(ctx, scope, targetAgents, state) + // Raw skills an earlier install gave an agent that now has the + // plugin would show every skill twice, so remove them as + // install does when it keeps the plugin. + if !f.check { + for _, a := range manifestPlugins { + if err := updateCleanupLegacyFn(ctx, a, scope); err != nil { + log.Debugf(ctx, "Legacy skill cleanup for %s failed: %v", a.DisplayName, err) } } + } + // The plugin agents left no agent that can take file skills in + // this scope, so there is nothing to reconcile. Without this, a + // project whose only project-capable agent has the plugin would + // fail the update with "no agents support project-scoped skills". + // Without install state the reconcile still runs, for the + // "no skills or plugins installed" and legacy-install guidance. + if state != nil && len(manifestPlugins) > 0 && !slices.ContainsFunc(fileAgents, func(a *agents.Agent) bool { + return scope != installer.ScopeProject || a.SupportsProjectScope + }) { + continue + } - // Reconcile file skills for non-plugin agents. Skip entirely for a - // pure-plugin install (no file skills to maintain); keep calling for - // no-state installs so the "no skills or plugins installed" guidance still fires. - if state == nil || len(state.Skills) > 0 { - opts := installer.UpdateOptions{ - Check: check, - Force: force, - NoNew: noNew, - NoPrune: noPrune, - Scope: scope, - } - opts.Skills = skills - - targetAgents := installed - if scope == installer.ScopeProject { - cwd, err := os.Getwd() - if err != nil { - return err - } - targetAgents = mergeAgents(installed, agents.DetectProjectInstalled(cwd)) - } - - fileAgents, manifestPlugins := excludePluginAgents(ctx, scope, targetAgents, state) - // Raw skills an earlier install gave an agent that now has the - // plugin would show every skill twice, so remove them as - // install does when it keeps the plugin. - if !check { - for _, a := range manifestPlugins { - if err := updateCleanupLegacyFn(ctx, a, scope); err != nil { - log.Debugf(ctx, "Legacy skill cleanup for %s failed: %v", a.DisplayName, err) - } - } - } - // The plugin agents left no agent that can take file skills in - // this scope, so there is nothing to reconcile. Without this, a - // project whose only project-capable agent has the plugin would - // fail the update with "no agents support project-scoped skills". - // Without install state the reconcile still runs, for the - // "no skills or plugins installed" and legacy-install guidance. - if state != nil && len(manifestPlugins) > 0 && !slices.ContainsFunc(fileAgents, func(a *agents.Agent) bool { - return scope != installer.ScopeProject || a.SupportsProjectScope - }) { - continue - } - - result, err := updateSkillsFn(ctx, src, fileAgents, opts) - if err != nil { - return err - } - if result != nil && result.CheckErr != nil { - firstAttemptErr = cmp.Or(firstAttemptErr, result.CheckErr) - firstErr = cmp.Or(firstErr, result.CheckErr) - } else { - updatedAnything = true - } - // A check that could not fetch the manifest has nothing to report. - if result != nil && result.CheckErr == nil && (check || len(result.Updated) > 0 || len(result.Added) > 0 || len(result.Removed) > 0) { - text := installer.FormatUpdateResult(result, check) - if text != "No changes." || len(migrationCandidates) == 0 { - cmdio.LogString(ctx, text) - } - } - } + result, err := updateSkillsFn(ctx, src, fileAgents, opts) + if err != nil { + return err } - if firstErr != nil && !updatedAnything { - if firstAttemptErr == nil { - // Only unknown-agent records: warned about above, but not a run - // that failed to do what it could. - funnel.finishWith(ctx, protos.AitoolsCommandOutcomeFailure, classifyInstallError(firstErr)) - return nil - } - if check { - return fmt.Errorf("nothing was checked: %w", firstAttemptErr) + if result != nil && result.CheckErr != nil { + firstAttemptErr = cmp.Or(firstAttemptErr, result.CheckErr) + firstErr = cmp.Or(firstErr, result.CheckErr) + } else { + updatedAnything = true + } + // A check that could not fetch the manifest has nothing to report. + if result != nil && result.CheckErr == nil && (f.check || len(result.Updated) > 0 || len(result.Added) > 0 || len(result.Removed) > 0) { + text := installer.FormatUpdateResult(result, f.check) + if text != "No changes." || len(migrationCandidates) == 0 { + cmdio.LogString(ctx, text) } - return fmt.Errorf("nothing was updated: %w", firstAttemptErr) } + } + } + if firstErr != nil && !updatedAnything { + if firstAttemptErr == nil { + // Only unknown-agent records: warned about above, but not a run + // that failed to do what it could. + funnel.finishWith(ctx, protos.AitoolsCommandOutcomeFailure, classifyInstallError(firstErr)) return nil - }, + } + if f.check { + return fmt.Errorf("nothing was checked: %w", firstAttemptErr) + } + return fmt.Errorf("nothing was updated: %w", firstAttemptErr) } - - cmd.Flags().BoolVar(&check, "check", false, "Show what would be updated without downloading") - cmd.Flags().BoolVar(&force, "force", false, "Re-download even if versions match") - cmd.Flags().BoolVar(&noNew, "no-new", false, "Don't auto-install new skills from manifest") - cmd.Flags().BoolVar(&noPrune, "no-prune", false, "Keep skills that vanished from the manifest instead of removing them") - cmd.Flags().StringVar(&skillsFlag, "skills", "", "Specific skills to update (comma-separated)") - cmd.Flags().StringVar(&scopeFlag, "scope", "", "Update scope: project, global, or both") - cmd.Flags().BoolVar(&projectFlag, "project", false, "Update project-scoped skills") - cmd.Flags().BoolVar(&globalFlag, "global", false, "Update globally-scoped skills") - markScopeBoolsDeprecated(cmd) - return instrumentCommand(cmd, protos.AitoolsCommandUpdate) + return nil } func pluginMigrationCandidatesForScope(ctx context.Context, scope string, state *installer.InstallState) ([]pluginMigrationCandidate, error) { diff --git a/cmd/apps/init.go b/cmd/apps/init.go index ed79271d25f..c010ad1a536 100644 --- a/cmd/apps/init.go +++ b/cmd/apps/init.go @@ -18,6 +18,7 @@ import ( "github.com/charmbracelet/huh" "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/aitools/agents" + "github.com/databricks/cli/libs/aitools/autoinstall" "github.com/databricks/cli/libs/aitools/installer" "github.com/databricks/cli/libs/apps/generator" "github.com/databricks/cli/libs/apps/initializer" @@ -1968,6 +1969,7 @@ func runPostCreateDeploy(ctx context.Context, profile string) error { cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr cmd.Stdin = os.Stdin + cmd.Env = autoinstall.ChildEnv() return cmd.Run() } @@ -1997,6 +1999,7 @@ func runPostCreateDev(ctx context.Context, mode prompt.RunMode, projectInit init cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr cmd.Stdin = os.Stdin + cmd.Env = autoinstall.ChildEnv() return cmd.Run() default: return nil diff --git a/cmd/root/autoinstall_test.go b/cmd/root/autoinstall_test.go new file mode 100644 index 00000000000..86b25091a65 --- /dev/null +++ b/cmd/root/autoinstall_test.go @@ -0,0 +1,62 @@ +package root_test + +import ( + "context" + "errors" + "io" + "testing" + + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/aitools/autoinstall" + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// setAutoInstallRun replaces autoinstall.Run for the test. The variable is +// global, so tests that call it must not run in parallel. +func setAutoInstallRun(t *testing.T, run func(*cobra.Command)) { + prev := autoinstall.Run + autoinstall.Run = run + t.Cleanup(func() { autoinstall.Run = prev }) +} + +func newAutoInstallTestRoot(t *testing.T) *root.Tree { + r := root.NewTree(t.Context()) + r.SetOut(io.Discard) + r.SetErr(io.Discard) + r.Add(root.CommandSpec{ + Use: "jobs", + Short: "jobs commands", + Build: func(ctx context.Context, cmd *cobra.Command) { + cmd.AddCommand(&cobra.Command{Use: "ok", RunE: func(*cobra.Command, []string) error { return nil }}) + cmd.AddCommand(&cobra.Command{Use: "fail", RunE: func(*cobra.Command, []string) error { return errors.New("failed") }}) + }, + }) + return r +} + +func TestPostRunCallsAutoInstallWithTheCommand(t *testing.T) { + var ran []string + setAutoInstallRun(t, func(cmd *cobra.Command) { ran = append(ran, cmd.CommandPath()) }) + + err := newAutoInstallTestRoot(t).Execute(t.Context(), []string{"jobs", "ok"}) + require.NoError(t, err) + assert.Equal(t, []string{"databricks jobs ok"}, ran) +} + +func TestPostRunSkipsAutoInstallAfterAFailedCommand(t *testing.T) { + var ran []string + setAutoInstallRun(t, func(cmd *cobra.Command) { ran = append(ran, cmd.CommandPath()) }) + + err := newAutoInstallTestRoot(t).Execute(t.Context(), []string{"jobs", "fail"}) + require.Error(t, err) + assert.Empty(t, ran) +} + +func TestPostRunWithoutAutoInstall(t *testing.T) { + setAutoInstallRun(t, nil) + + err := newAutoInstallTestRoot(t).Execute(t.Context(), []string{"jobs", "ok"}) + assert.NoError(t, err) +} diff --git a/cmd/root/root.go b/cmd/root/root.go index a988c22d285..5fc80ec87a4 100644 --- a/cmd/root/root.go +++ b/cmd/root/root.go @@ -13,6 +13,7 @@ import ( "github.com/databricks/cli/internal/build" "github.com/databricks/cli/libs/aitools/agents" + "github.com/databricks/cli/libs/aitools/autoinstall" "github.com/databricks/cli/libs/aitools/installer" "github.com/databricks/cli/libs/auth" "github.com/databricks/cli/libs/cmdctx" @@ -102,6 +103,12 @@ func newRootCommand(ctx context.Context) *cobra.Command { cmd.PersistentPostRunE = func(cmd *cobra.Command, args []string) error { // Wait for any active Bubble Tea programs to finish and restore terminal state cmdio.Wait(cmd.Context()) + + // On the first run of a new CLI version, install Databricks AI tools. + // Cobra runs post-run hooks only when the command succeeded. + if autoinstall.Run != nil { + autoinstall.Run(cmd) + } return nil } diff --git a/libs/aitools/agents/detect.go b/libs/aitools/agents/detect.go index 53ae8d085f8..f071e6c77e1 100644 --- a/libs/aitools/agents/detect.go +++ b/libs/aitools/agents/detect.go @@ -112,7 +112,7 @@ func (a *Agent) ProbePlugin(ctx context.Context) error { cctx, cancel := context.WithTimeout(ctx, pluginProbeTimeout) defer cancel() - if _, err := process.Background(cctx, []string{path, "plugin", "--help"}); err != nil { + if _, err := process.Background(cctx, []string{path, "plugin", "--help"}, process.WithProcessGroup()); err != nil { return fmt.Errorf("%s plugin subcommand unavailable: %w", a.Binary, errors.Join(ErrPluginUnsupported, err)) } return nil diff --git a/libs/aitools/agents/recommend.go b/libs/aitools/agents/recommend.go index ea66ebe4016..d968d994f21 100644 --- a/libs/aitools/agents/recommend.go +++ b/libs/aitools/agents/recommend.go @@ -3,6 +3,7 @@ package agents import ( "context" + "github.com/databricks/cli/libs/aitools/autoinstall" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/dbr" "github.com/databricks/cli/libs/log" @@ -53,7 +54,8 @@ func MaybeVersionCheckTip(ctx context.Context, installedPlugins func(context.Con // RecommendSkillsInstall checks if coding agents are detected without Databricks tooling (skills // or the databricks plugin) installed. -// In interactive mode, prompts the user to install now using installFn. In non-interactive mode, prints a hint. +// In interactive mode, prompts the user to install now using installFn; a "no" also turns +// auto-install off. In non-interactive mode, prints a hint. func RecommendSkillsInstall(ctx context.Context, installFn func(context.Context) error) error { if HasDatabricksToolingInstalled(ctx) { return nil @@ -70,6 +72,11 @@ func RecommendSkillsInstall(ctx context.Context, installFn func(context.Context) return err } if !yes { + // The automatic install after this command (and after later upgrades) + // must not install what the user just turned down. + if err := autoinstall.Decline(ctx); err != nil { + log.Warnf(ctx, "Could not turn off the automatic installation of Databricks AI tools: %v", err) + } return nil } diff --git a/libs/aitools/agents/recommend_test.go b/libs/aitools/agents/recommend_test.go index aaf978bb9d1..be744da9f7d 100644 --- a/libs/aitools/agents/recommend_test.go +++ b/libs/aitools/agents/recommend_test.go @@ -8,6 +8,7 @@ import ( "testing" "github.com/databricks/cli/internal/testutil" + "github.com/databricks/cli/libs/aitools/autoinstall" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/dbr" "github.com/databricks/cli/libs/telemetry" @@ -133,6 +134,11 @@ func TestRecommendSkillsInstallInteractiveDecline(t *testing.T) { assert.NoError(t, <-errc) // The prompt is logged when shown, so a declined prompt still counts. assert.Equal(t, []protos.AitoolsHintKind{protos.AitoolsHintKindAppsInitPrompt}, hintKinds(t, ctx)) + // The "no" also keeps the automatic install from installing. + state, err := autoinstall.Load(ctx) + require.NoError(t, err) + assert.True(t, state.OptedOut) + assert.True(t, autoinstall.Declined()) } func TestMaybeVersionCheckTip(t *testing.T) { diff --git a/libs/aitools/autoinstall/decide.go b/libs/aitools/autoinstall/decide.go new file mode 100644 index 00000000000..cb359a60e60 --- /dev/null +++ b/libs/aitools/autoinstall/decide.go @@ -0,0 +1,139 @@ +package autoinstall + +import ( + "slices" + + "github.com/databricks/cli/internal/build" + "golang.org/x/mod/semver" +) + +const ( + // MaxAttempts is how many failed attempts one CLI version gets. A new + // version starts over. + MaxAttempts = 10 + + // HoldoutPercent is the share of machines (by Bucket) that never + // auto-install, so the install rate can be compared with a control group. + HoldoutPercent = 10 +) + +// Reason is what Decide concluded. The reasons that are reported are also the +// outcome values of the auto-install telemetry event. +type Reason string + +const ( + // Not due: the run records and reports nothing. + ReasonSkippedCommand Reason = "skipped_command" + ReasonCacheDisabled Reason = "cache_disabled" + ReasonDevelopmentBuild Reason = "development_build" + ReasonAlreadyRan Reason = "already_ran" + ReasonRetriesExhausted Reason = "retries_exhausted" + + // Settle the version without installing. All but ReasonGateOff are + // reported: the feature is not launched while the gate is off. + ReasonSkippedCI Reason = "skipped_ci" + ReasonSkippedOptOut Reason = "skipped_optout" + ReasonGateOff Reason = "gate_off" + ReasonSkippedHoldout Reason = "skipped_holdout" + + // Install, or update an existing installation. + ReasonRun Reason = "run" +) + +// Due reports whether the run acts on the decision: it installs, or it +// settles the version. A run that is not due changes nothing. +func (r Reason) Due() bool { + switch r { + case ReasonSkippedCommand, ReasonCacheDisabled, ReasonDevelopmentBuild, ReasonAlreadyRan, ReasonRetriesExhausted: + return false + case ReasonSkippedCI, ReasonSkippedOptOut, ReasonGateOff, ReasonSkippedHoldout, ReasonRun: + return true + } + return true +} + +// Inputs is everything Decide depends on. +type Inputs struct { + // Command is the command path below the root, e.g. ["auth", "token"]. + Command []string + + // CacheDisabled is DATABRICKS_CACHE_ENABLED=false. + CacheDisabled bool + + // Version is the running CLI version. + Version string + + // State is the persisted state: last version, attempts and opt-out. + State State + + // CI is the CI env var check of the version-check notice. + CI bool + + // EnvOptOut is DATABRICKS_AITOOLS_AUTO_INSTALL=false, or a "no" earlier in + // this process (see Declined). + EnvOptOut bool + + // GateOn is the remote gate for Version. + GateOn bool + + // Bucket is the machine's stable holdout bucket, in [0, 100). + Bucket int +} + +// skippedCommands never trigger auto-install anywhere in their path: help +// and shell completion output is read by machines or shown on every +// keystroke, `version` is how a user checks what was just installed, and the +// aitools subtree manages the tools explicitly. "__complete" and +// "__completeNoDesc" are cobra.ShellCompRequestCmd and +// cobra.ShellCompNoDescRequestCmd. +var skippedCommands = []string{"aitools", "help", "version", "completion", "__complete", "__completeNoDesc"} + +// Decide returns whether this run should install, settle the version, or do +// nothing. The checks run from cheapest to most expensive, so callers can +// leave GateOn false and look the gate up only when Decide returns +// ReasonGateOff. Every skip reason precedes the gate except the holdout: +// the holdout arm only exists once the feature is on. +func Decide(in Inputs) Reason { + switch { + case isSkippedCommand(in.Command): + return ReasonSkippedCommand + case in.CacheDisabled: + return ReasonCacheDisabled + // Development and snapshot builds change version with every commit, so + // they would trigger on every build. + case build.IsDevelopmentVersion(in.Version): + return ReasonDevelopmentBuild + case in.State.LastVersion == in.Version && in.State.Attempts >= MaxAttempts: + return ReasonRetriesExhausted + case in.State.LastVersion == in.Version && in.State.Attempts == 0: + return ReasonAlreadyRan + // A newer CLI already ran here, e.g. an IDE ships its own older CLI next + // to the user's: alternating between the two must not run on every + // command. + case in.State.LastVersion != "" && semver.Compare("v"+in.Version, "v"+in.State.LastVersion) < 0: + return ReasonAlreadyRan + case in.CI: + return ReasonSkippedCI + case in.EnvOptOut, in.State.OptedOut: + return ReasonSkippedOptOut + case !in.GateOn: + return ReasonGateOff + case InHoldout(in.Bucket): + return ReasonSkippedHoldout + } + return ReasonRun +} + +// InHoldout reports whether a machine in bucket belongs to the holdout. +func InHoldout(bucket int) bool { + return bucket < HoldoutPercent +} + +func isSkippedCommand(path []string) bool { + for _, name := range path { + if slices.Contains(skippedCommands, name) { + return true + } + } + return slices.Equal(path, []string{"auth", "token"}) +} diff --git a/libs/aitools/autoinstall/decide_test.go b/libs/aitools/autoinstall/decide_test.go new file mode 100644 index 00000000000..e19a9c7491e --- /dev/null +++ b/libs/aitools/autoinstall/decide_test.go @@ -0,0 +1,83 @@ +package autoinstall + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestDecide(t *testing.T) { + // eligible is a run of a released version never seen before, with the gate on, in the treatment arm. + eligible := Inputs{ + Command: []string{"jobs", "list"}, + Version: "1.30.0", + GateOn: true, + Bucket: HoldoutPercent, + } + tests := []struct { + name string + modify func(in *Inputs) + want Reason + }{ + {name: "eligible", modify: func(in *Inputs) {}, want: ReasonRun}, + + {name: "aitools subtree", modify: func(in *Inputs) { in.Command = []string{"aitools", "list"} }, want: ReasonSkippedCommand}, + {name: "experimental aitools", modify: func(in *Inputs) { in.Command = []string{"experimental", "aitools", "tools", "query"} }, want: ReasonSkippedCommand}, + {name: "help", modify: func(in *Inputs) { in.Command = []string{"help"} }, want: ReasonSkippedCommand}, + {name: "version", modify: func(in *Inputs) { in.Command = []string{"version"} }, want: ReasonSkippedCommand}, + {name: "completion", modify: func(in *Inputs) { in.Command = []string{"completion", "zsh"} }, want: ReasonSkippedCommand}, + {name: "shell completion request", modify: func(in *Inputs) { in.Command = []string{"__complete"} }, want: ReasonSkippedCommand}, + {name: "shell completion request without descriptions", modify: func(in *Inputs) { in.Command = []string{"__completeNoDesc"} }, want: ReasonSkippedCommand}, + {name: "auth token", modify: func(in *Inputs) { in.Command = []string{"auth", "token"} }, want: ReasonSkippedCommand}, + {name: "other auth command", modify: func(in *Inputs) { in.Command = []string{"auth", "describe"} }, want: ReasonRun}, + + {name: "cache disabled", modify: func(in *Inputs) { in.CacheDisabled = true }, want: ReasonCacheDisabled}, + {name: "development build", modify: func(in *Inputs) { in.Version = "1.31.0-dev+abcdef123456" }, want: ReasonDevelopmentBuild}, + {name: "snapshot build", modify: func(in *Inputs) { in.Version = "1.30.1-dev+abcdef1" }, want: ReasonDevelopmentBuild}, + + {name: "same version, settled", modify: func(in *Inputs) { in.State.LastVersion = "1.30.0" }, want: ReasonAlreadyRan}, + {name: "same version, failed before", modify: func(in *Inputs) { in.State = State{LastVersion: "1.30.0", Attempts: 3} }, want: ReasonRun}, + {name: "same version, last attempt left", modify: func(in *Inputs) { in.State = State{LastVersion: "1.30.0", Attempts: MaxAttempts - 1} }, want: ReasonRun}, + {name: "same version, retries exhausted", modify: func(in *Inputs) { in.State = State{LastVersion: "1.30.0", Attempts: MaxAttempts} }, want: ReasonRetriesExhausted}, + {name: "new version after retries exhausted", modify: func(in *Inputs) { in.State = State{LastVersion: "1.29.0", Attempts: MaxAttempts} }, want: ReasonRun}, + {name: "older CLI after a newer one ran", modify: func(in *Inputs) { in.State.LastVersion = "1.31.0" }, want: ReasonAlreadyRan}, + {name: "older CLI while a newer one retries", modify: func(in *Inputs) { in.State = State{LastVersion: "1.31.0", Attempts: 2} }, want: ReasonAlreadyRan}, + {name: "newer CLI after an older one ran", modify: func(in *Inputs) { in.State.LastVersion = "1.29.5" }, want: ReasonRun}, + + {name: "CI", modify: func(in *Inputs) { in.CI = true }, want: ReasonSkippedCI}, + {name: "CI before the opt-out", modify: func(in *Inputs) { in.CI = true; in.EnvOptOut = true }, want: ReasonSkippedCI}, + {name: "env opt-out", modify: func(in *Inputs) { in.EnvOptOut = true }, want: ReasonSkippedOptOut}, + {name: "persisted opt-out", modify: func(in *Inputs) { in.State.OptedOut = true }, want: ReasonSkippedOptOut}, + {name: "opt-out before the gate", modify: func(in *Inputs) { in.State.OptedOut = true; in.GateOn = false }, want: ReasonSkippedOptOut}, + {name: "gate off", modify: func(in *Inputs) { in.GateOn = false }, want: ReasonGateOff}, + {name: "gate off hides the holdout", modify: func(in *Inputs) { in.GateOn = false; in.Bucket = 0 }, want: ReasonGateOff}, + {name: "holdout", modify: func(in *Inputs) { in.Bucket = HoldoutPercent - 1 }, want: ReasonSkippedHoldout}, + {name: "first bucket is holdout", modify: func(in *Inputs) { in.Bucket = 0 }, want: ReasonSkippedHoldout}, + {name: "last bucket is treatment", modify: func(in *Inputs) { in.Bucket = 99 }, want: ReasonRun}, + + {name: "skipped command before everything", modify: func(in *Inputs) { + in.Command = []string{"version"} + in.CacheDisabled = true + in.CI = true + }, want: ReasonSkippedCommand}, + {name: "settled version before CI", modify: func(in *Inputs) { in.State.LastVersion = "1.30.0"; in.CI = true }, want: ReasonAlreadyRan}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + in := eligible + tc.modify(&in) + assert.Equal(t, tc.want, Decide(in)) + }) + } +} + +func TestReasonDue(t *testing.T) { + notDue := []Reason{ReasonSkippedCommand, ReasonCacheDisabled, ReasonDevelopmentBuild, ReasonAlreadyRan, ReasonRetriesExhausted} + due := []Reason{ReasonSkippedCI, ReasonSkippedOptOut, ReasonGateOff, ReasonSkippedHoldout, ReasonRun} + for _, r := range notDue { + assert.False(t, r.Due(), r) + } + for _, r := range due { + assert.True(t, r.Due(), r) + } +} diff --git a/libs/aitools/autoinstall/hook.go b/libs/aitools/autoinstall/hook.go new file mode 100644 index 00000000000..c573cc7bd42 --- /dev/null +++ b/libs/aitools/autoinstall/hook.go @@ -0,0 +1,10 @@ +package autoinstall + +import "github.com/spf13/cobra" + +// Run, when set, is called by the root command after a command succeeded. It +// runs auto-install, which needs the install and update code in cmd/aitools. +// cmd/root cannot import that package (cmd/aitools imports cmd/root), so +// main.go sets Run. Runs that don't start in main.go, such as in-process +// tests, leave it nil and never auto-install. +var Run func(cmd *cobra.Command) diff --git a/libs/aitools/autoinstall/lock_unix.go b/libs/aitools/autoinstall/lock_unix.go new file mode 100644 index 00000000000..c183be7d109 --- /dev/null +++ b/libs/aitools/autoinstall/lock_unix.go @@ -0,0 +1,21 @@ +//go:build unix + +package autoinstall + +import ( + "errors" + "os" + + "golang.org/x/sys/unix" +) + +// tryLockFile takes an exclusive lock on f, failing at once with errLockHeld +// when another process holds it. The lock is released when f is closed, or by +// the OS when the process exits. +func tryLockFile(f *os.File) error { + err := unix.Flock(int(f.Fd()), unix.LOCK_EX|unix.LOCK_NB) + if errors.Is(err, unix.EWOULDBLOCK) { + return errLockHeld + } + return err +} diff --git a/libs/aitools/autoinstall/lock_windows.go b/libs/aitools/autoinstall/lock_windows.go new file mode 100644 index 00000000000..b49b8e0eeff --- /dev/null +++ b/libs/aitools/autoinstall/lock_windows.go @@ -0,0 +1,22 @@ +//go:build windows + +package autoinstall + +import ( + "errors" + "os" + + "golang.org/x/sys/windows" +) + +// tryLockFile takes an exclusive lock on f, failing at once with errLockHeld +// when another process holds it. The lock is released when f is closed, or by +// the OS when the process exits. +func tryLockFile(f *os.File) error { + flags := uint32(windows.LOCKFILE_EXCLUSIVE_LOCK | windows.LOCKFILE_FAIL_IMMEDIATELY) + err := windows.LockFileEx(windows.Handle(f.Fd()), flags, 0, 1, 0, &windows.Overlapped{}) + if errors.Is(err, windows.ERROR_LOCK_VIOLATION) { + return errLockHeld + } + return err +} diff --git a/libs/aitools/autoinstall/run.go b/libs/aitools/autoinstall/run.go new file mode 100644 index 00000000000..cd6d323d2bf --- /dev/null +++ b/libs/aitools/autoinstall/run.go @@ -0,0 +1,155 @@ +package autoinstall + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "errors" + "fmt" + "os" + "path/filepath" + "time" + + "github.com/databricks/cli/internal/build" + "github.com/databricks/cli/libs/clicompat" + "github.com/databricks/cli/libs/env" + "github.com/databricks/cli/libs/log" +) + +const ( + // versionEnv sets the CLI version auto-install sees. Acceptance tests use + // it to simulate releases and upgrades: their binaries are development + // builds, which never auto-install. Not a documented, user-facing setting. + versionEnv = "DATABRICKS_AITOOLS_AUTO_INSTALL_CLI_VERSION" + + // cacheEnabledEnv mirrors libs/cache's env knob: with the cache turned + // off, auto-install does nothing. + cacheEnabledEnv = "DATABRICKS_CACHE_ENABLED" + + // lockFileName is the lock of a run, held while it installs. + lockFileName = "auto-install.lock" + + // stateLockFileName is the lock of the state file, held while it is read + // and rewritten. + stateLockFileName = "auto-install.json.lock" + + // stateLockWait bounds the wait for the state lock. A live holder keeps it + // only to read or rewrite a small file. + stateLockWait = time.Second + + // lockRetryInterval is how often a held lock is tried again. + lockRetryInterval = 10 * time.Millisecond +) + +// Version returns the CLI version auto-install runs for. +func Version(ctx context.Context) string { + if v := env.Get(ctx, versionEnv); v != "" { + return v + } + return build.GetInfo().Version +} + +// CacheDisabled reports whether DATABRICKS_CACHE_ENABLED turns the CLI's cache +// off. +func CacheDisabled(ctx context.Context) bool { + enabled, ok := env.GetBool(ctx, cacheEnabledEnv) + return ok && !enabled +} + +// GateOn reports whether the compatibility manifest turns auto-install on for +// version. A manifest that cannot be resolved reads as off. +func GateOn(ctx context.Context, version string) bool { + on, err := clicompat.ResolveAutoInstall(ctx, version) + if err != nil { + log.Debugf(ctx, "auto-install: could not resolve the gate: %v", err) + return false + } + return on +} + +// NewID returns a random machine identifier for State.ID. +func NewID() string { + b := make([]byte, 16) + _, _ = rand.Read(b) + return hex.EncodeToString(b) +} + +// Bucket maps a machine identifier to its stable holdout bucket in [0, 100). +func Bucket(id string) int { + sum := sha256.Sum256([]byte("aitools-auto-install:" + id)) + return int(binary.BigEndian.Uint64(sum[:8]) % 100) +} + +// Record stores the outcome of a run for version: attempts is 0 when the +// version is settled, or the number of failed attempts so far. It re-reads the +// state and changes only the fields a run owns, so an opt-out stored while the +// run was installing is kept. +func Record(ctx context.Context, version string, attempts int, id string) error { + return updateState(ctx, func(s *State) { + s.LastVersion = version + s.Attempts = attempts + if s.ID == "" { + s.ID = id + } + }) +} + +// TryLock claims the machine-wide auto-install lock so that concurrent CLI +// processes (agents often run several at once) do not install at the same +// time. ok is false when another process holds it or it cannot be taken; that +// run then skips auto-install. It is an OS file lock, so a process that dies +// mid-run releases it. +func TryLock(ctx context.Context) (unlock func(), ok bool) { + unlock, err := lock(ctx, lockFileName, 0) + if err != nil { + log.Debugf(ctx, "auto-install: could not take the lock: %v", err) + return nil, false + } + return unlock, true +} + +// lock takes the exclusive lock file name next to the state. It retries for +// up to wait, so a holder that is stopped (a suspended process, say) delays +// the caller by at most wait, then fails it. The file is never removed: a +// process that removed it could let a third one lock a new file while a +// second still holds the old. +func lock(ctx context.Context, name string, wait time.Duration) (unlock func(), err error) { + path, err := statePath(ctx) + if err != nil { + return nil, err + } + path = filepath.Join(filepath.Dir(path), name) + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return nil, err + } + f, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0o600) + if err != nil { + return nil, err + } + deadline := time.Now().Add(wait) + for { + err = tryLock(f) + if err == nil { + return func() { _ = f.Close() }, nil + } + // Only a lock another process holds can come free; any other error + // (a filesystem without locks, say) fails the same way on every try. + if !errors.Is(err, errLockHeld) { + _ = f.Close() + return nil, fmt.Errorf("lock %s: %w", filepath.ToSlash(path), err) + } + if time.Now().After(deadline) || ctx.Err() != nil { + _ = f.Close() + return nil, fmt.Errorf("%s: %w", filepath.ToSlash(path), err) + } + time.Sleep(lockRetryInterval) + } +} + +// errLockHeld is the error of tryLockFile when another process holds the lock. +var errLockHeld = errors.New("held by another CLI process") + +// tryLock is tryLockFile; tests replace it. +var tryLock = tryLockFile diff --git a/libs/aitools/autoinstall/run_test.go b/libs/aitools/autoinstall/run_test.go new file mode 100644 index 00000000000..98597b9234b --- /dev/null +++ b/libs/aitools/autoinstall/run_test.go @@ -0,0 +1,108 @@ +package autoinstall + +import ( + "errors" + "os" + "sync" + "testing" + "time" + + "github.com/databricks/cli/libs/env" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestTryLockExcludesOtherRuns(t *testing.T) { + ctx := env.WithUserHomeDir(t.Context(), t.TempDir()) + + unlock, ok := TryLock(ctx) + require.True(t, ok) + _, ok = TryLock(ctx) + assert.False(t, ok, "a second run must not take a held lock") + + unlock() + unlock, ok = TryLock(ctx) + require.True(t, ok, "a released lock can be taken again") + unlock() +} + +func TestRecordKeepsConcurrentOptOut(t *testing.T) { + ctx := env.WithUserHomeDir(t.Context(), t.TempDir()) + + // Without the state lock, a Record that read the state before the opt-out + // was saved would write opted_out=false back. + var wg sync.WaitGroup + for range 20 { + wg.Go(func() { assert.NoError(t, Record(ctx, "1.30.0", 1, "machine")) }) + } + wg.Go(func() { assert.NoError(t, SetOptedOut(ctx, true)) }) + wg.Wait() + + s, err := Load(ctx) + require.NoError(t, err) + assert.True(t, s.OptedOut) + assert.Equal(t, "1.30.0", s.LastVersion) +} + +func TestLoadWaitsForStateLock(t *testing.T) { + ctx := env.WithUserHomeDir(t.Context(), t.TempDir()) + + // A writer holds the state lock while it replaces the file, which fails on + // Windows while another process has the file open. + unlock, err := lock(ctx, stateLockFileName, 0) + require.NoError(t, err) + done := make(chan struct{}) + go func() { + _, err := Load(ctx) + assert.NoError(t, err) + close(done) + }() + select { + case <-done: + t.Fatal("Load read the state while the state lock was held") + case <-time.After(100 * time.Millisecond): + } + unlock() + <-done +} + +func TestLoadFailsWhenStateLockStaysHeld(t *testing.T) { + ctx := env.WithUserHomeDir(t.Context(), t.TempDir()) + + // A stopped holder must not block every later command. + unlock, err := lock(ctx, stateLockFileName, 0) + require.NoError(t, err) + defer unlock() + start := time.Now() + _, err = Load(ctx) + assert.ErrorIs(t, err, errLockHeld) + assert.Less(t, time.Since(start), stateLockWait+2*time.Second) +} + +func TestLockReturnsOtherErrorsAtOnce(t *testing.T) { + ctx := env.WithUserHomeDir(t.Context(), t.TempDir()) + + // A filesystem without locks fails every try, so it is not retried. + errNoLocks := errors.New("no locks here") + orig := tryLock + t.Cleanup(func() { tryLock = orig }) + tries := 0 + tryLock = func(*os.File) error { + tries++ + return errNoLocks + } + _, err := lock(ctx, stateLockFileName, stateLockWait) + assert.ErrorIs(t, err, errNoLocks) + assert.Equal(t, 1, tries) +} + +func TestChildEnvCarriesTheRefusal(t *testing.T) { + t.Cleanup(func() { declined.Store(false) }) + + assert.NotContains(t, ChildEnv(), EnvVar+"=false") + + // A CLI child started after a "no" must not install either, even when + // the opt-out could not be stored. + declined.Store(true) + assert.Equal(t, EnvVar+"=false", ChildEnv()[len(ChildEnv())-1]) +} diff --git a/libs/aitools/autoinstall/state.go b/libs/aitools/autoinstall/state.go index 5227b14ef53..f48c09d1e83 100644 --- a/libs/aitools/autoinstall/state.go +++ b/libs/aitools/autoinstall/state.go @@ -10,6 +10,7 @@ import ( "io/fs" "os" "path/filepath" + "sync/atomic" "github.com/databricks/cli/libs/atomicfile" "github.com/databricks/cli/libs/env" @@ -31,9 +32,15 @@ type State struct { Attempts int `json:"attempts,omitempty"` // OptedOut turns auto-install off for every later version. It is set by - // `databricks aitools auto-install disable` and by a global-scope - // `databricks aitools uninstall`. + // `databricks aitools auto-install disable`, by a global-scope + // `databricks aitools uninstall` and by a "no" to the skills prompt of + // `databricks apps init`. OptedOut bool `json:"opted_out,omitempty"` + + // ID is a random per-machine identifier, created on the first run that + // decides anything. Only its Bucket is used, and it never leaves the + // machine. The CLI has no other stable machine identifier. + ID string `json:"id,omitempty"` } // statePath returns ~/.databricks/aitools/auto-install.json. @@ -45,8 +52,20 @@ func statePath(ctx context.Context) (string, error) { return filepath.Join(home, ".databricks", "aitools", stateFileName), nil } -// Load reads the state. A missing file is the zero State. +// Load reads the state. A missing file is the zero State. It holds the state +// lock while it reads: on Windows a file another process has open cannot be +// replaced, so an unlocked read could make a concurrent write fail. func Load(ctx context.Context) (State, error) { + unlock, err := lock(ctx, stateLockFileName, stateLockWait) + if err != nil { + return State{}, err + } + defer unlock() + return load(ctx) +} + +// load reads the state; the caller holds the state lock. +func load(ctx context.Context) (State, error) { path, err := statePath(ctx) if err != nil { return State{}, err @@ -65,8 +84,8 @@ func Load(ctx context.Context) (State, error) { return s, nil } -// Save writes the state atomically. -func Save(ctx context.Context, s State) error { +// save writes the state atomically; the caller holds the state lock. +func save(ctx context.Context, s State) error { path, err := statePath(ctx) if err != nil { return err @@ -82,13 +101,63 @@ func Save(ctx context.Context, s State) error { // rest of the state. A state file that cannot be read is replaced: the user's // explicit choice matters more than a broken record. func SetOptedOut(ctx context.Context, optedOut bool) error { - s, err := Load(ctx) + unlock, err := lock(ctx, stateLockFileName, stateLockWait) + if err != nil { + return err + } + defer unlock() + s, err := load(ctx) if err != nil { log.Debugf(ctx, "Replacing unreadable auto-install state: %v", err) s = State{} } s.OptedOut = optedOut - return Save(ctx, s) + return save(ctx, s) +} + +// declined is set once the user turned the tools down in this process. +var declined atomic.Bool + +// Decline turns auto-install off after the user turned the tools down: for +// this process at once, and for later ones through the stored opt-out. The +// error is that of storing the opt-out; this process honors the answer either +// way. +func Decline(ctx context.Context) error { + declined.Store(true) + return SetOptedOut(ctx, true) +} + +// Declined reports whether Decline ran in this process. +func Declined() bool { + return declined.Load() +} + +// ChildEnv returns the environment for a CLI process this one starts: the +// current environment, with auto-install turned off when the user declined in +// this process, so the child honors the answer even if storing it failed. +func ChildEnv() []string { + environ := os.Environ() + if Declined() { + environ = append(environ, EnvVar+"=false") + } + return environ +} + +// updateState applies change to the stored state and saves it. Every read and +// change of the state goes through the state lock, so that concurrent CLI +// processes do not overwrite each other's changes. +func updateState(ctx context.Context, change func(*State)) error { + unlock, err := lock(ctx, stateLockFileName, stateLockWait) + if err != nil { + return err + } + defer unlock() + s, err := load(ctx) + if err != nil { + return err + } + change(&s) + return save(ctx, s) } // EnvVar turns auto-install off when set to false (or 0, no, off). diff --git a/libs/aitools/installer/plugin.go b/libs/aitools/installer/plugin.go index d623a75c496..99d4b8fb32f 100644 --- a/libs/aitools/installer/plugin.go +++ b/libs/aitools/installer/plugin.go @@ -81,7 +81,9 @@ func (e *BlockedError) Error() string { func runAgentCmd(ctx context.Context, timeout time.Duration, argv []string) (string, error) { cctx, cancel := context.WithTimeout(ctx, timeout) defer cancel() - out, err := process.Background(cctx, argv) + // A timed-out agent CLI is stopped with its descendants: one left running + // would hold the output pipes open and block past the timeout. + out, err := process.Background(cctx, argv, process.WithProcessGroup()) if err != nil && errors.Is(cctx.Err(), context.DeadlineExceeded) { return out, &agentCmdTimeoutError{timeout: timeout, err: err} } diff --git a/libs/aitools/installer/plugin_unix_test.go b/libs/aitools/installer/plugin_unix_test.go new file mode 100644 index 00000000000..b31e16aa328 --- /dev/null +++ b/libs/aitools/installer/plugin_unix_test.go @@ -0,0 +1,23 @@ +//go:build unix + +package installer + +import ( + "errors" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestRunAgentCmdTimeoutStopsDescendants(t *testing.T) { + // The agent CLI leaves a child holding its output pipes. Killing only the + // CLI would leave the command waiting for the child to exit. + start := time.Now() + _, err := runAgentCmd(t.Context(), 100*time.Millisecond, []string{"sh", "-c", "sleep 30 & sleep 30"}) + require.Error(t, err) + assert.Less(t, time.Since(start), 5*time.Second) + _, timedOut := errors.AsType[*agentCmdTimeoutError](err) + assert.True(t, timedOut) +} diff --git a/libs/aitools/installer/update.go b/libs/aitools/installer/update.go index 30f1d98ab48..903d0ad25c6 100644 --- a/libs/aitools/installer/update.go +++ b/libs/aitools/installer/update.go @@ -30,6 +30,12 @@ type UpdateOptions struct { Check bool // dry run: show what would change without downloading Skills []string // empty = all installed Scope string // ScopeGlobal or ScopeProject (default: global) + + // KeepModified keeps an installed skill at its version when its files are + // not exactly the ones the CLI recorded (the user edited them, or there is + // no record to check against), instead of replacing it. Automatic updates + // set it: they must never overwrite what the user changed. + KeepModified bool } // UpdateResult describes what UpdateSkills did (or would do in check mode). @@ -165,6 +171,12 @@ func UpdateSkills(ctx context.Context, src ManifestSource, targetAgents []*agent continue } + if wasInstalled && opts.KeepModified && !skillPrunable(ctx, baseDir, name, scope, cwd, state) { + log.Warnf(ctx, "Warning: keeping %q at %s: its installed files were modified.", name, oldVersion) + result.Unchanged = append(result.Unchanged, name) + continue + } + update := SkillUpdate{ Name: name, OldVersion: oldVersion, diff --git a/libs/aitools/installer/update_test.go b/libs/aitools/installer/update_test.go index dad9840ce0a..28d36ee9c38 100644 --- a/libs/aitools/installer/update_test.go +++ b/libs/aitools/installer/update_test.go @@ -127,6 +127,42 @@ func TestUpdateVersionDiffDetected(t *testing.T) { assert.Equal(t, "0.2.0", state.Skills["databricks-sql"]) } +func TestUpdateKeepModifiedKeepsEditedSkill(t *testing.T) { + tmp := setupTestHome(t) + ctx := cmdio.MockDiscard(t.Context()) + setupFetchMock(t) + t.Setenv("DATABRICKS_SKILLS_REF", testSkillsRef) + + src := &mockManifestSource{manifest: testManifest()} + agent := testAgent(tmp) + require.NoError(t, InstallSkillsForAgents(ctx, src, []*agents.Agent{agent}, InstallOptions{})) + + // The user edits databricks-sql; a new release updates both skills. + globalDir := filepath.Join(tmp, ".databricks", "aitools", "skills") + edited := filepath.Join(globalDir, "databricks-sql", "SKILL.md") + require.NoError(t, os.WriteFile(edited, []byte("user edit"), 0o644)) + t.Setenv("DATABRICKS_SKILLS_REF", "v0.2.0") + updatedManifest := testManifest() + for name, meta := range updatedManifest.Skills { + meta.Version = "0.2.0" + updatedManifest.Skills[name] = meta + } + + result, err := UpdateSkills(ctx, &mockManifestSource{manifest: updatedManifest}, []*agents.Agent{agent}, UpdateOptions{KeepModified: true}) + require.NoError(t, err) + + require.Len(t, result.Updated, 1) + assert.Equal(t, "databricks-jobs", result.Updated[0].Name) + assert.Contains(t, result.Unchanged, "databricks-sql") + data, err := os.ReadFile(edited) + require.NoError(t, err) + assert.Equal(t, "user edit", string(data)) + state, err := LoadState(globalDir) + require.NoError(t, err) + assert.Equal(t, "0.1.0", state.Skills["databricks-sql"]) + assert.Equal(t, "0.2.0", state.Skills["databricks-jobs"]) +} + func TestUpdateCheckDryRun(t *testing.T) { tmp := setupTestHome(t) ctx := cmdio.MockDiscard(t.Context()) diff --git a/libs/telemetry/protos/README.md b/libs/telemetry/protos/README.md index aacdcca2e50..5da4cc71908 100644 --- a/libs/telemetry/protos/README.md +++ b/libs/telemetry/protos/README.md @@ -28,3 +28,15 @@ wait for it. Neither does a run of `install`, `update` or `uninstall` that ends confirmation step (a flag error, a cancelled prompt, no agent detected, a `--path` dump), nor an `update` or `uninstall` that finds nothing installed. A run that does not wait still uploads, within the upload timeout, when auth has already resolved. + +On the first run of a new CLI version, the CLI can install Databricks AI tools for the coding agents +on the machine (see `databricks aitools auto-install --help`). It logs an `aitools_auto_install_event` +with the outcome (`INSTALLED`, `SKIPPED_CI`, `SKIPPED_OPTOUT`, `SKIPPED_HOLDOUT`, `SKIPPED_NO_AGENTS`, +`FAILED`, `RETRIES_EXHAUSTED`), the machine's holdout arm and, for a failure, its error +category: once per version and machine, plus once per retry of a failed attempt. Nothing is logged +for a version whose outcome is settled, or while the feature is turned off remotely (the CI and +opt-out checks come first and are logged either way). An install with no tools installed yet also +logs an `aitools_install_event` with `trigger` `AUTO_FIRST_RUN`; installs by `databricks aitools +install` leave `trigger` unset. The holdout arm comes from a random machine id kept in +`~/.databricks/aitools/auto-install.json`; the id itself is never sent. Like the other aitools +events, these upload at exit only when the command resolved auth. diff --git a/libs/telemetry/protos/aitools_events.go b/libs/telemetry/protos/aitools_events.go index 4084430eff1..7095064d049 100644 --- a/libs/telemetry/protos/aitools_events.go +++ b/libs/telemetry/protos/aitools_events.go @@ -160,3 +160,59 @@ type AitoolsCommandEvent struct { // omitted when none or both scopes are selected. Scope AitoolsInstallScope `json:"scope,omitempty"` } + +// AitoolsAutoInstallOutcome mirrors AitoolsAutoInstallOutcome.Type: how the +// automatic installation of Databricks AI tools on the first run of a new CLI +// version ended. +type AitoolsAutoInstallOutcome string + +const ( + AitoolsAutoInstallOutcomeUnspecified AitoolsAutoInstallOutcome = "TYPE_UNSPECIFIED" + // The tools were installed, or an existing installation was updated. + AitoolsAutoInstallOutcomeInstalled AitoolsAutoInstallOutcome = "INSTALLED" + // The run was in CI (the CI env var is set). + AitoolsAutoInstallOutcomeSkippedCI AitoolsAutoInstallOutcome = "SKIPPED_CI" + // The user turned auto-install off: DATABRICKS_AITOOLS_AUTO_INSTALL=false, + // `aitools auto-install disable`, a global `aitools uninstall`, or "no" + // to the skills prompt of `apps init`. + AitoolsAutoInstallOutcomeSkippedOptOut AitoolsAutoInstallOutcome = "SKIPPED_OPTOUT" + // The machine is in the holdout cohort. + AitoolsAutoInstallOutcomeSkippedHoldout AitoolsAutoInstallOutcome = "SKIPPED_HOLDOUT" + // No supported coding agent has a config directory on the machine. + AitoolsAutoInstallOutcomeSkippedNoAgents AitoolsAutoInstallOutcome = "SKIPPED_NO_AGENTS" + // The attempt failed or timed out; a later run of the same version retries. + AitoolsAutoInstallOutcomeFailed AitoolsAutoInstallOutcome = "FAILED" + // The attempt failed and was the last one this CLI version gets. + AitoolsAutoInstallOutcomeRetriesExhausted AitoolsAutoInstallOutcome = "RETRIES_EXHAUSTED" +) + +// AitoolsHoldoutArm mirrors AitoolsHoldoutArm.Type: the auto-install +// experiment arm of the machine, from a stable per-machine bucket. +type AitoolsHoldoutArm string + +const ( + AitoolsHoldoutArmUnspecified AitoolsHoldoutArm = "TYPE_UNSPECIFIED" + // The machine never auto-installs (control group). + AitoolsHoldoutArmHoldout AitoolsHoldoutArm = "HOLDOUT" + // The machine auto-installs. + AitoolsHoldoutArmTreatment AitoolsHoldoutArm = "TREATMENT" +) + +// AitoolsAutoInstallEvent is emitted when the CLI decides about installing +// Databricks AI tools on the first run of a new version: once per version and +// machine, plus once per retry after a failed attempt. A version whose outcome +// is settled emits nothing, and so does a run that the remote gate turns off; +// the CI and opt-out checks come before the gate, so they are emitted either +// way. An install with no tools installed yet also emits an +// AitoolsInstallEvent with Trigger AUTO_FIRST_RUN. +type AitoolsAutoInstallEvent struct { + Outcome AitoolsAutoInstallOutcome `json:"outcome"` + HoldoutArm AitoolsHoldoutArm `json:"holdout_arm"` + + // Whether the run could prompt (cmdio.IsPromptSupported and no coding + // agent detected). A measurable false, so no omitempty. + Interactive bool `json:"interactive"` + + // Set for FAILED and RETRIES_EXHAUSTED. + ErrorCategory AitoolsErrorCategory `json:"error_category,omitempty"` +} diff --git a/libs/telemetry/protos/aitools_install.go b/libs/telemetry/protos/aitools_install.go index 406dde89a58..051076f6241 100644 --- a/libs/telemetry/protos/aitools_install.go +++ b/libs/telemetry/protos/aitools_install.go @@ -84,8 +84,21 @@ type AitoolsAgentResult struct { ErrorCategory AitoolsErrorCategory `json:"error_category"` } +// AitoolsInstallTrigger mirrors AitoolsInstallTrigger.Type in the +// databricks_cli lumberjack proto: what started an install other than the +// `databricks aitools install` command. The command's own installs leave it +// unset. +type AitoolsInstallTrigger string + +const ( + AitoolsInstallTriggerUnspecified AitoolsInstallTrigger = "TYPE_UNSPECIFIED" + // The install the CLI runs on its own on the first run of a new version. + AitoolsInstallTriggerAutoFirstRun AitoolsInstallTrigger = "AUTO_FIRST_RUN" +) + // AitoolsInstallEvent is emitted on every execution of the `databricks aitools -// install` command. +// install` command, and for the install the CLI runs on the first run of a new +// version (Trigger AUTO_FIRST_RUN). type AitoolsInstallEvent struct { // Agents the user targeted, deduplicated and sorted for stable output. Agents []AitoolsAgentType `json:"agents,omitempty"` @@ -105,4 +118,7 @@ type AitoolsInstallEvent struct { // AgentResults records the per-agent failure/skip categories, one entry per // non-successful agent. Empty when every targeted agent succeeded. AgentResults []AitoolsAgentResult `json:"agent_results,omitempty"` + + // Trigger is set when the install did not come from the command. + Trigger AitoolsInstallTrigger `json:"trigger,omitempty"` } diff --git a/libs/telemetry/protos/frontend_log.go b/libs/telemetry/protos/frontend_log.go index f3c37736cc1..8a7e5bbdb11 100644 --- a/libs/telemetry/protos/frontend_log.go +++ b/libs/telemetry/protos/frontend_log.go @@ -28,4 +28,5 @@ type DatabricksCliLog struct { AirRunEvent *AirRunEvent `json:"air_run_event,omitempty"` AitoolsHintEvent *AitoolsHintEvent `json:"aitools_hint_event,omitempty"` AitoolsCommandEvent *AitoolsCommandEvent `json:"aitools_command_event,omitempty"` + AitoolsAutoInstallEvent *AitoolsAutoInstallEvent `json:"aitools_auto_install_event,omitempty"` } diff --git a/libs/versioncheck/notice.go b/libs/versioncheck/notice.go index dc3a9d5c864..14dc0a62b41 100644 --- a/libs/versioncheck/notice.go +++ b/libs/versioncheck/notice.go @@ -243,7 +243,7 @@ func notifyEnabled(ctx context.Context, cmd *cobra.Command) bool { func gatherConditions(ctx context.Context, cmd *cobra.Command) notifyConditions { optedOut, _ := env.GetBool(ctx, disableEnv) - ci, _ := env.GetBool(ctx, ciEnv) + ci := InCI(ctx) cacheDisabled := false if enabled, ok := env.GetBool(ctx, cacheEnabledEnv); ok && !enabled { cacheDisabled = true @@ -267,6 +267,12 @@ func gatherConditions(ctx context.Context, cmd *cobra.Command) notifyConditions } } +// InCI reports whether the CLI runs in continuous integration. +func InCI(ctx context.Context) bool { + ci, _ := env.GetBool(ctx, ciEnv) + return ci +} + func isJSONOutput(cmd *cobra.Command) bool { f := cmd.Flag("output") return f != nil && f.Value.String() == "json" diff --git a/main.go b/main.go index 10c3686a223..0b4e8e48a0b 100644 --- a/main.go +++ b/main.go @@ -6,6 +6,8 @@ import ( "path/filepath" "github.com/databricks/cli/cmd" + "github.com/databricks/cli/cmd/aitools" + "github.com/databricks/cli/libs/aitools/autoinstall" // Registers a disk-cached HostMetadataResolver factory on the SDK so every // *config.Config the CLI constructs reuses the cached /.well-known lookup. @@ -23,6 +25,11 @@ func main() { os.Setenv("DATABRICKS_CLI_PATH", arg0) } + // cmd/root runs auto-install after a command but cannot import cmd/aitools, + // which imports it. It is wired here, not in a package init, for the same + // reason as DATABRICKS_CLI_PATH above. + autoinstall.Run = aitools.AutoInstall + ctx := context.Background() err := cmd.New(ctx).Execute(ctx, os.Args[1:]) if err != nil { From 12bfe17dc98ab2865558eabbdc8bd77ec64e4e3b Mon Sep 17 00:00:00 2001 From: Anton Nekipelov <226657+anton-107@users.noreply.github.com> Date: Mon, 16 Nov 2026 19:25:01 +0000 Subject: [PATCH 3/7] aitools: ask before auto-installing AI tools in a terminal When a person runs the CLI in a terminal, auto-install asks before its first install of a version: "Install Databricks AI tools for ? [Y/n]" (1 minute). - The question is asked only when stdin, stdout and stderr are all terminals and no coding agent is detected. If only some streams are terminals, nothing is installed and the version is not settled, so a later run asks. Runs with no terminal keep the non-interactive install. - Enter/y/yes installs; any other answer stores the opt-out and reports DECLINED; no answer stops without opting out. - Only a fresh install asks; updates and retries after an accepted, failed attempt do not. A planning failure on the first attempt settles the version and reports FAILED. - The question is a plain-text line (cmdio.AskCancellable), so NO_COLOR, TERM=dumb and Git Bash still get it. New cmdio.IsLinePromptSupported and cmdio.HasTTY express the stream checks. Co-authored-by: Isaac --- cmd/aitools/autoinstall.go | 6 +- cmd/aitools/autoinstall_run.go | 95 ++++++++- cmd/aitools/autoinstall_run_test.go | 251 ++++++++++++++++++++++++ libs/aitools/autoinstall/state.go | 4 +- libs/cmdio/capabilities.go | 15 ++ libs/cmdio/capabilities_test.go | 37 ++++ libs/cmdio/io.go | 15 ++ libs/cmdio/testing.go | 11 ++ libs/telemetry/protos/README.md | 2 +- libs/telemetry/protos/aitools_events.go | 8 +- 10 files changed, 433 insertions(+), 11 deletions(-) create mode 100644 cmd/aitools/autoinstall_run_test.go diff --git a/cmd/aitools/autoinstall.go b/cmd/aitools/autoinstall.go index e650ec1db8e..eef5a1924a5 100644 --- a/cmd/aitools/autoinstall.go +++ b/cmd/aitools/autoinstall.go @@ -19,11 +19,11 @@ On the first run of a new CLI version, the CLI can install the Databricks AI tools (skills and plugins) for the coding agents already on this machine, as 'databricks aitools install --scope=global' does, or update an existing installation, as 'databricks aitools update' does. It never runs in CI and only -touches the global scope. +touches the global scope. In a terminal it asks before the first install. It stays off after 'databricks aitools auto-install disable', a global-scope -'databricks aitools uninstall', or a "no" to the skills prompt of -'databricks apps init', until 'databricks aitools auto-install enable'. +'databricks aitools uninstall', a "no" at that prompt, or a "no" to the skills +prompt of 'databricks apps init', until 'databricks aitools auto-install enable'. Setting ` + autoinstall.EnvVar + `=false turns it off in that environment.`, } cmd.AddCommand(newAutoInstallToggleCmd(true)) diff --git a/cmd/aitools/autoinstall_run.go b/cmd/aitools/autoinstall_run.go index 524e8457e10..8af0041332f 100644 --- a/cmd/aitools/autoinstall_run.go +++ b/cmd/aitools/autoinstall_run.go @@ -21,6 +21,7 @@ import ( "github.com/databricks/cli/libs/telemetry" "github.com/databricks/cli/libs/telemetry/protos" "github.com/databricks/cli/libs/versioncheck" + "github.com/databricks/databricks-sdk-go/useragent" "github.com/spf13/cobra" ) @@ -34,6 +35,9 @@ const ( // agent CLIs it runs). One that runs out of time counts as a failed // attempt and is retried on a later run. installTimeout = 10 * time.Second + + // promptTimeout bounds the wait for an answer at the install prompt. + promptTimeout = time.Minute ) // autoInstallDisable is the opt-out command named in the notice. @@ -42,8 +46,9 @@ const autoInstallDisable = "databricks aitools auto-install disable" // AutoInstall installs Databricks AI tools for the coding agents on this // machine, or updates an existing installation, on the first run of a new CLI // version. The root command runs it after a command succeeded. It never fails -// the command and never writes to stdout: it prints at most one line to -// stderr, and why it skipped or failed only to the debug log. +// the command and never writes to stdout. On stderr it prints the install +// prompt (in a terminal) and one line on what it did; why it skipped or failed +// goes only to the debug log. func AutoInstall(cmd *cobra.Command) { ctx := cmd.Context() defer func() { @@ -120,7 +125,7 @@ func autoInstall(ctx context.Context, path []string) { version: version, id: id, arm: holdoutArm(in.Bucket), - interactive: installPromptsSupported(ctx), + interactive: autoInstallAsks(ctx), } if !reason.Due() { // Another process settled the version while this one waited. @@ -151,6 +156,29 @@ func (r autoInstallRun) attempt(ctx context.Context, prior autoinstall.State) { } qctx := quietContext(ctx) tools, err := planTools(qctx) + // A terminal user is asked before the first install of a version, once: + // a retry was answered "yes" (or ran non-interactively), so it is not + // asked again. + if r.interactive && attempt == 1 { + switch { + case err != nil: + // A retry would install without asking, so this version settles. + log.Debugf(ctx, "auto-install: failed before asking: %v", err) + r.settleFailed(ctx, err) + return + case tools.existing == nil && len(tools.plan) > 0: + if !cmdio.IsLinePromptSupported(ctx) { + // The question could not be seen or answered safely, and + // installing without it would skip the consent. A later run + // in a terminal asks. + log.Debugf(ctx, "auto-install: skipped: cannot ask, stdin, stdout or stderr is not a terminal") + return + } + if !r.ask(ctx, tools.plan) { + return + } + } + } // Count the attempt before making it: an attempt that never finishes (the // process is killed) still counts, and a state that cannot be saved stops // auto-install instead of retrying it on every run. @@ -193,6 +221,59 @@ func (r autoInstallRun) attempt(ctx context.Context, prior autoinstall.State) { } } +// autoInstallAsks reports whether a person may be at the terminal, so +// auto-install asks before installing: stdin, stdout or stderr is a terminal +// and no coding agent is detected (see installPromptsSupported). It asks only +// when all three are terminals and skips otherwise; the install without +// asking is left to runs with no terminal at all. The question is a +// plain-text line, so NO_COLOR, TERM=dumb and Git Bash, which turn off the TUI +// prompts of install, do not skip it. +func autoInstallAsks(ctx context.Context) bool { + return cmdio.HasTTY(ctx) && useragent.AgentProvider() == "" +} + +// ask asks whether to install for the agents in plan and reports whether to go +// ahead. The version is settled before asking, so the question is asked once +// per version even when it is never answered: the process is killed, stdin +// is closed, or nobody answers within promptTimeout (an unrecognized agent in +// a pseudo-terminal). Only "no" stores the opt-out. +func (r autoInstallRun) ask(ctx context.Context, plan []agentPlanItem) bool { + if err := autoinstall.Record(ctx, r.version, 0, r.id); err != nil { + log.Debugf(ctx, "auto-install: skipped: could not save the state: %v", err) + return false + } + names := make([]string, len(plan)) + for i, it := range plan { + names[i] = it.agent.DisplayName + } + actx, cancel := context.WithTimeout(ctx, promptTimeout) + defer cancel() + ans, err := cmdio.AskCancellable(actx, "Install Databricks AI tools for "+strings.Join(names, ", ")+"? [Y/n]", "") + if err != nil { + log.Debugf(ctx, "auto-install: skipped: the prompt was not answered: %v", err) + if errors.Is(err, context.DeadlineExceeded) { + cmdio.LogString(ctx, "") + } + return false + } + switch strings.ToLower(strings.TrimSpace(ans)) { + case "", "y", "yes": + return true + } + r.decline(ctx) + return false +} + +// decline stores a "no" at the prompt as the opt-out, so no later version +// asks again. +func (r autoInstallRun) decline(ctx context.Context) { + if err := autoinstall.SetOptedOut(ctx, true); err != nil { + log.Warnf(ctx, "Could not save the answer, so a later CLI version asks again: %v", err) + } + r.settle(ctx, protos.AitoolsAutoInstallOutcomeDeclined) + cmdio.LogString(ctx, "Skipped. You won't be asked again; to install Databricks AI tools later, run: databricks aitools install") +} + // skipOutcomes are the reported outcomes of the skips that settle a version. var skipOutcomes = map[autoinstall.Reason]protos.AitoolsAutoInstallOutcome{ autoinstall.ReasonSkippedCI: protos.AitoolsAutoInstallOutcomeSkippedCI, @@ -208,6 +289,14 @@ type autoInstallRun struct { interactive bool } +// settleFailed records the version as done and reports a failed attempt. +func (r autoInstallRun) settleFailed(ctx context.Context, err error) { + if recErr := autoinstall.Record(ctx, r.version, 0, r.id); recErr != nil { + log.Debugf(ctx, "auto-install: could not save the state: %v", recErr) + } + r.report(ctx, protos.AitoolsAutoInstallOutcomeFailed, classifyInstallError(err)) +} + // settle records the version as done and reports outcome. func (r autoInstallRun) settle(ctx context.Context, outcome protos.AitoolsAutoInstallOutcome) { log.Debugf(ctx, "auto-install: %s", strings.ToLower(string(outcome))) diff --git a/cmd/aitools/autoinstall_run_test.go b/cmd/aitools/autoinstall_run_test.go new file mode 100644 index 00000000000..ddecd1449dd --- /dev/null +++ b/cmd/aitools/autoinstall_run_test.go @@ -0,0 +1,251 @@ +package aitools + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/databricks/cli/libs/aitools/autoinstall" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/flags" + "github.com/databricks/cli/libs/telemetry" + "github.com/databricks/cli/libs/telemetry/protos" + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const autoInstallTestManifest = `{ + "version": "2", + "updated_at": "2026-01-01T00:00:00Z", + "skills": { + "databricks-test": {"version": "1.0.0", "description": "Test skill", "files": ["SKILL.md"], "repo_dir": "skills"} + } +}` + +// The interactive prompt needs a terminal, which acceptance tests do not +// have, so these cases drive the hook the root command runs with stdin as a +// test terminal; the rest of the terminal is detected as in a real run. Like the acceptance tests in acceptance/cmd/aitools/auto-install, +// they check only what the user and the backend see: the prompt and notice +// on stderr, the skills in Cursor's directory, the stored state and the +// telemetry event. +func TestAutoInstallPrompt(t *testing.T) { + tests := []struct { + name string + state string + input string + // installState replaces the global install state file. + installState string + // env is set on top of a terminal with color. + env map[string]string + // stdinPiped, stdoutRedirected and stderrRedirected make that + // stream not a terminal. + stdinPiped bool + stdoutRedirected bool + stderrRedirected bool + + wantPrompt bool + wantStderr []string + wantSkills bool + wantState autoinstall.State + wantOutcome protos.AitoolsAutoInstallOutcome + }{ + { + name: "enter accepts", + wantPrompt: true, + input: "\n", + wantStderr: []string{"Installed Databricks AI tools for Cursor. Disable: databricks aitools auto-install disable"}, + wantSkills: true, + wantState: autoinstall.State{LastVersion: "1.30.0", ID: "treatment-machine"}, + wantOutcome: protos.AitoolsAutoInstallOutcomeInstalled, + }, + { + name: "yes accepts", + wantPrompt: true, + input: "Y\n", + wantStderr: []string{"Installed Databricks AI tools for Cursor."}, + wantSkills: true, + wantState: autoinstall.State{LastVersion: "1.30.0", ID: "treatment-machine"}, + wantOutcome: protos.AitoolsAutoInstallOutcomeInstalled, + }, + { + name: "no declines and opts out", + wantPrompt: true, + input: "n\n", + wantStderr: []string{"Skipped. You won't be asked again; to install Databricks AI tools later, run: databricks aitools install"}, + wantState: autoinstall.State{LastVersion: "1.30.0", OptedOut: true, ID: "treatment-machine"}, + wantOutcome: protos.AitoolsAutoInstallOutcomeDeclined, + }, + { + name: "no answer settles the version without opting out", + input: "", + wantPrompt: true, + wantState: autoinstall.State{LastVersion: "1.30.0", ID: "treatment-machine"}, + }, + { + name: "a failure before asking settles the version", + installState: "not json", + wantState: autoinstall.State{LastVersion: "1.30.0", ID: "treatment-machine"}, + wantOutcome: protos.AitoolsAutoInstallOutcomeFailed, + }, + { + name: "NO_COLOR still asks", + env: map[string]string{"NO_COLOR": "1"}, + wantPrompt: true, + input: "\n", + wantStderr: []string{"Installed Databricks AI tools for Cursor."}, + wantSkills: true, + wantState: autoinstall.State{LastVersion: "1.30.0", ID: "treatment-machine"}, + wantOutcome: protos.AitoolsAutoInstallOutcomeInstalled, + }, + { + name: "TERM=dumb still asks", + env: map[string]string{"TERM": "dumb"}, + wantPrompt: true, + input: "n\n", + wantState: autoinstall.State{LastVersion: "1.30.0", OptedOut: true, ID: "treatment-machine"}, + wantOutcome: protos.AitoolsAutoInstallOutcomeDeclined, + }, + { + name: "redirected stderr skips without installing", + stderrRedirected: true, + input: "\n", + wantState: autoinstall.State{ID: "treatment-machine"}, + }, + { + name: "piped stdin skips without installing", + stdinPiped: true, + input: "\n", + wantState: autoinstall.State{ID: "treatment-machine"}, + }, + { + name: "stdout to a pager skips without installing", + stdoutRedirected: true, + input: "\n", + wantState: autoinstall.State{ID: "treatment-machine"}, + }, + { + name: "no terminal at all installs without asking", + stdinPiped: true, + stdoutRedirected: true, + stderrRedirected: true, + wantStderr: []string{"Installed Databricks AI tools for Cursor."}, + wantSkills: true, + wantState: autoinstall.State{LastVersion: "1.30.0", ID: "treatment-machine"}, + wantOutcome: protos.AitoolsAutoInstallOutcomeInstalled, + }, + { + name: "a retry is not asked again", + state: `{"id": "treatment-machine", "last_version": "1.30.0", "attempts": 1}`, + input: "", + wantStderr: []string{"Installed Databricks AI tools for Cursor."}, + wantSkills: true, + wantState: autoinstall.State{LastVersion: "1.30.0", ID: "treatment-machine"}, + wantOutcome: protos.AitoolsAutoInstallOutcomeInstalled, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + home := setupAutoInstallHome(t, tc.state) + if tc.installState != "" { + dir := filepath.Join(home, ".databricks", "aitools", "skills") + require.NoError(t, os.MkdirAll(dir, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, ".state.json"), []byte(tc.installState), 0o600)) + } + t.Setenv("TERM", "xterm") + for k, v := range tc.env { + t.Setenv(k, v) + } + var stdout, stderr bytes.Buffer + outW, errW := cmdio.FakeTTY(&stdout), cmdio.FakeTTY(&stderr) + if tc.stdoutRedirected { + outW = &stdout + } + if tc.stderrRedirected { + errW = &stderr + } + in := strings.NewReader(tc.input) + cio := cmdio.NewTestIOWithInputTTY(t.Context(), in, outW, errW) + if tc.stdinPiped { + cio = cmdio.NewIO(t.Context(), flags.OutputText, in, outW, errW, "", "") + } + ctx := cmdio.InContext(t.Context(), cio) + ctx = telemetry.WithNewLogger(ctx) + + root := &cobra.Command{Use: "databricks"} + jobs := &cobra.Command{Use: "jobs"} + list := &cobra.Command{Use: "list"} + root.AddCommand(jobs) + jobs.AddCommand(list) + list.SetContext(ctx) + AutoInstall(list) + + assert.Empty(t, stdout.String()) + assert.Equal(t, tc.wantPrompt, strings.Contains(stderr.String(), "Install Databricks AI tools for Cursor? [Y/n]: "), "prompted") + for _, want := range tc.wantStderr { + assert.Contains(t, stderr.String(), want) + } + + _, err := os.Stat(filepath.Join(home, ".cursor", "skills", "databricks-test", "SKILL.md")) + assert.Equal(t, tc.wantSkills, err == nil, "Cursor has the skill") + + state, err := autoinstall.Load(ctx) + require.NoError(t, err) + assert.Equal(t, tc.wantState, state) + + var outcomes []protos.AitoolsAutoInstallOutcome + for _, e := range telemetry.BufferedEvents(ctx) { + if e := e.AitoolsAutoInstallEvent; e != nil { + outcomes = append(outcomes, e.Outcome) + assert.Equal(t, !tc.stdinPiped || !tc.stdoutRedirected || !tc.stderrRedirected, e.Interactive) + assert.Equal(t, protos.AitoolsHoldoutArmTreatment, e.HoldoutArm) + } + } + if tc.wantOutcome == "" { + assert.Empty(t, outcomes) + } else { + assert.Equal(t, []protos.AitoolsAutoInstallOutcome{tc.wantOutcome}, outcomes) + } + }) + } +} + +// setupAutoInstallHome gives the run a clean environment with Cursor's config +// dir, a skills server, the gate on and a released CLI version. state seeds +// the auto-install state (a treatment-arm machine id by default). +func setupAutoInstallHome(t *testing.T, state string) string { + t.Helper() + home := t.TempDir() + cacheDir := t.TempDir() + url := serveSkills(t, autoInstallTestManifest, map[string]string{ + "/test-ref/skills/databricks-test/SKILL.md": "---\nname: databricks-test\n---\n", + }) + kv := []string{ + "HOME", home, + "USERPROFILE", home, + "XDG_CONFIG_HOME", filepath.Join(home, ".config"), + "APPDATA", filepath.Join(home, "AppData", "Roaming"), + "DATABRICKS_CACHE_DIR", cacheDir, + "DATABRICKS_SKILLS_REF", "test-ref", + "DATABRICKS_SKILLS_BASE_URL", url, + "DATABRICKS_AITOOLS_AUTO_INSTALL_CLI_VERSION", "1.30.0", + } + // Windows sockets need SystemRoot. + if v, ok := os.LookupEnv("SystemRoot"); ok { + kv = append(kv, "SystemRoot", v) + } + isolateAgents(t, kv...) + + require.NoError(t, os.MkdirAll(filepath.Join(home, ".cursor"), 0o755)) + if state == "" { + state = `{"id": "treatment-machine"}` + } + stateDir := filepath.Join(home, ".databricks", "aitools") + require.NoError(t, os.MkdirAll(stateDir, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(stateDir, "auto-install.json"), []byte(state), 0o600)) + gate := `{"0.0.0": {"appkit": "0.84.0", "skills": "0.2.30", "auto_install": true}}` + require.NoError(t, os.WriteFile(filepath.Join(cacheDir, "compat-manifest.json"), []byte(gate), 0o600)) + return home +} diff --git a/libs/aitools/autoinstall/state.go b/libs/aitools/autoinstall/state.go index f48c09d1e83..b1e75dd3665 100644 --- a/libs/aitools/autoinstall/state.go +++ b/libs/aitools/autoinstall/state.go @@ -33,8 +33,8 @@ type State struct { // OptedOut turns auto-install off for every later version. It is set by // `databricks aitools auto-install disable`, by a global-scope - // `databricks aitools uninstall` and by a "no" to the skills prompt of - // `databricks apps init`. + // `databricks aitools uninstall`, by declining the install prompt and by a + // "no" to the skills prompt of `databricks apps init`. OptedOut bool `json:"opted_out,omitempty"` // ID is a random per-machine identifier, created on the first run that diff --git a/libs/cmdio/capabilities.go b/libs/cmdio/capabilities.go index 47017b53acc..9bcf3110777 100644 --- a/libs/cmdio/capabilities.go +++ b/libs/cmdio/capabilities.go @@ -42,6 +42,21 @@ func (c Capabilities) SupportsPrompt() bool { return c.SupportsInteractive() && c.stdinIsTTY && !c.isGitBash } +// SupportsLinePrompt returns true if the terminal supports a plain-text line +// prompt (see [Ask]): it writes the question to stderr and reads the answer +// from stdin. stdout must be a terminal too: when it is piped to a pager, the +// pager reads the same terminal, so its keystrokes could answer the prompt. +// Unlike SupportsPrompt it does not depend on color or Git Bash, which only +// matter for the TUI prompts. +func (c Capabilities) SupportsLinePrompt() bool { + return c.stdinIsTTY && c.stdoutIsTTY && c.stderrIsTTY +} + +// HasTTY returns true if stdin, stdout or stderr is a terminal. +func (c Capabilities) HasTTY() bool { + return c.stdinIsTTY || c.stdoutIsTTY || c.stderrIsTTY +} + // SupportsColor returns true if the given writer supports colored output. // This checks both TTY status and environment variables (NO_COLOR, TERM=dumb). func (c Capabilities) SupportsColor(w io.Writer) bool { diff --git a/libs/cmdio/capabilities_test.go b/libs/cmdio/capabilities_test.go index cb3cf5954bc..78f3cc6f837 100644 --- a/libs/cmdio/capabilities_test.go +++ b/libs/cmdio/capabilities_test.go @@ -135,6 +135,43 @@ func TestCapabilities_SupportsPrompt(t *testing.T) { } } +func TestCapabilities_SupportsLinePrompt(t *testing.T) { + gitBashPS1 := `\u@\h $MSYSTEM \w$ ` + tests := []struct { + name string + env map[string]string + }{ + {name: "NO_COLOR set", env: map[string]string{"NO_COLOR": "1"}}, + {name: "TERM=dumb", env: map[string]string{"TERM": "dumb"}}, + {name: "Git Bash", env: map[string]string{"MSYSTEM": "MINGW64", "TERM": "xterm", "PS1": gitBashPS1}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Setenv("TERM", "xterm") + for k, v := range tt.env { + t.Setenv(k, v) + } + caps := newCapabilities(t.Context(), nil, io.Discard, io.Discard) + caps.stdinIsTTY = true + caps.stdoutIsTTY = true + caps.stderrIsTTY = true + assert.False(t, caps.SupportsPrompt()) + assert.True(t, caps.SupportsLinePrompt()) + }) + } + + assert.False(t, Capabilities{stdinIsTTY: false, stdoutIsTTY: true, stderrIsTTY: true}.SupportsLinePrompt(), "stdin not TTY") + assert.False(t, Capabilities{stdinIsTTY: true, stdoutIsTTY: false, stderrIsTTY: true}.SupportsLinePrompt(), "stdout not TTY") + assert.False(t, Capabilities{stdinIsTTY: true, stdoutIsTTY: true, stderrIsTTY: false}.SupportsLinePrompt(), "stderr not TTY") +} + +func TestCapabilities_HasTTY(t *testing.T) { + assert.False(t, Capabilities{}.HasTTY()) + assert.True(t, Capabilities{stdinIsTTY: true}.HasTTY()) + assert.True(t, Capabilities{stdoutIsTTY: true}.HasTTY()) + assert.True(t, Capabilities{stderrIsTTY: true}.HasTTY()) +} + func TestDetectGitBash(t *testing.T) { ctx := t.Context() assert.False(t, detectGitBash(ctx)) diff --git a/libs/cmdio/io.go b/libs/cmdio/io.go index 368317e8ed5..8c4770e8a4e 100644 --- a/libs/cmdio/io.go +++ b/libs/cmdio/io.go @@ -83,6 +83,21 @@ func IsPromptSupported(ctx context.Context) bool { return c.capabilities.SupportsPrompt() } +// IsLinePromptSupported reports whether a plain-text line prompt ([Ask]) can +// be shown and answered: stdin, stdout and stderr are terminals. Unlike +// IsPromptSupported, it does not depend on NO_COLOR, TERM=dumb or Git Bash. +func IsLinePromptSupported(ctx context.Context) bool { + c := fromContext(ctx) + return c.capabilities.SupportsLinePrompt() +} + +// HasTTY reports whether stdin, stdout or stderr is a terminal, so that a +// person may be running the command. +func HasTTY(ctx context.Context) bool { + c := fromContext(ctx) + return c.capabilities.HasTTY() +} + // IsInputTTY reports whether stdin is a terminal, so that a user can type // input into the command. Unlike IsPromptSupported, it does not depend on // stderr, NO_COLOR, TERM=dumb or Git Bash, which only matter for rendering diff --git a/libs/cmdio/testing.go b/libs/cmdio/testing.go index 5e35973ae33..e7d3b00185a 100644 --- a/libs/cmdio/testing.go +++ b/libs/cmdio/testing.go @@ -102,6 +102,17 @@ func SetupTest(ctx context.Context, opts TestOptions) (context.Context, *Test) { } } +// NewTestIOWithInputTTY returns a cmdIO that detects its capabilities from ctx +// and the streams as [NewIO] does, except that stdin counts as a terminal. +// Wrap out or err with [FakeTTY] to make them terminals too. It exists so +// tests can check how NO_COLOR, TERM=dumb and redirected output affect a +// command that reads an answer from stdin. +func NewTestIOWithInputTTY(ctx context.Context, in io.Reader, out, err io.Writer) *cmdIO { + c := NewIO(ctx, flags.OutputText, in, out, err, "", "") + c.capabilities.stdinIsTTY = true + return c +} + // NewTestContextWithStdout creates a cmdio context that captures stdout output. // Stderr is discarded. Use this for testing data output and results. func NewTestContextWithStdout(ctx context.Context) (context.Context, *bytes.Buffer) { diff --git a/libs/telemetry/protos/README.md b/libs/telemetry/protos/README.md index 5da4cc71908..5fa57ace46b 100644 --- a/libs/telemetry/protos/README.md +++ b/libs/telemetry/protos/README.md @@ -32,7 +32,7 @@ within the upload timeout, when auth has already resolved. On the first run of a new CLI version, the CLI can install Databricks AI tools for the coding agents on the machine (see `databricks aitools auto-install --help`). It logs an `aitools_auto_install_event` with the outcome (`INSTALLED`, `SKIPPED_CI`, `SKIPPED_OPTOUT`, `SKIPPED_HOLDOUT`, `SKIPPED_NO_AGENTS`, -`FAILED`, `RETRIES_EXHAUSTED`), the machine's holdout arm and, for a failure, its error +`DECLINED`, `FAILED`, `RETRIES_EXHAUSTED`), the machine's holdout arm and, for a failure, its error category: once per version and machine, plus once per retry of a failed attempt. Nothing is logged for a version whose outcome is settled, or while the feature is turned off remotely (the CI and opt-out checks come first and are logged either way). An install with no tools installed yet also diff --git a/libs/telemetry/protos/aitools_events.go b/libs/telemetry/protos/aitools_events.go index 7095064d049..38b66507e7b 100644 --- a/libs/telemetry/protos/aitools_events.go +++ b/libs/telemetry/protos/aitools_events.go @@ -173,13 +173,17 @@ const ( // The run was in CI (the CI env var is set). AitoolsAutoInstallOutcomeSkippedCI AitoolsAutoInstallOutcome = "SKIPPED_CI" // The user turned auto-install off: DATABRICKS_AITOOLS_AUTO_INSTALL=false, - // `aitools auto-install disable`, a global `aitools uninstall`, or "no" - // to the skills prompt of `apps init`. + // `aitools auto-install disable`, a global `aitools uninstall`, "no" at + // the prompt of an earlier version, or "no" to the skills prompt of + // `apps init`. AitoolsAutoInstallOutcomeSkippedOptOut AitoolsAutoInstallOutcome = "SKIPPED_OPTOUT" // The machine is in the holdout cohort. AitoolsAutoInstallOutcomeSkippedHoldout AitoolsAutoInstallOutcome = "SKIPPED_HOLDOUT" // No supported coding agent has a config directory on the machine. AitoolsAutoInstallOutcomeSkippedNoAgents AitoolsAutoInstallOutcome = "SKIPPED_NO_AGENTS" + // The user answered "no" at the install prompt, which also turns + // auto-install off. + AitoolsAutoInstallOutcomeDeclined AitoolsAutoInstallOutcome = "DECLINED" // The attempt failed or timed out; a later run of the same version retries. AitoolsAutoInstallOutcomeFailed AitoolsAutoInstallOutcome = "FAILED" // The attempt failed and was the last one this CLI version gets. From 8926e091f13a5bf6b6388f614c8435256c129003 Mon Sep 17 00:00:00 2001 From: Anton Nekipelov <226657+anton-107@users.noreply.github.com> Date: Tue, 17 Nov 2026 14:26:39 +0000 Subject: [PATCH 4/7] aitools: address review on the auto-install opt-out - A global uninstall opts out only when it removed something, so a mistyped --skills or an empty scope no longer turns auto-install off. - `aitools auto-install ` now fails instead of printing help and exiting 0. - Document auto_install in internal/build/README.md (format, the copy-forward step for new entries, the trust model). - Say that any non-true DATABRICKS_AITOOLS_AUTO_INSTALL value opts out, and move EnvVar next to the other constant. - Add unit tests for the state file; pin the env var in the acceptance test and log the reset steps instead of discarding them. Co-authored-by: Isaac --- .../auto-install/enable-disable/output.txt | 20 +++++ .../auto-install/enable-disable/script | 14 +++- acceptance/cmd/aitools/auto-install/test.toml | 2 + cmd/aitools/autoinstall.go | 5 +- cmd/aitools/uninstall.go | 17 ++-- internal/build/README.md | 6 +- libs/aitools/autoinstall/state.go | 7 +- libs/aitools/autoinstall/state_test.go | 80 +++++++++++++++++++ libs/clicompat/clicompat_test.go | 5 +- 9 files changed, 137 insertions(+), 19 deletions(-) create mode 100644 libs/aitools/autoinstall/state_test.go diff --git a/acceptance/cmd/aitools/auto-install/enable-disable/output.txt b/acceptance/cmd/aitools/auto-install/enable-disable/output.txt index c5b3db00b1f..d651e45cd40 100644 --- a/acceptance/cmd/aitools/auto-install/enable-disable/output.txt +++ b/acceptance/cmd/aitools/auto-install/enable-disable/output.txt @@ -38,6 +38,26 @@ Automatic installation of Databricks AI tools is disabled. "opted_out": true } +=== a global uninstall that removes nothing leaves it on + +>>> [CLI] aitools uninstall --scope=global +Error: no skills or plugins installed + +Exit code: 1 + +>>> auto_install_state +{} + +=== a global uninstall of a mistyped skill leaves it on + +>>> [CLI] aitools uninstall --scope=global --skills databricks-tset +Error: skill "databricks-tset" is not installed + +Exit code: 1 + +>>> auto_install_state +{} + === a global uninstall turns it off >>> [CLI] aitools uninstall --scope=global diff --git a/acceptance/cmd/aitools/auto-install/enable-disable/script b/acceptance/cmd/aitools/auto-install/enable-disable/script index 7b802bfb85a..6bf99ed965d 100644 --- a/acceptance/cmd/aitools/auto-install/enable-disable/script +++ b/acceptance/cmd/aitools/auto-install/enable-disable/script @@ -19,13 +19,21 @@ title "disable replaces a state file it cannot read" echo "not json" > "${USERPROFILE:-$HOME}/.databricks/aitools/auto-install.json" trace $CLI aitools auto-install disable trace auto_install_state -$CLI aitools auto-install enable > /dev/null 2>&1 +$CLI aitools auto-install enable &> LOG.enable-reset-1 -title "a global uninstall turns it off" +title "a global uninstall that removes nothing leaves it on" +errcode trace $CLI aitools uninstall --scope=global +trace auto_install_state + +title "a global uninstall of a mistyped skill leaves it on" $CLI aitools install --skills-only --scope=global --agents=cursor &> LOG.install-global +errcode trace $CLI aitools uninstall --scope=global --skills databricks-tset +trace auto_install_state + +title "a global uninstall turns it off" trace $CLI aitools uninstall --scope=global trace auto_install_state -$CLI aitools auto-install enable > /dev/null 2>&1 +$CLI aitools auto-install enable &> LOG.enable-reset-2 title "a project uninstall leaves it on" mkdir -p "${USERPROFILE:-$HOME}/project" diff --git a/acceptance/cmd/aitools/auto-install/test.toml b/acceptance/cmd/aitools/auto-install/test.toml index 9fef185795b..da6d94db55c 100644 --- a/acceptance/cmd/aitools/auto-install/test.toml +++ b/acceptance/cmd/aitools/auto-install/test.toml @@ -3,6 +3,8 @@ # and files). Env.DATABRICKS_SKILLS_BASE_URL = "$DATABRICKS_HOST" Env.DATABRICKS_SKILLS_REF = "test-ref" +# Pin it so an opt-out exported in the developer's shell does not change the output. +Env.DATABRICKS_AITOOLS_AUTO_INSTALL = "true" Ignore = [ "home", diff --git a/cmd/aitools/autoinstall.go b/cmd/aitools/autoinstall.go index 7dd77896490..9cafbf4d5cf 100644 --- a/cmd/aitools/autoinstall.go +++ b/cmd/aitools/autoinstall.go @@ -4,6 +4,7 @@ import ( "context" "fmt" + "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/aitools/autoinstall" "github.com/databricks/cli/libs/cmdio" "github.com/spf13/cobra" @@ -23,7 +24,9 @@ touches the global scope. It stays off after 'databricks aitools auto-install disable' or a global-scope 'databricks aitools uninstall', until 'databricks aitools auto-install enable'. -Setting ` + autoinstall.EnvVar + `=false turns it off in that environment.`, +Setting ` + autoinstall.EnvVar + ` to anything but a true value (true, 1, yes, +on) turns it off in that environment.`, + RunE: root.ReportUnknownSubcommand, } cmd.AddCommand(newAutoInstallToggleCmd(true)) cmd.AddCommand(newAutoInstallToggleCmd(false)) diff --git a/cmd/aitools/uninstall.go b/cmd/aitools/uninstall.go index 5a12d73a8d9..88af82e36a6 100644 --- a/cmd/aitools/uninstall.go +++ b/cmd/aitools/uninstall.go @@ -29,8 +29,8 @@ func NewUninstallCmd() *cobra.Command { By default, removes all skills. Use --skills to remove specific skills only. -A global-scope uninstall also turns off the automatic installation of -Databricks AI tools on CLI upgrades; 'databricks aitools auto-install enable' +A global-scope uninstall that removes anything also turns off the automatic +installation of Databricks AI tools on CLI upgrades; 'databricks aitools auto-install enable' turns it back on. The command exits with code 1 when every removal it attempts fails.`, @@ -109,18 +109,19 @@ The command exits with code 1 when every removal it attempts fails.`, if state != nil && skillsInstalled(state, opts.Skills) { waitForTelemetryAuth(ctx) } + result, err := uninstallSkillsFn(ctx, opts) + if err != nil { + return err + } // Auto-install only ever installs in global scope, so only a global // uninstall could be undone by it on the next CLI upgrade. A partial - // one counts too: the update it runs re-adds removed skills. - if scope == installer.ScopeGlobal { + // one counts too: the update it runs re-adds removed skills. A run + // that removed nothing (a mistyped skill, nothing installed) does not. + if scope == installer.ScopeGlobal && (result.PluginsRemoved > 0 || result.SkillsChanged > 0) { if err := autoinstall.SetOptedOut(ctx, true); err != nil { log.Warnf(ctx, "Could not turn off the automatic installation of Databricks AI tools: %v", err) } } - result, err := uninstallSkillsFn(ctx, opts) - if err != nil { - return err - } // A plugin or skill that could not be removed is only a warning, unless // no copy of any skill and no plugin was removed at all. A plugin record // for an agent this CLI does not know is not an attempt: no command can diff --git a/internal/build/README.md b/internal/build/README.md index 0adf8c77a22..d82afac3751 100644 --- a/internal/build/README.md +++ b/internal/build/README.md @@ -19,6 +19,8 @@ Each key is a CLI version in semver format. Each entry defines a **range floor** `skills` is an exact pin for its CLI range or `"latest"` to install the skills repository's latest published GitHub release. When the latest-release lookup fails or the selected skills manifest is missing, ranges tracking latest use `AgentSkillsFallbackVersion` from `internal/build/clicompat.go`. Pinned ranges use their embedded manifest's skills version as the fallback. Changing either fallback takes effect in the next CLI build. Released binaries keep the fallback they shipped with. +An entry can also set `"auto_install": true` to turn on the automatic installation of Databricks AI tools on the first run of a new CLI version in its range (`aitools auto-install`). It is off when absent. A change reaches a machine on its first run of a CLI version after the change; machines that resolve to a cached or embedded manifest keep the value they have. + This skills fallback only applies to raw-file delivery. Version checks omit update availability when latest is unknown, and plugin operations don't substitute the fallback version. Raw-skills updates and reinstalls reject automatic downgrades, including with `--force`: the fallback must be at least as new as the installed release. Existing installs without a comparable release require a successful latest-release fetch. Explicit `DATABRICKS_SKILLS_REF` overrides and compatibility pins can still select an older release. @@ -62,7 +64,7 @@ When updating AppKit or recording a tested skills fallback: 1. **Run evals** on the new AppKit version. If there is no regression, proceed. 2. **Open a PR** to update `cli-compat.json`. The change depends on the type of release: - **No breaking changes**: update the highest entry's `appkit` in-place, keeping `"skills": "latest"`. To record a tested skills fallback, update `AgentSkillsFallbackVersion` in `internal/build/clicompat.go`. Do NOT add a new versioned key. - - **Breaking changes** (the new AppKit templates require specific `apps init` features, or the new skills version requires CLI commands that older CLIs lack): add a new entry keyed to the **minimum CLI version** that supports the new features. Set `"skills": "latest"` on the new entry and update the compiled skills fallback to a tested release. If skills compatibility changed, replace `"latest"` in the affected older ranges with the last compatible release. Preserve unrelated AppKit and skills compatibility boundaries. + - **Breaking changes** (the new AppKit templates require specific `apps init` features, or the new skills version requires CLI commands that older CLIs lack): add a new entry keyed to the **minimum CLI version** that supports the new features. Set `"skills": "latest"` on the new entry and update the compiled skills fallback to a tested release. If skills compatibility changed, replace `"latest"` in the affected older ranges with the last compatible release. Preserve unrelated AppKit and skills compatibility boundaries. If the previous highest entry sets `"auto_install": true`, copy it to the new entry: a new entry without it turns auto-install off for every newer CLI. This process is manual for now but can be automated as part of the release workflow in the future. Use the `/bump-cli-compat` Claude Code skill to automate the update and PR creation. @@ -82,4 +84,4 @@ Entries MUST NOT be removed from the manifest. Older CLI binaries use the lowest ## Trust model -The live manifest is fetched over HTTPS from GitHub (`raw.githubusercontent.com`). The trust boundary is: TLS certificate validation + GitHub's access controls + write access to the `main` branch of `databricks/cli`. A compromised manifest can only steer clients to existing published tags (AppKit or skills); it cannot inject arbitrary code. The CLI binary always ships an embedded fallback manifest that limits exposure to cache-TTL windows (1 hour). The local cache (`~/.cache/databricks/compat-manifest.json`) is trust-on-disk: an attacker with user-level write access to the cache directory could swap in a malicious manifest pointing to different tags. +The live manifest is fetched over HTTPS from GitHub (`raw.githubusercontent.com`). The trust boundary is: TLS certificate validation + GitHub's access controls + write access to the `main` branch of `databricks/cli`. A compromised manifest can steer clients to existing published tags (AppKit or skills), and through `auto_install` make them install those published skills and plugins without a command from the user; it cannot inject arbitrary code. The CLI binary always ships an embedded fallback manifest that limits exposure to cache-TTL windows (1 hour). The local cache (`~/.cache/databricks/compat-manifest.json`) is trust-on-disk: an attacker with user-level write access to the cache directory could swap in a malicious manifest pointing to different tags. diff --git a/libs/aitools/autoinstall/state.go b/libs/aitools/autoinstall/state.go index 5227b14ef53..313b429434e 100644 --- a/libs/aitools/autoinstall/state.go +++ b/libs/aitools/autoinstall/state.go @@ -21,6 +21,10 @@ import ( // and this state must outlive upgrades and `databricks cache clear`. const stateFileName = "auto-install.json" +// EnvVar turns auto-install off when set to anything but a true value (true, +// 1, yes, on), including an empty or unparseable value. +const EnvVar = "DATABRICKS_AITOOLS_AUTO_INSTALL" + // State is what auto-install remembers across CLI versions. type State struct { // LastVersion is the CLI version auto-install last ran for. @@ -91,9 +95,6 @@ func SetOptedOut(ctx context.Context, optedOut bool) error { return Save(ctx, s) } -// EnvVar turns auto-install off when set to false (or 0, no, off). -const EnvVar = "DATABRICKS_AITOOLS_AUTO_INSTALL" - // EnvOptedOut reports whether EnvVar turns auto-install off. func EnvOptedOut(ctx context.Context) bool { enabled, ok := env.GetBool(ctx, EnvVar) diff --git a/libs/aitools/autoinstall/state_test.go b/libs/aitools/autoinstall/state_test.go new file mode 100644 index 00000000000..ea4e5c8476c --- /dev/null +++ b/libs/aitools/autoinstall/state_test.go @@ -0,0 +1,80 @@ +package autoinstall + +import ( + "os" + "path/filepath" + "testing" + + "github.com/databricks/cli/libs/env" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestLoadMissingFileIsZeroState(t *testing.T) { + ctx := env.WithUserHomeDir(t.Context(), t.TempDir()) + + s, err := Load(ctx) + require.NoError(t, err) + assert.Equal(t, State{}, s) +} + +func TestLoadCorruptFileFails(t *testing.T) { + home := t.TempDir() + ctx := env.WithUserHomeDir(t.Context(), home) + path := filepath.Join(home, ".databricks", "aitools", stateFileName) + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) + require.NoError(t, os.WriteFile(path, []byte("not json"), 0o600)) + + _, err := Load(ctx) + assert.ErrorContains(t, err, "parse ") +} + +func TestSetOptedOutKeepsOtherFields(t *testing.T) { + ctx := env.WithUserHomeDir(t.Context(), t.TempDir()) + require.NoError(t, Save(ctx, State{LastVersion: "1.2.3", Attempts: 2})) + + require.NoError(t, SetOptedOut(ctx, true)) + s, err := Load(ctx) + require.NoError(t, err) + assert.Equal(t, State{LastVersion: "1.2.3", Attempts: 2, OptedOut: true}, s) + + require.NoError(t, SetOptedOut(ctx, false)) + s, err = Load(ctx) + require.NoError(t, err) + assert.Equal(t, State{LastVersion: "1.2.3", Attempts: 2}, s) +} + +func TestSetOptedOutReplacesCorruptFile(t *testing.T) { + home := t.TempDir() + ctx := env.WithUserHomeDir(t.Context(), home) + path := filepath.Join(home, ".databricks", "aitools", stateFileName) + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) + require.NoError(t, os.WriteFile(path, []byte("not json"), 0o600)) + + require.NoError(t, SetOptedOut(ctx, true)) + s, err := Load(ctx) + require.NoError(t, err) + assert.Equal(t, State{OptedOut: true}, s) +} + +func TestEnvOptedOut(t *testing.T) { + ctx := t.Context() + assert.False(t, EnvOptedOut(ctx), "unset") + + for value, want := range map[string]bool{ + "true": false, + "1": false, + "yes": false, + "on": false, + "false": true, + "0": true, + "no": true, + "off": true, + "": true, + "enabled": true, + } { + t.Run(value, func(t *testing.T) { + assert.Equal(t, want, EnvOptedOut(env.Set(ctx, EnvVar, value))) + }) + } +} diff --git a/libs/clicompat/clicompat_test.go b/libs/clicompat/clicompat_test.go index 1fda6a53979..f4ce7613f71 100644 --- a/libs/clicompat/clicompat_test.go +++ b/libs/clicompat/clicompat_test.go @@ -353,8 +353,9 @@ func TestResolveAutoInstall(t *testing.T) { } } -// The remote manifest is this same file on main, so turning auto_install on -// anywhere is a launch decision: change this test in the PR that makes it. +// TestManifestLeavesAutoInstallOff checks that no entry turns auto_install on. +// The remote manifest is this same file on main, so turning it on anywhere is a +// launch decision: change this test in the PR that makes it. func TestManifestLeavesAutoInstallOff(t *testing.T) { m, err := parseManifest(build.CLICompatManifestJSON) require.NoError(t, err) From cdf01c0f2ec045c358262b90055eeb0fd8bad4a3 Mon Sep 17 00:00:00 2001 From: Anton Nekipelov <226657+anton-107@users.noreply.github.com> Date: Tue, 17 Nov 2026 14:31:44 +0000 Subject: [PATCH 5/7] aitools: keep the machine ID and harden the auto-install prompt Address review on #7703: - A run that cannot ask saves the machine ID it drew (autoinstall.SaveID), so partial-terminal runs no longer redraw the holdout bucket. It now skips before the lock and the gate lookup. - Only "n"/"no" declines; any other answer, EOF, a timeout or Ctrl+C is reported as UNANSWERED and settles the version without opting out. - Discard type-ahead before asking, ask only in the terminal's foreground process group, and treat Ctrl+C at the prompt as no answer. - Skip the prompt when a goroutine still reads stdin (apps dev-remote). - Use autoinstall.Decline and promise "won't be asked again" only when the opt-out was saved. - Name only agents that would get something; reuse cmdio.HasTerminal and FakeTTYReader instead of new helpers. - Fix the auto-install event docs. Co-authored-by: Isaac --- cmd/aitools/autoinstall_run.go | 106 ++++++++++++++++++------ cmd/aitools/autoinstall_run_test.go | 99 +++++++++++++++++----- libs/aitools/autoinstall/run.go | 11 +++ libs/aitools/autoinstall/run_test.go | 11 +++ libs/aitools/autoinstall/state.go | 6 ++ libs/apps/vite/bridge.go | 2 + libs/cmdio/capabilities.go | 5 -- libs/cmdio/capabilities_test.go | 7 -- libs/cmdio/io.go | 39 +++++++-- libs/cmdio/io_test.go | 14 ++++ libs/cmdio/terminal_darwin.go | 22 +++++ libs/cmdio/terminal_linux.go | 21 +++++ libs/cmdio/terminal_other.go | 17 ++++ libs/cmdio/terminal_windows.go | 18 ++++ libs/cmdio/testing.go | 11 --- libs/telemetry/protos/README.md | 11 ++- libs/telemetry/protos/aitools_events.go | 20 +++-- 17 files changed, 334 insertions(+), 86 deletions(-) create mode 100644 libs/cmdio/terminal_darwin.go create mode 100644 libs/cmdio/terminal_linux.go create mode 100644 libs/cmdio/terminal_other.go create mode 100644 libs/cmdio/terminal_windows.go diff --git a/cmd/aitools/autoinstall_run.go b/cmd/aitools/autoinstall_run.go index 8af0041332f..85663aed05c 100644 --- a/cmd/aitools/autoinstall_run.go +++ b/cmd/aitools/autoinstall_run.go @@ -7,6 +7,8 @@ import ( "io" "log/slog" "maps" + "os" + "os/signal" "runtime" "slices" "strings" @@ -89,10 +91,20 @@ func autoInstall(ctx context.Context, path []string) { log.Debugf(ctx, "auto-install: skipped: %v", err) return } - if reason := autoinstall.Decide(in); !reason.Due() { + reason := autoinstall.Decide(in) + if !reason.Due() { log.Debugf(ctx, "auto-install: skipped: %s", reason) return } + // A run that would ask before installing but cannot skips before the lock + // and the gate lookup: it would skip after them too, and it repeats on + // every such run until a run in a terminal asks. + if reason == autoinstall.ReasonGateOff && in.State.LastVersion != version && autoInstallAsks(ctx) && !cmdio.IsLinePromptSupported(ctx) { + if tools, err := planTools(quietContext(ctx)); err == nil && len(tools.newAgents()) > 0 { + skipUnaskable(ctx, in.State.ID, autoinstall.NewID()) + return + } + } unlock, ok := autoinstall.TryLock(ctx) if !ok { @@ -114,7 +126,7 @@ func autoInstall(ctx context.Context, path []string) { // The gate costs a manifest lookup, so it is resolved only when it is // what decides the run. - reason := autoinstall.Decide(in) + reason = autoinstall.Decide(in) if reason == autoinstall.ReasonGateOff { gateCtx, cancel := context.WithTimeout(ctx, gateTimeout) in.GateOn = autoinstall.GateOn(gateCtx, version) @@ -166,15 +178,14 @@ func (r autoInstallRun) attempt(ctx context.Context, prior autoinstall.State) { log.Debugf(ctx, "auto-install: failed before asking: %v", err) r.settleFailed(ctx, err) return - case tools.existing == nil && len(tools.plan) > 0: + case len(tools.newAgents()) > 0: + // autoInstall skips this case before the lock; the state it + // decided on may have changed since. if !cmdio.IsLinePromptSupported(ctx) { - // The question could not be seen or answered safely, and - // installing without it would skip the consent. A later run - // in a terminal asks. - log.Debugf(ctx, "auto-install: skipped: cannot ask, stdin, stdout or stderr is not a terminal") + skipUnaskable(ctx, prior.ID, r.id) return } - if !r.ask(ctx, tools.plan) { + if !r.ask(ctx, tools.newAgents()) { return } } @@ -224,53 +235,80 @@ func (r autoInstallRun) attempt(ctx context.Context, prior autoinstall.State) { // autoInstallAsks reports whether a person may be at the terminal, so // auto-install asks before installing: stdin, stdout or stderr is a terminal // and no coding agent is detected (see installPromptsSupported). It asks only -// when all three are terminals and skips otherwise; the install without -// asking is left to runs with no terminal at all. The question is a -// plain-text line, so NO_COLOR, TERM=dumb and Git Bash, which turn off the TUI -// prompts of install, do not skip it. +// when the question can be seen and answered (cmdio.IsLinePromptSupported) +// and skips otherwise; the install without asking is left to runs with no +// terminal at all. The question is a plain-text line, so NO_COLOR, TERM=dumb +// and Git Bash, which turn off the TUI prompts of install, do not skip it. func autoInstallAsks(ctx context.Context) bool { - return cmdio.HasTTY(ctx) && useragent.AgentProvider() == "" + return cmdio.HasTerminal(ctx) && useragent.AgentProvider() == "" +} + +// skipUnaskable skips a run that would ask before installing but cannot: the +// question could not be seen or answered safely, and installing without it +// would skip the consent. The version is not settled, so a later run in a +// terminal asks. A machine with no stored ID (stored) gets id, so that every +// such run does not draw a new holdout bucket. +func skipUnaskable(ctx context.Context, stored, id string) { + log.Debugf(ctx, "auto-install: skipped: cannot ask (stdin, stdout or stderr is not a terminal, the CLI runs in the background, or stdin is still being read)") + if stored != "" { + return + } + if err := autoinstall.SaveID(ctx, id); err != nil { + log.Debugf(ctx, "auto-install: could not save the state: %v", err) + } } -// ask asks whether to install for the agents in plan and reports whether to go +// ask asks whether to install for the agents names and reports whether to go // ahead. The version is settled before asking, so the question is asked once // per version even when it is never answered: the process is killed, stdin -// is closed, or nobody answers within promptTimeout (an unrecognized agent in -// a pseudo-terminal). Only "no" stores the opt-out. -func (r autoInstallRun) ask(ctx context.Context, plan []agentPlanItem) bool { +// is closed, Ctrl+C is pressed, nobody answers within promptTimeout (an +// unrecognized agent in a pseudo-terminal), or the answer is neither yes nor +// no. Only "no" stores the opt-out. What was typed before the question, such +// as the next command typed while this one ran, is discarded unread. +func (r autoInstallRun) ask(ctx context.Context, names []string) bool { if err := autoinstall.Record(ctx, r.version, 0, r.id); err != nil { log.Debugf(ctx, "auto-install: skipped: could not save the state: %v", err) return false } - names := make([]string, len(plan)) - for i, it := range plan { - names[i] = it.agent.DisplayName + if err := cmdio.DiscardInput(ctx); err != nil { + log.Debugf(ctx, "auto-install: could not discard the pending input: %v", err) } actx, cancel := context.WithTimeout(ctx, promptTimeout) defer cancel() + // Ctrl+C dismisses the question instead of killing a command that + // succeeded. + actx, stop := signal.NotifyContext(actx, os.Interrupt) + defer stop() ans, err := cmdio.AskCancellable(actx, "Install Databricks AI tools for "+strings.Join(names, ", ")+"? [Y/n]", "") if err != nil { log.Debugf(ctx, "auto-install: skipped: the prompt was not answered: %v", err) - if errors.Is(err, context.DeadlineExceeded) { - cmdio.LogString(ctx, "") - } + // End the question's line. + cmdio.LogString(ctx, "") + r.settle(ctx, protos.AitoolsAutoInstallOutcomeUnanswered) return false } switch strings.ToLower(strings.TrimSpace(ans)) { case "", "y", "yes": return true + case "n", "no": + r.decline(ctx) + return false } - r.decline(ctx) + log.Debugf(ctx, "auto-install: skipped: the answer was neither yes nor no") + r.settle(ctx, protos.AitoolsAutoInstallOutcomeUnanswered) + cmdio.LogString(ctx, "Skipped. To install Databricks AI tools later, run: databricks aitools install") return false } // decline stores a "no" at the prompt as the opt-out, so no later version // asks again. func (r autoInstallRun) decline(ctx context.Context) { - if err := autoinstall.SetOptedOut(ctx, true); err != nil { + r.settle(ctx, protos.AitoolsAutoInstallOutcomeDeclined) + if err := autoinstall.Decline(ctx); err != nil { log.Warnf(ctx, "Could not save the answer, so a later CLI version asks again: %v", err) + cmdio.LogString(ctx, "Skipped. To install Databricks AI tools later, run: databricks aitools install") + return } - r.settle(ctx, protos.AitoolsAutoInstallOutcomeDeclined) cmdio.LogString(ctx, "Skipped. You won't be asked again; to install Databricks AI tools later, run: databricks aitools install") } @@ -391,6 +429,22 @@ func planTools(ctx context.Context) (toolsPlan, error) { return toolsPlan{globalDir: globalDir, plan: plan}, nil } +// newAgents names the agents a fresh install gives tools to: skills, or a +// plugin whose CLI is on PATH. An agent that keeps the plugin it has gets +// nothing. It is empty when an existing installation is updated. +func (t toolsPlan) newAgents() []string { + if t.existing != nil { + return nil + } + var names []string + for _, it := range t.plan { + if it.delivery == deliverySkills || (it.delivery == deliveryPlugin && !it.cliMissing) { + names = append(names, it.agent.DisplayName) + } + } + return names +} + // agentCLIsStoppable reports whether a timeout stops an agent CLI together // with its descendants. On Windows it stops only the CLI, and a descendant // left running could still be installing when the next run takes the lock, diff --git a/cmd/aitools/autoinstall_run_test.go b/cmd/aitools/autoinstall_run_test.go index ddecd1449dd..8a09bd4c52d 100644 --- a/cmd/aitools/autoinstall_run_test.go +++ b/cmd/aitools/autoinstall_run_test.go @@ -2,12 +2,16 @@ package aitools import ( "bytes" + "context" + "io" "os" "path/filepath" "strings" "testing" + "github.com/databricks/cli/libs/aitools/agents" "github.com/databricks/cli/libs/aitools/autoinstall" + "github.com/databricks/cli/libs/aitools/installer" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/flags" "github.com/databricks/cli/libs/telemetry" @@ -27,10 +31,11 @@ const autoInstallTestManifest = `{ // The interactive prompt needs a terminal, which acceptance tests do not // have, so these cases drive the hook the root command runs with stdin as a -// test terminal; the rest of the terminal is detected as in a real run. Like the acceptance tests in acceptance/cmd/aitools/auto-install, -// they check only what the user and the backend see: the prompt and notice -// on stderr, the skills in Cursor's directory, the stored state and the -// telemetry event. +// test terminal; the rest of the terminal is detected as in a real run. Like +// the acceptance tests in acceptance/cmd/aitools/auto-install, they check +// only what the user and the backend see: the prompt and notice on stderr, +// the skills in Cursor's directory, the stored state and the telemetry +// event. func TestAutoInstallPrompt(t *testing.T) { tests := []struct { name string @@ -79,10 +84,19 @@ func TestAutoInstallPrompt(t *testing.T) { wantOutcome: protos.AitoolsAutoInstallOutcomeDeclined, }, { - name: "no answer settles the version without opting out", - input: "", - wantPrompt: true, - wantState: autoinstall.State{LastVersion: "1.30.0", ID: "treatment-machine"}, + name: "no answer settles the version without opting out", + input: "", + wantPrompt: true, + wantState: autoinstall.State{LastVersion: "1.30.0", ID: "treatment-machine"}, + wantOutcome: protos.AitoolsAutoInstallOutcomeUnanswered, + }, + { + name: "another answer settles the version without opting out", + input: "git status\n", + wantPrompt: true, + wantStderr: []string{"Skipped. To install Databricks AI tools later, run: databricks aitools install"}, + wantState: autoinstall.State{LastVersion: "1.30.0", ID: "treatment-machine"}, + wantOutcome: protos.AitoolsAutoInstallOutcomeUnanswered, }, { name: "a failure before asking settles the version", @@ -149,6 +163,7 @@ func TestAutoInstallPrompt(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { home := setupAutoInstallHome(t, tc.state) + t.Cleanup(autoinstall.ResetDeclined) if tc.installState != "" { dir := filepath.Join(home, ".databricks", "aitools", "skills") require.NoError(t, os.MkdirAll(dir, 0o755)) @@ -166,21 +181,13 @@ func TestAutoInstallPrompt(t *testing.T) { if tc.stderrRedirected { errW = &stderr } - in := strings.NewReader(tc.input) - cio := cmdio.NewTestIOWithInputTTY(t.Context(), in, outW, errW) - if tc.stdinPiped { - cio = cmdio.NewIO(t.Context(), flags.OutputText, in, outW, errW, "", "") + var in io.Reader = strings.NewReader(tc.input) + if !tc.stdinPiped { + in = cmdio.FakeTTYReader(in) } - ctx := cmdio.InContext(t.Context(), cio) + ctx := cmdio.InContext(t.Context(), cmdio.NewIO(t.Context(), flags.OutputText, in, outW, errW, "", "")) ctx = telemetry.WithNewLogger(ctx) - - root := &cobra.Command{Use: "databricks"} - jobs := &cobra.Command{Use: "jobs"} - list := &cobra.Command{Use: "list"} - root.AddCommand(jobs) - jobs.AddCommand(list) - list.SetContext(ctx) - AutoInstall(list) + runAutoInstallHook(ctx) assert.Empty(t, stdout.String()) assert.Equal(t, tc.wantPrompt, strings.Contains(stderr.String(), "Install Databricks AI tools for Cursor? [Y/n]: "), "prompted") @@ -212,6 +219,56 @@ func TestAutoInstallPrompt(t *testing.T) { } } +// A run that cannot ask on a machine with no stored ID saves the ID it drew, +// so that later runs keep the machine's holdout bucket. +func TestAutoInstallKeepsTheIDWhenItCannotAsk(t *testing.T) { + setupAutoInstallHome(t, "{}") + t.Setenv("TERM", "xterm") + var stderr bytes.Buffer + run := func() autoinstall.State { + ctx := cmdio.InContext(t.Context(), cmdio.NewIO(t.Context(), flags.OutputText, strings.NewReader("\n"), io.Discard, cmdio.FakeTTY(&stderr), "", "")) + ctx = telemetry.WithNewLogger(ctx) + runAutoInstallHook(ctx) + assert.Empty(t, telemetry.BufferedEvents(ctx)) + state, err := autoinstall.Load(ctx) + require.NoError(t, err) + return state + } + + first := run() + assert.NotEmpty(t, first.ID) + assert.Empty(t, first.LastVersion, "the version is not settled") + assert.Equal(t, first, run()) + assert.NotContains(t, stderr.String(), "Install Databricks AI tools") +} + +func TestToolsPlanNewAgents(t *testing.T) { + claude := &agents.Agent{DisplayName: "Claude Code"} + cursor := &agents.Agent{DisplayName: "Cursor"} + codex := &agents.Agent{DisplayName: "Codex CLI"} + plan := []agentPlanItem{ + {agent: claude, delivery: deliveryPlugin}, + {agent: cursor, delivery: deliverySkills}, + {agent: codex, delivery: deliveryPlugin, cliMissing: true}, + } + + assert.Equal(t, []string{"Claude Code", "Cursor"}, toolsPlan{plan: plan}.newAgents()) + assert.Empty(t, toolsPlan{plan: plan[2:]}.newAgents(), "an agent that keeps its plugin gets nothing") + assert.Empty(t, toolsPlan{existing: &installer.InstallState{}, plan: plan}.newAgents(), "an update does not ask") +} + +// runAutoInstallHook runs the hook as the root command does after +// `databricks jobs list` succeeded. +func runAutoInstallHook(ctx context.Context) { + root := &cobra.Command{Use: "databricks"} + jobs := &cobra.Command{Use: "jobs"} + list := &cobra.Command{Use: "list"} + root.AddCommand(jobs) + jobs.AddCommand(list) + list.SetContext(ctx) + AutoInstall(list) +} + // setupAutoInstallHome gives the run a clean environment with Cursor's config // dir, a skills server, the gate on and a released CLI version. state seeds // the auto-install state (a treatment-arm machine id by default). diff --git a/libs/aitools/autoinstall/run.go b/libs/aitools/autoinstall/run.go index cd6d323d2bf..642c6891813 100644 --- a/libs/aitools/autoinstall/run.go +++ b/libs/aitools/autoinstall/run.go @@ -96,6 +96,17 @@ func Record(ctx context.Context, version string, attempts int, id string) error }) } +// SaveID stores id as the machine identifier unless one is stored already, +// without settling a version. A run that skips before Record calls it, so +// that the machine keeps its holdout bucket on later runs. +func SaveID(ctx context.Context, id string) error { + return updateState(ctx, func(s *State) { + if s.ID == "" { + s.ID = id + } + }) +} + // TryLock claims the machine-wide auto-install lock so that concurrent CLI // processes (agents often run several at once) do not install at the same // time. ok is false when another process holds it or it cannot be taken; that diff --git a/libs/aitools/autoinstall/run_test.go b/libs/aitools/autoinstall/run_test.go index 98597b9234b..fbcd7ed41ba 100644 --- a/libs/aitools/autoinstall/run_test.go +++ b/libs/aitools/autoinstall/run_test.go @@ -44,6 +44,17 @@ func TestRecordKeepsConcurrentOptOut(t *testing.T) { assert.Equal(t, "1.30.0", s.LastVersion) } +func TestSaveIDKeepsTheFirstID(t *testing.T) { + ctx := env.WithUserHomeDir(t.Context(), t.TempDir()) + + require.NoError(t, SaveID(ctx, "first")) + require.NoError(t, SaveID(ctx, "second")) + + s, err := Load(ctx) + require.NoError(t, err) + assert.Equal(t, State{ID: "first"}, s, "the version is not settled") +} + func TestLoadWaitsForStateLock(t *testing.T) { ctx := env.WithUserHomeDir(t.Context(), t.TempDir()) diff --git a/libs/aitools/autoinstall/state.go b/libs/aitools/autoinstall/state.go index b1e75dd3665..540734eb242 100644 --- a/libs/aitools/autoinstall/state.go +++ b/libs/aitools/autoinstall/state.go @@ -132,6 +132,12 @@ func Declined() bool { return declined.Load() } +// ResetDeclined forgets a Decline in this process. It exists for tests that +// decline more than once in one process. +func ResetDeclined() { + declined.Store(false) +} + // ChildEnv returns the environment for a CLI process this one starts: the // current environment, with auto-install turned off when the user declined in // this process, so the child honors the answer even if storing it failed. diff --git a/libs/apps/vite/bridge.go b/libs/apps/vite/bridge.go index 7b658e83c7d..8964d69d228 100644 --- a/libs/apps/vite/bridge.go +++ b/libs/apps/vite/bridge.go @@ -981,6 +981,8 @@ func (vb *Bridge) Start() error { }) if !vb.autoApprove { + // Stop does not end the read, so it can outlive the command. + cmdio.MarkStdinReadInBackground() go vb.readStdinLines(os.Stdin) } diff --git a/libs/cmdio/capabilities.go b/libs/cmdio/capabilities.go index 9bcf3110777..6d4f6fdbb60 100644 --- a/libs/cmdio/capabilities.go +++ b/libs/cmdio/capabilities.go @@ -52,11 +52,6 @@ func (c Capabilities) SupportsLinePrompt() bool { return c.stdinIsTTY && c.stdoutIsTTY && c.stderrIsTTY } -// HasTTY returns true if stdin, stdout or stderr is a terminal. -func (c Capabilities) HasTTY() bool { - return c.stdinIsTTY || c.stdoutIsTTY || c.stderrIsTTY -} - // SupportsColor returns true if the given writer supports colored output. // This checks both TTY status and environment variables (NO_COLOR, TERM=dumb). func (c Capabilities) SupportsColor(w io.Writer) bool { diff --git a/libs/cmdio/capabilities_test.go b/libs/cmdio/capabilities_test.go index 78f3cc6f837..ac869c39bba 100644 --- a/libs/cmdio/capabilities_test.go +++ b/libs/cmdio/capabilities_test.go @@ -165,13 +165,6 @@ func TestCapabilities_SupportsLinePrompt(t *testing.T) { assert.False(t, Capabilities{stdinIsTTY: true, stdoutIsTTY: true, stderrIsTTY: false}.SupportsLinePrompt(), "stderr not TTY") } -func TestCapabilities_HasTTY(t *testing.T) { - assert.False(t, Capabilities{}.HasTTY()) - assert.True(t, Capabilities{stdinIsTTY: true}.HasTTY()) - assert.True(t, Capabilities{stdoutIsTTY: true}.HasTTY()) - assert.True(t, Capabilities{stderrIsTTY: true}.HasTTY()) -} - func TestDetectGitBash(t *testing.T) { ctx := t.Context() assert.False(t, detectGitBash(ctx)) diff --git a/libs/cmdio/io.go b/libs/cmdio/io.go index 8c4770e8a4e..43438534ec2 100644 --- a/libs/cmdio/io.go +++ b/libs/cmdio/io.go @@ -4,8 +4,10 @@ import ( "context" "errors" "io" + "os" "strings" "sync" + "sync/atomic" tea "github.com/charmbracelet/bubbletea" "github.com/databricks/cli/libs/flags" @@ -84,18 +86,41 @@ func IsPromptSupported(ctx context.Context) bool { } // IsLinePromptSupported reports whether a plain-text line prompt ([Ask]) can -// be shown and answered: stdin, stdout and stderr are terminals. Unlike -// IsPromptSupported, it does not depend on NO_COLOR, TERM=dumb or Git Bash. +// be shown and answered: stdin, stdout and stderr are terminals, this process +// is in the terminal's foreground process group (a background job that reads +// it is stopped), and no goroutine reads stdin in the background (see +// [MarkStdinReadInBackground]). Unlike IsPromptSupported, it does not depend +// on NO_COLOR, TERM=dumb or Git Bash. func IsLinePromptSupported(ctx context.Context) bool { c := fromContext(ctx) - return c.capabilities.SupportsLinePrompt() + if !c.capabilities.SupportsLinePrompt() || stdinReadInBackground.Load() { + return false + } + f, ok := c.in.(*os.File) + return !ok || inForeground(f) } -// HasTTY reports whether stdin, stdout or stderr is a terminal, so that a -// person may be running the command. -func HasTTY(ctx context.Context) bool { +// stdinReadInBackground is set once a goroutine reads stdin for the rest of +// the process. +var stdinReadInBackground atomic.Bool + +// MarkStdinReadInBackground records that a goroutine reads os.Stdin and may +// outlive the command, so later line prompts in this process are not asked: +// that goroutine could take their answer. +func MarkStdinReadInBackground() { + stdinReadInBackground.Store(true) +} + +// DiscardInput discards what was typed at the stdin terminal but not read +// yet, so that a prompt asked after a long-running command reads only what is +// typed after the question. +func DiscardInput(ctx context.Context) error { c := fromContext(ctx) - return c.capabilities.HasTTY() + f, ok := c.in.(*os.File) + if !ok || !c.capabilities.stdinIsTTY { + return nil + } + return discardInput(f) } // IsInputTTY reports whether stdin is a terminal, so that a user can type diff --git a/libs/cmdio/io_test.go b/libs/cmdio/io_test.go index c573e78ccce..7f0b229556d 100644 --- a/libs/cmdio/io_test.go +++ b/libs/cmdio/io_test.go @@ -3,6 +3,7 @@ package cmdio import ( "bytes" "io" + "strings" "testing" "time" @@ -77,3 +78,16 @@ func TestHasTerminal(t *testing.T) { }) } } + +func TestIsLinePromptSupported(t *testing.T) { + ctx := t.Context() + terminal := InContext(ctx, NewIO(ctx, flags.OutputText, FakeTTYReader(strings.NewReader("")), FakeTTY(io.Discard), FakeTTY(io.Discard), "", "")) + assert.True(t, IsLinePromptSupported(terminal)) + + piped := InContext(ctx, NewIO(ctx, flags.OutputText, strings.NewReader(""), FakeTTY(io.Discard), FakeTTY(io.Discard), "", "")) + assert.False(t, IsLinePromptSupported(piped)) + + t.Cleanup(func() { stdinReadInBackground.Store(false) }) + MarkStdinReadInBackground() + assert.False(t, IsLinePromptSupported(terminal), "another goroutine reads stdin") +} diff --git a/libs/cmdio/terminal_darwin.go b/libs/cmdio/terminal_darwin.go new file mode 100644 index 00000000000..c8bfc935431 --- /dev/null +++ b/libs/cmdio/terminal_darwin.go @@ -0,0 +1,22 @@ +package cmdio + +import ( + "os" + + "golang.org/x/sys/unix" +) + +// inForeground reports whether this process is in the foreground process +// group of the terminal f. A background job that reads its terminal is +// stopped (SIGTTIN) until the user brings it to the foreground. +func inForeground(f *os.File) bool { + pgrp, err := unix.IoctlGetInt(int(f.Fd()), unix.TIOCGPGRP) + return err == nil && pgrp == unix.Getpgrp() +} + +// discardInput discards what was typed at the terminal f but not read yet. +// It is tcflush with TCIFLUSH, which libc implements as TIOCFLUSH with FREAD. +func discardInput(f *os.File) error { + const fread = 1 + return unix.IoctlSetPointerInt(int(f.Fd()), unix.TIOCFLUSH, fread) +} diff --git a/libs/cmdio/terminal_linux.go b/libs/cmdio/terminal_linux.go new file mode 100644 index 00000000000..e69b0b3bd42 --- /dev/null +++ b/libs/cmdio/terminal_linux.go @@ -0,0 +1,21 @@ +package cmdio + +import ( + "os" + + "golang.org/x/sys/unix" +) + +// inForeground reports whether this process is in the foreground process +// group of the terminal f. A background job that reads its terminal is +// stopped (SIGTTIN) until the user brings it to the foreground. +func inForeground(f *os.File) bool { + pgrp, err := unix.IoctlGetInt(int(f.Fd()), unix.TIOCGPGRP) + return err == nil && pgrp == unix.Getpgrp() +} + +// discardInput discards what was typed at the terminal f but not read yet +// (tcflush with TCIFLUSH). +func discardInput(f *os.File) error { + return unix.IoctlSetInt(int(f.Fd()), unix.TCFLSH, unix.TCIFLUSH) +} diff --git a/libs/cmdio/terminal_other.go b/libs/cmdio/terminal_other.go new file mode 100644 index 00000000000..1e42d7819ca --- /dev/null +++ b/libs/cmdio/terminal_other.go @@ -0,0 +1,17 @@ +//go:build !linux && !darwin && !windows + +package cmdio + +import "os" + +// inForeground reports whether this process may read the terminal f. The +// release builds' platforms check it; others assume it may. +func inForeground(*os.File) bool { + return true +} + +// discardInput discards what was typed at the terminal f but not read yet. +// Only the release builds' platforms implement it. +func discardInput(*os.File) error { + return nil +} diff --git a/libs/cmdio/terminal_windows.go b/libs/cmdio/terminal_windows.go new file mode 100644 index 00000000000..2462ff5e829 --- /dev/null +++ b/libs/cmdio/terminal_windows.go @@ -0,0 +1,18 @@ +package cmdio + +import ( + "os" + + "golang.org/x/sys/windows" +) + +// inForeground reports whether this process may read the console f. Windows +// consoles have no background jobs, so it always may. +func inForeground(*os.File) bool { + return true +} + +// discardInput discards what was typed at the console f but not read yet. +func discardInput(f *os.File) error { + return windows.FlushConsoleInputBuffer(windows.Handle(f.Fd())) +} diff --git a/libs/cmdio/testing.go b/libs/cmdio/testing.go index e7d3b00185a..5e35973ae33 100644 --- a/libs/cmdio/testing.go +++ b/libs/cmdio/testing.go @@ -102,17 +102,6 @@ func SetupTest(ctx context.Context, opts TestOptions) (context.Context, *Test) { } } -// NewTestIOWithInputTTY returns a cmdIO that detects its capabilities from ctx -// and the streams as [NewIO] does, except that stdin counts as a terminal. -// Wrap out or err with [FakeTTY] to make them terminals too. It exists so -// tests can check how NO_COLOR, TERM=dumb and redirected output affect a -// command that reads an answer from stdin. -func NewTestIOWithInputTTY(ctx context.Context, in io.Reader, out, err io.Writer) *cmdIO { - c := NewIO(ctx, flags.OutputText, in, out, err, "", "") - c.capabilities.stdinIsTTY = true - return c -} - // NewTestContextWithStdout creates a cmdio context that captures stdout output. // Stderr is discarded. Use this for testing data output and results. func NewTestContextWithStdout(ctx context.Context) (context.Context, *bytes.Buffer) { diff --git a/libs/telemetry/protos/README.md b/libs/telemetry/protos/README.md index 5fa57ace46b..eb6e80c7e54 100644 --- a/libs/telemetry/protos/README.md +++ b/libs/telemetry/protos/README.md @@ -32,10 +32,13 @@ within the upload timeout, when auth has already resolved. On the first run of a new CLI version, the CLI can install Databricks AI tools for the coding agents on the machine (see `databricks aitools auto-install --help`). It logs an `aitools_auto_install_event` with the outcome (`INSTALLED`, `SKIPPED_CI`, `SKIPPED_OPTOUT`, `SKIPPED_HOLDOUT`, `SKIPPED_NO_AGENTS`, -`DECLINED`, `FAILED`, `RETRIES_EXHAUSTED`), the machine's holdout arm and, for a failure, its error -category: once per version and machine, plus once per retry of a failed attempt. Nothing is logged -for a version whose outcome is settled, or while the feature is turned off remotely (the CI and -opt-out checks come first and are logged either way). An install with no tools installed yet also +`DECLINED`, `UNANSWERED`, `FAILED`, `RETRIES_EXHAUSTED`), the machine's holdout arm and, for a +failure, its error category: once per version and machine, plus once per retry of a failed attempt. +Nothing is logged for a version whose outcome is settled, or while the feature is turned off remotely +(the CI and opt-out checks come first and are logged either way). A run that has to ask before +installing but cannot (a stream is not a terminal, or the CLI runs in the background) logs nothing +and leaves the version to a later run that can ask; a process killed at the prompt logs nothing +either. An install with no tools installed yet also logs an `aitools_install_event` with `trigger` `AUTO_FIRST_RUN`; installs by `databricks aitools install` leave `trigger` unset. The holdout arm comes from a random machine id kept in `~/.databricks/aitools/auto-install.json`; the id itself is never sent. Like the other aitools diff --git a/libs/telemetry/protos/aitools_events.go b/libs/telemetry/protos/aitools_events.go index 38b66507e7b..f6355426946 100644 --- a/libs/telemetry/protos/aitools_events.go +++ b/libs/telemetry/protos/aitools_events.go @@ -184,7 +184,13 @@ const ( // The user answered "no" at the install prompt, which also turns // auto-install off. AitoolsAutoInstallOutcomeDeclined AitoolsAutoInstallOutcome = "DECLINED" - // The attempt failed or timed out; a later run of the same version retries. + // The install prompt got no answer: stdin was closed, Ctrl+C was pressed, + // it timed out, or the answer was neither yes nor no. Nothing is installed + // and auto-install stays on, so the next CLI version asks again. + AitoolsAutoInstallOutcomeUnanswered AitoolsAutoInstallOutcome = "UNANSWERED" + // The attempt failed or timed out; a later run of the same version + // retries. A failure before the install prompt settles the version + // instead, since a retry would install without asking. AitoolsAutoInstallOutcomeFailed AitoolsAutoInstallOutcome = "FAILED" // The attempt failed and was the last one this CLI version gets. AitoolsAutoInstallOutcomeRetriesExhausted AitoolsAutoInstallOutcome = "RETRIES_EXHAUSTED" @@ -207,14 +213,18 @@ const ( // machine, plus once per retry after a failed attempt. A version whose outcome // is settled emits nothing, and so does a run that the remote gate turns off; // the CI and opt-out checks come before the gate, so they are emitted either -// way. An install with no tools installed yet also emits an -// AitoolsInstallEvent with Trigger AUTO_FIRST_RUN. +// way. A run that has to ask before installing but cannot emits nothing and +// leaves the version to a later run. An install with no tools installed yet +// also emits an AitoolsInstallEvent with Trigger AUTO_FIRST_RUN. type AitoolsAutoInstallEvent struct { Outcome AitoolsAutoInstallOutcome `json:"outcome"` HoldoutArm AitoolsHoldoutArm `json:"holdout_arm"` - // Whether the run could prompt (cmdio.IsPromptSupported and no coding - // agent detected). A measurable false, so no omitempty. + // Whether a person may be at the terminal (cmdio.HasTerminal and no + // coding agent detected). Such a run asks before the first install of a + // version, so DECLINED and UNANSWERED come only with true; INSTALLED with + // true may still have run without asking (an update, or a retry after an + // accepted attempt). A measurable false, so no omitempty. Interactive bool `json:"interactive"` // Set for FAILED and RETRIES_EXHAUSTED. From 879027fe49490870b54c119ec0d36cc468808a82 Mon Sep 17 00:00:00 2001 From: Anton Nekipelov <226657+anton-107@users.noreply.github.com> Date: Tue, 17 Nov 2026 14:39:49 +0000 Subject: [PATCH 6/7] aitools: address review of the first-run auto-install - apps init children keep DATABRICKS_CLI_SELF_EXEC: ChildEnv takes the environ and composes with agents.SelfExecEnv. - Agent CLIs run through process.BackgroundInGroup, which forwards the first SIGINT/SIGTERM to the child's process group and then re-raises it, so Ctrl-C stops `claude plugin install` again in aitools install/update. - Skips reuse versioncheck.IsExemptCommand, agents.IsCredentialHelperCommand (covers `auth docker token`) and agents.InCI (CI=, TF_BUILD...), and add Databricks Runtime. Cache-disabled runs no longer read the state. - A gate lookup that only reaches the embedded manifest fails instead of reading as off; it uses up an attempt and does not settle the version. - An unfinished first install is marked in the state, so its skills are finished on a later run instead of being kept as untracked forever. - Auto-install updates no longer migrate or remove raw skills. - New outcomes UP_TO_DATE, KEPT and SKIPPED_PLATFORM; the update notice names only the agents whose skills or plugin changed. - Saying "no" in apps init says that auto-install is off too. - The explicit opt-out writes without the lock where locks are unsupported. - Tests: ResetDeclined, environ passed in; acceptance helpers use LOG.* and no `|| true`. Co-authored-by: Isaac --- .../aitools/auto-install/retries/output.txt | 1 + .../cmd/aitools/auto-install/script.prepare | 4 +- .../cmd/aitools/auto-install/skipped/script | 6 +- .../aitools/auto-install/upgrade/output.txt | 2 +- cmd/aitools/autoinstall.go | 4 +- cmd/aitools/autoinstall_plan_test.go | 93 ++++++++- cmd/aitools/autoinstall_run.go | 194 +++++++++++------- cmd/aitools/update.go | 21 +- cmd/apps/init.go | 13 +- cmd/apps/init_test.go | 15 ++ libs/aitools/agents/detect.go | 2 +- libs/aitools/agents/hint.go | 12 +- libs/aitools/agents/recommend.go | 3 + libs/aitools/agents/recommend_test.go | 10 +- libs/aitools/autoinstall/decide.go | 36 ++-- libs/aitools/autoinstall/decide_test.go | 17 +- libs/aitools/autoinstall/run.go | 27 ++- libs/aitools/autoinstall/run_test.go | 47 ++++- libs/aitools/autoinstall/state.go | 33 ++- libs/aitools/installer/plugin.go | 2 +- libs/clicompat/clicompat.go | 34 ++- libs/clicompat/clicompat_test.go | 14 ++ libs/process/interrupt_other.go | 12 ++ libs/process/interrupt_unix.go | 47 +++++ libs/process/interrupt_unix_test.go | 62 ++++++ libs/telemetry/protos/README.md | 15 +- libs/telemetry/protos/aitools_events.go | 16 +- libs/versioncheck/notice.go | 14 +- libs/versioncheck/notice_test.go | 2 +- 29 files changed, 569 insertions(+), 189 deletions(-) create mode 100644 libs/process/interrupt_other.go create mode 100644 libs/process/interrupt_unix.go create mode 100644 libs/process/interrupt_unix_test.go diff --git a/acceptance/cmd/aitools/auto-install/retries/output.txt b/acceptance/cmd/aitools/auto-install/retries/output.txt index 5e2d4d073bc..8bd4aa922db 100644 --- a/acceptance/cmd/aitools/auto-install/retries/output.txt +++ b/acceptance/cmd/aitools/auto-install/retries/output.txt @@ -9,6 +9,7 @@ { "last_version": "1.30.0", "attempts": 9, + "installing": true, "id": "treatment-machine" } diff --git a/acceptance/cmd/aitools/auto-install/script.prepare b/acceptance/cmd/aitools/auto-install/script.prepare index 8d8617052af..5f77180c74b 100644 --- a/acceptance/cmd/aitools/auto-install/script.prepare +++ b/acceptance/cmd/aitools/auto-install/script.prepare @@ -59,7 +59,7 @@ gate() { # cursor_skills lists the skills installed for Cursor. cursor_skills() { - local skills - skills=$("$ls_bin" "${USERPROFILE:-$HOME}/.cursor/skills" 2>/dev/null || true) + local dir="${USERPROFILE:-$HOME}/.cursor/skills" skills="" + if [ -d "$dir" ]; then skills=$("$ls_bin" "$dir"); fi echo "${skills:-(no skills)}" } diff --git a/acceptance/cmd/aitools/auto-install/skipped/script b/acceptance/cmd/aitools/auto-install/skipped/script index 0e02ca9153d..e4b76797c0c 100644 --- a/acceptance/cmd/aitools/auto-install/skipped/script +++ b/acceptance/cmd/aitools/auto-install/skipped/script @@ -25,9 +25,9 @@ export DATABRICKS_AITOOLS_AUTO_INSTALL_CLI_VERSION=1.33.0 title "commands that never trigger it" trace $CLI version trace $CLI aitools auto-install enable -$CLI completion bash > /dev/null 2>&1 -$CLI __complete jobs > /dev/null 2>&1 -$CLI help jobs > /dev/null 2>&1 +$CLI completion bash &> LOG.completion +$CLI __complete jobs &> LOG.complete +$CLI help jobs &> LOG.help trace print_telemetry trace DATABRICKS_CACHE_ENABLED=false $CLI jobs list trace print_telemetry diff --git a/acceptance/cmd/aitools/auto-install/upgrade/output.txt b/acceptance/cmd/aitools/auto-install/upgrade/output.txt index 4eaed0398a9..f608694937f 100644 --- a/acceptance/cmd/aitools/auto-install/upgrade/output.txt +++ b/acceptance/cmd/aitools/auto-install/upgrade/output.txt @@ -42,7 +42,7 @@ my notes >>> DATABRICKS_AITOOLS_AUTO_INSTALL_CLI_VERSION=1.32.0 DATABRICKS_CLI_GITHUB_API_URL=[DATABRICKS_URL]/older [CLI] jobs list >>> print_telemetry -{"command":"jobs_list","exit_code":0,"aitools_auto_install_event":{"outcome":"INSTALLED","holdout_arm":"TREATMENT","interactive":false}} +{"command":"jobs_list","exit_code":0,"aitools_auto_install_event":{"outcome":"KEPT","holdout_arm":"TREATMENT","interactive":false}} >>> skill_file databricks-kept kept 2.0.0 diff --git a/cmd/aitools/autoinstall.go b/cmd/aitools/autoinstall.go index e650ec1db8e..e92fed8c28e 100644 --- a/cmd/aitools/autoinstall.go +++ b/cmd/aitools/autoinstall.go @@ -18,8 +18,8 @@ func newAutoInstallCmd() *cobra.Command { On the first run of a new CLI version, the CLI can install the Databricks AI tools (skills and plugins) for the coding agents already on this machine, as 'databricks aitools install --scope=global' does, or update an existing -installation, as 'databricks aitools update' does. It never runs in CI and only -touches the global scope. +installation, as 'databricks aitools update' does. It never runs in CI or on +Databricks Runtime, and only touches the global scope. It stays off after 'databricks aitools auto-install disable', a global-scope 'databricks aitools uninstall', or a "no" to the skills prompt of diff --git a/cmd/aitools/autoinstall_plan_test.go b/cmd/aitools/autoinstall_plan_test.go index 2fa0f4934b2..14dc7ef3411 100644 --- a/cmd/aitools/autoinstall_plan_test.go +++ b/cmd/aitools/autoinstall_plan_test.go @@ -1,11 +1,16 @@ package aitools import ( + "maps" "os" "path/filepath" "testing" + "github.com/databricks/cli/libs/aitools/agents" + "github.com/databricks/cli/libs/aitools/autoinstall" "github.com/databricks/cli/libs/aitools/installer" + "github.com/databricks/cli/libs/telemetry/protos" + "github.com/spf13/cobra" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -41,29 +46,45 @@ func TestPlanToolsRunsNoAgentCLIWhereTimeoutsCannotStopThem(t *testing.T) { autoInstallHome(t) setAgentCLIsStoppable(t, true) - tools, err := planTools(t.Context()) + tools, err := planTools(t.Context(), autoinstall.State{}) require.NoError(t, err) assert.Equal(t, []string{"Claude Code", "Cursor"}, planAgentNames(tools.plan)) // The plugin install would run Claude Code's CLI. setAgentCLIsStoppable(t, false) - tools, err = planTools(t.Context()) + tools, err = planTools(t.Context(), autoinstall.State{}) require.NoError(t, err) assert.Equal(t, []string{"Cursor"}, planAgentNames(tools.plan)) - assert.Empty(t, tools.keep) + assert.Empty(t, tools.skip) // Updating the recorded plugins would run agent CLIs too. saveGlobalState(t, map[string]string{"databricks-sql": "0.1.0"}) - tools, err = planTools(t.Context()) + tools, err = planTools(t.Context(), autoinstall.State{}) require.NoError(t, err) assert.Nil(t, tools.existing) - assert.NotEmpty(t, tools.keep) + assert.Equal(t, protos.AitoolsAutoInstallOutcomeSkippedPlatform, tools.skip) setAgentCLIsStoppable(t, true) - tools, err = planTools(t.Context()) + tools, err = planTools(t.Context(), autoinstall.State{}) require.NoError(t, err) assert.NotNil(t, tools.existing) - assert.Empty(t, tools.keep) + assert.Empty(t, tools.skip) +} + +func TestPlanToolsReportsWhyNothingIsInstalled(t *testing.T) { + home := autoInstallHome(t) + + // Claude Code alone installs through its CLI, which is not run here. + require.NoError(t, os.RemoveAll(filepath.Join(home, ".cursor"))) + setAgentCLIsStoppable(t, false) + tools, err := planTools(t.Context(), autoinstall.State{}) + require.NoError(t, err) + assert.Equal(t, protos.AitoolsAutoInstallOutcomeSkippedPlatform, tools.skip) + + require.NoError(t, os.RemoveAll(filepath.Join(home, ".claude"))) + tools, err = planTools(t.Context(), autoinstall.State{}) + require.NoError(t, err) + assert.Equal(t, protos.AitoolsAutoInstallOutcomeSkippedNoAgents, tools.skip) } func TestPlanToolsKeepsSkillsWithoutInstallState(t *testing.T) { @@ -74,15 +95,69 @@ func TestPlanToolsKeepsSkillsWithoutInstallState(t *testing.T) { require.NoError(t, os.MkdirAll(filepath.Join(dir, "databricks-sql"), 0o755)) require.NoError(t, os.WriteFile(filepath.Join(dir, "databricks-sql", "SKILL.md"), []byte("user edit"), 0o644)) - tools, err := planTools(t.Context()) + tools, err := planTools(t.Context(), autoinstall.State{}) require.NoError(t, err) - assert.NotEmpty(t, tools.keep) + assert.Equal(t, protos.AitoolsAutoInstallOutcomeKept, tools.skip) assert.Empty(t, tools.plan) result, err := tools.install(t.Context()) require.NoError(t, err) + assert.Equal(t, protos.AitoolsAutoInstallOutcomeKept, result.outcome) assert.Empty(t, result.agents) data, err := os.ReadFile(filepath.Join(dir, "databricks-sql", "SKILL.md")) require.NoError(t, err) assert.Equal(t, "user edit", string(data)) + + // Skills that an unfinished install of auto-install left are its own, so + // a later run finishes the install. + tools, err = planTools(t.Context(), autoinstall.State{Installing: true}) + require.NoError(t, err) + assert.Empty(t, tools.skip) + assert.True(t, tools.fresh()) +} + +func TestChangedAgents(t *testing.T) { + autoInstallHome(t) + dir := saveGlobalState(t, map[string]string{"databricks-sql": "0.1.0"}) + require.NoError(t, os.MkdirAll(filepath.Join(dir, "databricks-sql"), 0o755)) + before, err := installer.LoadState(dir) + require.NoError(t, err) + + // A new release in which every outdated skill was kept changed nothing. + after := *before + after.Release = "v0.3.0" + assert.Empty(t, changedAgents(t.Context(), before, &after)) + + // Only the agent whose plugin was updated changed. + after.Plugins = maps.Clone(before.Plugins) + rec := after.Plugins[agents.NameClaudeCode] + rec.Version = "0.3.0" + after.Plugins[agents.NameClaudeCode] = rec + assert.Equal(t, []string{"Claude Code"}, changedAgents(t.Context(), before, &after)) +} + +func TestSkipsAutoInstall(t *testing.T) { + root := &cobra.Command{Use: "databricks"} + add := func(parent *cobra.Command, name string) *cobra.Command { + c := &cobra.Command{Use: name} + parent.AddCommand(c) + return c + } + jobsList := add(add(root, "jobs"), "list") + auth := add(root, "auth") + authToken := add(auth, "token") + dockerToken := add(add(auth, "docker"), "token") + authDescribe := add(auth, "describe") + aitoolsList := add(add(root, "aitools"), "list") + experimentalAitools := add(add(add(root, "experimental"), "aitools"), "tools") + version := add(root, "version") + completion := add(add(root, "completion"), "zsh") + complete := add(root, cobra.ShellCompRequestCmd) + + for _, cmd := range []*cobra.Command{authToken, dockerToken, aitoolsList, experimentalAitools, version, completion, complete} { + assert.True(t, skipsAutoInstall(cmd), cmd.CommandPath()) + } + for _, cmd := range []*cobra.Command{jobsList, authDescribe} { + assert.False(t, skipsAutoInstall(cmd), cmd.CommandPath()) + } } diff --git a/cmd/aitools/autoinstall_run.go b/cmd/aitools/autoinstall_run.go index 524e8457e10..88cb1444bd0 100644 --- a/cmd/aitools/autoinstall_run.go +++ b/cmd/aitools/autoinstall_run.go @@ -16,6 +16,7 @@ import ( "github.com/databricks/cli/libs/aitools/autoinstall" "github.com/databricks/cli/libs/aitools/installer" "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/dbr" "github.com/databricks/cli/libs/flags" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/telemetry" @@ -26,8 +27,8 @@ import ( const ( // gateTimeout bounds the gate lookup, one compatibility manifest fetch. - // A lookup that runs out reads as off: every machine pays it once per - // version while the gate is off, so it is kept short. + // Every machine pays it once per version while the gate is off, so it is + // kept short. A lookup that fails counts as a failed attempt. gateTimeout = 3 * time.Second // installTimeout bounds the install or update (its downloads and the @@ -51,31 +52,44 @@ func AutoInstall(cmd *cobra.Command) { log.Debugf(ctx, "auto-install: panic: %v", r) } }() - autoInstall(ctx, commandPath(cmd)) + autoInstall(ctx, skipsAutoInstall(cmd)) } -// commandPath returns the names of cmd and its parents below the root. -func commandPath(cmd *cobra.Command) []string { - var path []string - for c := cmd; c.HasParent(); c = c.Parent() { - path = append([]string{c.Name()}, path...) +// skipsAutoInstall reports whether cmd never triggers auto-install: the +// commands the new-release notice skips (`version` is how a user checks what +// was just installed; help and shell completion output is read by machines or +// shown on every keystroke), the credential helpers that other tools call for +// a token, and the aitools commands, which manage the tools explicitly. +// +// Runs without a terminal and runs with JSON output are not skipped, unlike +// for the notice: coding agents run the CLI that way, and they are what the +// tools are for. Auto-install writes nothing to stdout. +func skipsAutoInstall(cmd *cobra.Command) bool { + if versioncheck.IsExemptCommand(cmd) || agents.IsCredentialHelperCommand(cmd) { + return true + } + for c := cmd; c != nil; c = c.Parent() { + if c.Name() == "aitools" { + return true + } } - return path + return false } -func autoInstall(ctx context.Context, path []string) { +func autoInstall(ctx context.Context, skippedCommand bool) { version := autoinstall.Version(ctx) in := autoinstall.Inputs{ - Command: path, - CacheDisabled: autoinstall.CacheDisabled(ctx), - Version: version, - CI: versioncheck.InCI(ctx), + SkippedCommand: skippedCommand, + OnRuntime: dbr.RunsOnRuntime(ctx), + CacheDisabled: autoinstall.CacheDisabled(ctx), + Version: version, + CI: agents.InCI(ctx), // A "no" earlier in this process counts even if storing it failed. EnvOptOut: autoinstall.EnvOptedOut(ctx) || autoinstall.Declined(), } - // Commands and builds that never auto-install skip before the state is - // read, and without a log line: they are most runs. - if reason := autoinstall.Decide(in); reason == autoinstall.ReasonSkippedCommand || reason == autoinstall.ReasonDevelopmentBuild { + // What never auto-installs skips before the state is read, and without a + // log line: these are most runs. + if slices.Contains(stateFreeSkips, autoinstall.Decide(in)) { return } var err error @@ -112,8 +126,20 @@ func autoInstall(ctx context.Context, path []string) { reason := autoinstall.Decide(in) if reason == autoinstall.ReasonGateOff { gateCtx, cancel := context.WithTimeout(ctx, gateTimeout) - in.GateOn = autoinstall.GateOn(gateCtx, version) + in.GateOn, err = autoinstall.GateOn(gateCtx, version) cancel() + if err != nil { + // The gate is unknown, so the version is not settled: a later run + // looks it up again. Like a failed install, the lookup uses up an + // attempt, so a machine that is offline stops paying for it. The + // feature may be off, so nothing is reported. + attempt := autoinstall.NextAttempt(in.State, version) + log.Debugf(ctx, "auto-install: attempt %d of %d: could not resolve the gate: %v", attempt, autoinstall.MaxAttempts, err) + if err := autoinstall.Record(ctx, version, attempt, id); err != nil { + log.Debugf(ctx, "auto-install: could not save the state: %v", err) + } + return + } reason = autoinstall.Decide(in) } run := autoInstallRun{ @@ -145,12 +171,9 @@ func autoInstall(ctx context.Context, path []string) { // attempt installs or updates the tools and reports the outcome. prior is the // state the run was decided on. func (r autoInstallRun) attempt(ctx context.Context, prior autoinstall.State) { - attempt := 1 - if prior.LastVersion == r.version { - attempt = prior.Attempts + 1 - } + attempt := autoinstall.NextAttempt(prior, r.version) qctx := quietContext(ctx) - tools, err := planTools(qctx) + tools, err := planTools(qctx, prior) // Count the attempt before making it: an attempt that never finishes (the // process is killed) still counts, and a state that cannot be saved stops // auto-install instead of retrying it on every run. @@ -158,6 +181,11 @@ func (r autoInstallRun) attempt(ctx context.Context, prior autoinstall.State) { log.Debugf(ctx, "auto-install: skipped: could not save the state: %v", recErr) return } + if err == nil && tools.fresh() { + // Skills an unfinished install leaves have no install state; the + // marker tells a later run that they are its own. + err = autoinstall.MarkInstalling(ctx) + } var result installResult if err == nil { runCtx, cancel := context.WithTimeout(qctx, installTimeout) @@ -166,10 +194,10 @@ func (r autoInstallRun) attempt(ctx context.Context, prior autoinstall.State) { } if _, refused := errors.AsType[*installer.RefusedError](err); refused { // The installed skills are newer than the release this CLI would - // install (the downgrade guard). They are installed, and retrying - // changes nothing until a newer release, which a later version gets. + // install (the downgrade guard). Retrying changes nothing until a + // newer release, which a later version gets. log.Debugf(ctx, "auto-install: kept the installed tools: %v", err) - err = nil + result, err = installResult{outcome: protos.AitoolsAutoInstallOutcomeKept}, nil } switch { case err != nil: @@ -179,11 +207,9 @@ func (r autoInstallRun) attempt(ctx context.Context, prior autoinstall.State) { } log.Debugf(ctx, "auto-install: attempt %d of %d failed: %v", attempt, autoinstall.MaxAttempts, err) r.report(ctx, outcome, classifyInstallError(err)) - case result.noAgents: - r.settle(ctx, protos.AitoolsAutoInstallOutcomeSkippedNoAgents) default: - r.settle(ctx, protos.AitoolsAutoInstallOutcomeInstalled) - if len(result.agents) > 0 { + r.settle(ctx, result.outcome) + if result.outcome == protos.AitoolsAutoInstallOutcomeInstalled && len(result.agents) > 0 { verb := "Installed" if result.updated { verb = "Updated" @@ -193,6 +219,14 @@ func (r autoInstallRun) attempt(ctx context.Context, prior autoinstall.State) { } } +// stateFreeSkips are the reasons Decide finds without the state. +var stateFreeSkips = []autoinstall.Reason{ + autoinstall.ReasonSkippedCommand, + autoinstall.ReasonOnRuntime, + autoinstall.ReasonCacheDisabled, + autoinstall.ReasonDevelopmentBuild, +} + // skipOutcomes are the reported outcomes of the skips that settle a version. var skipOutcomes = map[autoinstall.Reason]protos.AitoolsAutoInstallOutcome{ autoinstall.ReasonSkippedCI: protos.AitoolsAutoInstallOutcomeSkippedCI, @@ -242,8 +276,8 @@ func holdoutArm(bucket int) protos.AitoolsHoldoutArm { // installResult is what one auto-install attempt did. type installResult struct { - // noAgents is set when no supported agent has a config directory. - noAgents bool + // outcome is the reported outcome of an attempt that did not fail. + outcome protos.AitoolsAutoInstallOutcome // updated is set when an existing installation was updated. updated bool // agents names the agents that got new or updated tools. @@ -251,21 +285,29 @@ type installResult struct { } // toolsPlan is what an auto-install attempt does: update the existing global -// installation, or install for the agents in plan. +// installation, install for the agents in plan, or leave the tools alone. type toolsPlan struct { globalDir string existing *installer.InstallState plan []agentPlanItem - // keep, when set, is why the installed tools are left as they are. - keep string + // skip, when set, is the outcome of a plan that changes nothing, and + // skipWhy says why. + skip protos.AitoolsAutoInstallOutcome + skipWhy string +} + +// fresh reports whether the plan is a first install. +func (t toolsPlan) fresh() bool { + return t.skip == "" && t.existing == nil && len(t.plan) > 0 } // planTools plans what `databricks aitools install --scope=global` runs // without prompts, for the agents whose config directory exists. An existing // global installation is updated as `databricks aitools update --scope=global` -// does instead, which keeps its downgrade guard, and keeps the skills whose -// files the user modified at their version. Planning reads only local state. -func planTools(ctx context.Context) (toolsPlan, error) { +// does instead, which keeps its downgrade guard, keeps the skills whose files +// the user modified at their version, and does not migrate skills to plugins. +// Planning reads only local state; prior is the auto-install state. +func planTools(ctx context.Context, prior autoinstall.State) (toolsPlan, error) { globalDir, err := installer.GlobalSkillsDir(ctx) if err != nil { return toolsPlan{}, err @@ -275,29 +317,29 @@ func planTools(ctx context.Context) (toolsPlan, error) { return toolsPlan{}, err } if existing != nil { - if !agentCLIsStoppable { - // The update runs agent CLIs to update recorded plugins and to - // migrate raw skills to plugins. - candidates, err := pluginMigrationCandidatesForScope(ctx, installer.ScopeGlobal, existing) - if err != nil { - return toolsPlan{}, err - } - if len(existing.Plugins) > 0 || len(candidates) > 0 { - return toolsPlan{keep: "the update would run agent CLIs"}, nil - } + // The update runs agent CLIs to update the recorded plugins. + if !agentCLIsStoppable && len(existing.Plugins) > 0 { + return toolsPlan{skip: protos.AitoolsAutoInstallOutcomeSkippedPlatform, skipWhy: "the update would run agent CLIs"}, nil } return toolsPlan{globalDir: globalDir, existing: existing}, nil } // Skills without install state have no checksums to tell the user's edits - // from ours, and an install would replace them. - if installer.HasUntrackedSkills(ctx, installer.ScopeGlobal) { - return toolsPlan{keep: "skills without install state are on disk"}, nil + // from ours, and an install would replace them. The ones an unfinished + // install of auto-install left are its own. + if !prior.Installing && installer.HasUntrackedSkills(ctx, installer.ScopeGlobal) { + return toolsPlan{skip: protos.AitoolsAutoInstallOutcomeKept, skipWhy: "skills without install state are on disk"}, nil } plan := buildPlan(ctx, agents.DetectInstalled(ctx), installer.ScopeGlobal, false, false) + if len(plan) == 0 { + return toolsPlan{skip: protos.AitoolsAutoInstallOutcomeSkippedNoAgents, skipWhy: "no supported coding agent has a config directory"}, nil + } if !agentCLIsStoppable { plan = slices.DeleteFunc(plan, func(it agentPlanItem) bool { return it.delivery == deliveryPlugin && !it.cliMissing }) + if len(plan) == 0 { + return toolsPlan{skip: protos.AitoolsAutoInstallOutcomeSkippedPlatform, skipWhy: "every agent installs through its CLI"}, nil + } } return toolsPlan{globalDir: globalDir, plan: plan}, nil } @@ -310,16 +352,13 @@ var agentCLIsStoppable = runtime.GOOS != "windows" // install carries out the plan. func (t toolsPlan) install(ctx context.Context) (installResult, error) { - if t.keep != "" { - log.Debugf(ctx, "auto-install: kept the installed tools: %s", t.keep) - return installResult{}, nil + if t.skip != "" { + log.Debugf(ctx, "auto-install: nothing to do: %s", t.skipWhy) + return installResult{outcome: t.skip}, nil } if t.existing != nil { return updateTools(ctx, t.globalDir, t.existing) } - if len(t.plan) == 0 { - return installResult{noAgents: true}, nil - } opts := installer.InstallOptions{Scope: installer.ScopeGlobal} outcomes, err := executePlan(ctx, &installer.GitHubManifestSource{}, t.plan, opts, installInvocation{scope: installer.ScopeGlobal}, true) topErr := topLevelFailure(err) @@ -334,7 +373,7 @@ func (t toolsPlan) install(ctx context.Context) (installResult, error) { names = append(names, o.agent.DisplayName) } } - return installResult{agents: names}, err + return installResult{outcome: protos.AitoolsAutoInstallOutcomeInstalled, agents: names}, err } // updateTools runs the update command's body for the global scope. Its funnel @@ -351,23 +390,36 @@ func updateTools(ctx context.Context, globalDir string, before *installer.Instal if err != nil { return installResult{}, err } - result := installResult{updated: true} - if after != nil && toolsChanged(before, after) { - for _, e := range buildAgentEntries(ctx, map[string]*installer.InstallState{installer.ScopeGlobal: after}) { - if _, ok := e.Installed[installer.ScopeGlobal]; ok { - result.agents = append(result.agents, e.DisplayName) - } - } + result := installResult{outcome: protos.AitoolsAutoInstallOutcomeUpToDate, updated: true} + if after != nil { + result.agents = changedAgents(ctx, before, after) + } + if len(result.agents) > 0 { + result.outcome = protos.AitoolsAutoInstallOutcomeInstalled } return result, nil } -// toolsChanged reports whether an update changed the installed skills or -// plugin versions. -func toolsChanged(before, after *installer.InstallState) bool { - return before.Release != after.Release || - !maps.Equal(before.Skills, after.Skills) || - !maps.EqualFunc(before.Plugins, after.Plugins, func(a, b installer.PluginRecord) bool { return a.Version == b.Version }) +// changedAgents names the agents an update changed: those with skills when a +// skill version changed, and those whose plugin version changed. The release +// is not compared: it advances even when every outdated skill was kept. +func changedAgents(ctx context.Context, before, after *installer.InstallState) []string { + skillsChanged := !maps.Equal(before.Skills, after.Skills) + var names []string + for _, e := range buildAgentEntries(ctx, map[string]*installer.InstallState{installer.ScopeGlobal: after}) { + info, ok := e.Installed[installer.ScopeGlobal] + if !ok { + continue + } + changed := skillsChanged + if info.Delivery == deliveryPlugin.String() { + changed = before.Plugins[e.Name].Version != after.Plugins[e.Name].Version + } + if changed { + names = append(names, e.DisplayName) + } + } + return names } // quietContext discards what the installer prints and logs its warnings at diff --git a/cmd/aitools/update.go b/cmd/aitools/update.go index 9c6269d35c3..8ab71a87e65 100644 --- a/cmd/aitools/update.go +++ b/cmd/aitools/update.go @@ -78,7 +78,8 @@ type updateFlags struct { skills, scope string project, global bool // keepModified is not a flag: auto-install sets it (see - // installer.UpdateOptions.KeepModified). + // installer.UpdateOptions.KeepModified). It also keeps the raw skills of + // agents that have, or could get, the plugin. keepModified bool } @@ -153,9 +154,16 @@ func runUpdate(ctx context.Context, f updateFlags) error { updatedAnything = true } - migrationCandidates, err := pluginMigrationCandidatesForScope(ctx, scope, state) - if err != nil { - return err + // With keepModified (auto-install), raw skills are not migrated to + // the plugin: the migration removes them without checking whether + // the user modified them, and the agent would load the plugin's + // copies instead. + var migrationCandidates []pluginMigrationCandidate + if !f.keepModified { + migrationCandidates, err = pluginMigrationCandidatesForScope(ctx, scope, state) + if err != nil { + return err + } } if f.check { printPluginMigrationCheckResults(ctx, migrationCandidates) @@ -226,8 +234,9 @@ func runUpdate(ctx context.Context, f updateFlags) error { fileAgents, manifestPlugins := excludePluginAgents(ctx, scope, targetAgents, state) // Raw skills an earlier install gave an agent that now has the // plugin would show every skill twice, so remove them as - // install does when it keeps the plugin. - if !f.check { + // install does when it keeps the plugin. Auto-install leaves + // them, as it does not migrate them either. + if !f.check && !f.keepModified { for _, a := range manifestPlugins { if err := updateCleanupLegacyFn(ctx, a, scope); err != nil { log.Debugf(ctx, "Legacy skill cleanup for %s failed: %v", a.DisplayName, err) diff --git a/cmd/apps/init.go b/cmd/apps/init.go index c010ad1a536..5e704e53e4c 100644 --- a/cmd/apps/init.go +++ b/cmd/apps/init.go @@ -1965,14 +1965,20 @@ func runPostCreateDeploy(ctx context.Context, profile string) error { args = append(args, "--profile", profile) } cmd := exec.CommandContext(ctx, executable, args...) - cmd.Env = agents.SelfExecEnv(os.Environ()) + cmd.Env = selfExecEnv(os.Environ()) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr cmd.Stdin = os.Stdin - cmd.Env = autoinstall.ChildEnv() return cmd.Run() } +// selfExecEnv returns environ for a CLI process apps init starts: it prints no +// coding-agent hint (the parent's output already has one), and it does not +// auto-install when the user turned the tools down in this process. +func selfExecEnv(environ []string) []string { + return agents.SelfExecEnv(autoinstall.ChildEnv(environ)) +} + // runPostCreateDev runs the dev or dev-remote command in the current directory. func runPostCreateDev(ctx context.Context, mode prompt.RunMode, projectInit initializer.Initializer, workDir, profile string) error { switch mode { @@ -1995,11 +2001,10 @@ func runPostCreateDev(ctx context.Context, mode prompt.RunMode, projectInit init args = append(args, "--profile", profile) } cmd := exec.CommandContext(ctx, executable, args...) - cmd.Env = agents.SelfExecEnv(os.Environ()) + cmd.Env = selfExecEnv(os.Environ()) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr cmd.Stdin = os.Stdin - cmd.Env = autoinstall.ChildEnv() return cmd.Run() default: return nil diff --git a/cmd/apps/init_test.go b/cmd/apps/init_test.go index 013e3a9253a..3551102727a 100644 --- a/cmd/apps/init_test.go +++ b/cmd/apps/init_test.go @@ -13,6 +13,7 @@ import ( "strings" "testing" + "github.com/databricks/cli/libs/aitools/autoinstall" "github.com/databricks/cli/libs/apps/manifest" "github.com/databricks/cli/libs/apps/pkgmanager" "github.com/databricks/cli/libs/apps/prompt" @@ -1810,3 +1811,17 @@ func TestPackageManagerValidation(t *testing.T) { }) } } + +func TestSelfExecEnvCarriesBothMarkers(t *testing.T) { + t.Cleanup(autoinstall.ResetDeclined) + ctx := env.WithUserHomeDir(t.Context(), t.TempDir()) + + // The child prints no coding-agent hint, as before a "no" was given. + assert.Equal(t, []string{"HOME=/home/user", "DATABRICKS_CLI_SELF_EXEC=1"}, selfExecEnv([]string{"HOME=/home/user"})) + + // After a "no", the child also does not auto-install. + require.NoError(t, autoinstall.Decline(ctx)) + environ := selfExecEnv([]string{"HOME=/home/user"}) + assert.Contains(t, environ, "DATABRICKS_CLI_SELF_EXEC=1") + assert.Contains(t, environ, autoinstall.EnvVar+"=false") +} diff --git a/libs/aitools/agents/detect.go b/libs/aitools/agents/detect.go index f071e6c77e1..cb4ba22cfb4 100644 --- a/libs/aitools/agents/detect.go +++ b/libs/aitools/agents/detect.go @@ -112,7 +112,7 @@ func (a *Agent) ProbePlugin(ctx context.Context) error { cctx, cancel := context.WithTimeout(ctx, pluginProbeTimeout) defer cancel() - if _, err := process.Background(cctx, []string{path, "plugin", "--help"}, process.WithProcessGroup()); err != nil { + if _, err := process.BackgroundInGroup(cctx, []string{path, "plugin", "--help"}); err != nil { return fmt.Errorf("%s plugin subcommand unavailable: %w", a.Binary, errors.Join(ErrPluginUnsupported, err)) } return nil diff --git a/libs/aitools/agents/hint.go b/libs/aitools/agents/hint.go index bee5ed0afaa..6d8bab1d385 100644 --- a/libs/aitools/agents/hint.go +++ b/libs/aitools/agents/hint.go @@ -186,7 +186,7 @@ func MaybeHint(ctx context.Context, cmd *cobra.Command, pluginRecorded PluginRec if isHintExemptCommand(cmd) { return } - if isCredentialHelperCommand(cmd) { + if IsCredentialHelperCommand(cmd) { return } if env.Get(ctx, selfExecEnv) != "" { @@ -237,7 +237,7 @@ func (h *agentHint) claim(ctx context.Context, key string) bool { // stdin even when the output goes through a pipe (`2>&1 | less`), so any // terminal means a human may read the tag raw. func claudeNativeHintLine(ctx context.Context) string { - if env.Get(ctx, claudeChildSessionEnv) == "" || inCI(ctx) { + if env.Get(ctx, claudeChildSessionEnv) == "" || InCI(ctx) { return "" } if cmdio.HasTerminal(ctx) { @@ -246,13 +246,13 @@ func claudeNativeHintLine(ctx context.Context) string { return claudeNativeHint } -// inCI reports whether the CLI runs in CI: the generic CI variable set to +// InCI reports whether the CLI runs in CI: the generic CI variable set to // anything but an explicit false (some providers set it to their own name), or // any provider the SDK detects for its user agent (e.g. Azure Pipelines' // TF_BUILD, which doesn't set CI). The false values are the ones env.GetBool // accepts; GetBool itself can't be used because it also maps a provider name // to false. -func inCI(ctx context.Context) bool { +func InCI(ctx context.Context) bool { switch strings.ToLower(env.Get(ctx, ciEnv)) { case "", "0", "f", "false", "no", "off": default: @@ -261,9 +261,9 @@ func inCI(ctx context.Context) bool { return useragent.CiCdProvider() != "" } -// isCredentialHelperCommand reports whether cmd is `databricks auth token` or +// IsCredentialHelperCommand reports whether cmd is `databricks auth token` or // `databricks auth docker token`. -func isCredentialHelperCommand(cmd *cobra.Command) bool { +func IsCredentialHelperCommand(cmd *cobra.Command) bool { if cmd.Name() != tokenCommandName { return false } diff --git a/libs/aitools/agents/recommend.go b/libs/aitools/agents/recommend.go index d968d994f21..31d3b94f064 100644 --- a/libs/aitools/agents/recommend.go +++ b/libs/aitools/agents/recommend.go @@ -76,7 +76,10 @@ func RecommendSkillsInstall(ctx context.Context, installFn func(context.Context) // must not install what the user just turned down. if err := autoinstall.Decline(ctx); err != nil { log.Warnf(ctx, "Could not turn off the automatic installation of Databricks AI tools: %v", err) + return nil } + // The prompt asked about now; say that the answer also holds later. + cmdio.LogString(ctx, "Databricks AI tools won't be installed automatically either. To turn that back on, run: databricks aitools auto-install enable") return nil } diff --git a/libs/aitools/agents/recommend_test.go b/libs/aitools/agents/recommend_test.go index be744da9f7d..5430b147c0b 100644 --- a/libs/aitools/agents/recommend_test.go +++ b/libs/aitools/agents/recommend_test.go @@ -101,6 +101,7 @@ func TestRecommendSkillsInstallNonInteractive(t *testing.T) { } func TestRecommendSkillsInstallInteractiveDecline(t *testing.T) { + t.Cleanup(autoinstall.ResetDeclined) tmpDir := t.TempDir() t.Setenv("HOME", tmpDir) t.Setenv("USERPROFILE", tmpDir) @@ -120,7 +121,11 @@ func TestRecommendSkillsInstallInteractiveDecline(t *testing.T) { ctx = telemetry.WithNewLogger(ctx) // Drain stderr so the prompt write doesn't block. - go func() { _, _ = io.Copy(io.Discard, testIO.Stderr) }() + stderr := make(chan string, 1) + go func() { + b, _ := io.ReadAll(testIO.Stderr) + stderr <- string(b) + }() errc := make(chan error, 1) go func() { @@ -139,6 +144,9 @@ func TestRecommendSkillsInstallInteractiveDecline(t *testing.T) { require.NoError(t, err) assert.True(t, state.OptedOut) assert.True(t, autoinstall.Declined()) + // The user is told that the answer also holds for later installs. + testIO.Done() + assert.Contains(t, <-stderr, "databricks aitools auto-install enable") } func TestMaybeVersionCheckTip(t *testing.T) { diff --git a/libs/aitools/autoinstall/decide.go b/libs/aitools/autoinstall/decide.go index cb359a60e60..0b4d2832a48 100644 --- a/libs/aitools/autoinstall/decide.go +++ b/libs/aitools/autoinstall/decide.go @@ -1,8 +1,6 @@ package autoinstall import ( - "slices" - "github.com/databricks/cli/internal/build" "golang.org/x/mod/semver" ) @@ -24,6 +22,7 @@ type Reason string const ( // Not due: the run records and reports nothing. ReasonSkippedCommand Reason = "skipped_command" + ReasonOnRuntime Reason = "on_runtime" ReasonCacheDisabled Reason = "cache_disabled" ReasonDevelopmentBuild Reason = "development_build" ReasonAlreadyRan Reason = "already_ran" @@ -44,7 +43,7 @@ const ( // settles the version. A run that is not due changes nothing. func (r Reason) Due() bool { switch r { - case ReasonSkippedCommand, ReasonCacheDisabled, ReasonDevelopmentBuild, ReasonAlreadyRan, ReasonRetriesExhausted: + case ReasonSkippedCommand, ReasonOnRuntime, ReasonCacheDisabled, ReasonDevelopmentBuild, ReasonAlreadyRan, ReasonRetriesExhausted: return false case ReasonSkippedCI, ReasonSkippedOptOut, ReasonGateOff, ReasonSkippedHoldout, ReasonRun: return true @@ -54,8 +53,12 @@ func (r Reason) Due() bool { // Inputs is everything Decide depends on. type Inputs struct { - // Command is the command path below the root, e.g. ["auth", "token"]. - Command []string + // SkippedCommand is set for a command that never triggers auto-install. + SkippedCommand bool + + // OnRuntime is set on Databricks Runtime, where the home directory does + // not outlive the cluster and the CLI version is the platform's. + OnRuntime bool // CacheDisabled is DATABRICKS_CACHE_ENABLED=false. CacheDisabled bool @@ -66,7 +69,7 @@ type Inputs struct { // State is the persisted state: last version, attempts and opt-out. State State - // CI is the CI env var check of the version-check notice. + // CI is set when the CLI runs in continuous integration. CI bool // EnvOptOut is DATABRICKS_AITOOLS_AUTO_INSTALL=false, or a "no" earlier in @@ -80,14 +83,6 @@ type Inputs struct { Bucket int } -// skippedCommands never trigger auto-install anywhere in their path: help -// and shell completion output is read by machines or shown on every -// keystroke, `version` is how a user checks what was just installed, and the -// aitools subtree manages the tools explicitly. "__complete" and -// "__completeNoDesc" are cobra.ShellCompRequestCmd and -// cobra.ShellCompNoDescRequestCmd. -var skippedCommands = []string{"aitools", "help", "version", "completion", "__complete", "__completeNoDesc"} - // Decide returns whether this run should install, settle the version, or do // nothing. The checks run from cheapest to most expensive, so callers can // leave GateOn false and look the gate up only when Decide returns @@ -95,8 +90,10 @@ var skippedCommands = []string{"aitools", "help", "version", "completion", "__co // the holdout arm only exists once the feature is on. func Decide(in Inputs) Reason { switch { - case isSkippedCommand(in.Command): + case in.SkippedCommand: return ReasonSkippedCommand + case in.OnRuntime: + return ReasonOnRuntime case in.CacheDisabled: return ReasonCacheDisabled // Development and snapshot builds change version with every commit, so @@ -128,12 +125,3 @@ func Decide(in Inputs) Reason { func InHoldout(bucket int) bool { return bucket < HoldoutPercent } - -func isSkippedCommand(path []string) bool { - for _, name := range path { - if slices.Contains(skippedCommands, name) { - return true - } - } - return slices.Equal(path, []string{"auth", "token"}) -} diff --git a/libs/aitools/autoinstall/decide_test.go b/libs/aitools/autoinstall/decide_test.go index e19a9c7491e..352f78637b1 100644 --- a/libs/aitools/autoinstall/decide_test.go +++ b/libs/aitools/autoinstall/decide_test.go @@ -9,7 +9,6 @@ import ( func TestDecide(t *testing.T) { // eligible is a run of a released version never seen before, with the gate on, in the treatment arm. eligible := Inputs{ - Command: []string{"jobs", "list"}, Version: "1.30.0", GateOn: true, Bucket: HoldoutPercent, @@ -21,15 +20,8 @@ func TestDecide(t *testing.T) { }{ {name: "eligible", modify: func(in *Inputs) {}, want: ReasonRun}, - {name: "aitools subtree", modify: func(in *Inputs) { in.Command = []string{"aitools", "list"} }, want: ReasonSkippedCommand}, - {name: "experimental aitools", modify: func(in *Inputs) { in.Command = []string{"experimental", "aitools", "tools", "query"} }, want: ReasonSkippedCommand}, - {name: "help", modify: func(in *Inputs) { in.Command = []string{"help"} }, want: ReasonSkippedCommand}, - {name: "version", modify: func(in *Inputs) { in.Command = []string{"version"} }, want: ReasonSkippedCommand}, - {name: "completion", modify: func(in *Inputs) { in.Command = []string{"completion", "zsh"} }, want: ReasonSkippedCommand}, - {name: "shell completion request", modify: func(in *Inputs) { in.Command = []string{"__complete"} }, want: ReasonSkippedCommand}, - {name: "shell completion request without descriptions", modify: func(in *Inputs) { in.Command = []string{"__completeNoDesc"} }, want: ReasonSkippedCommand}, - {name: "auth token", modify: func(in *Inputs) { in.Command = []string{"auth", "token"} }, want: ReasonSkippedCommand}, - {name: "other auth command", modify: func(in *Inputs) { in.Command = []string{"auth", "describe"} }, want: ReasonRun}, + {name: "skipped command", modify: func(in *Inputs) { in.SkippedCommand = true }, want: ReasonSkippedCommand}, + {name: "Databricks Runtime", modify: func(in *Inputs) { in.OnRuntime = true }, want: ReasonOnRuntime}, {name: "cache disabled", modify: func(in *Inputs) { in.CacheDisabled = true }, want: ReasonCacheDisabled}, {name: "development build", modify: func(in *Inputs) { in.Version = "1.31.0-dev+abcdef123456" }, want: ReasonDevelopmentBuild}, @@ -56,7 +48,8 @@ func TestDecide(t *testing.T) { {name: "last bucket is treatment", modify: func(in *Inputs) { in.Bucket = 99 }, want: ReasonRun}, {name: "skipped command before everything", modify: func(in *Inputs) { - in.Command = []string{"version"} + in.SkippedCommand = true + in.OnRuntime = true in.CacheDisabled = true in.CI = true }, want: ReasonSkippedCommand}, @@ -72,7 +65,7 @@ func TestDecide(t *testing.T) { } func TestReasonDue(t *testing.T) { - notDue := []Reason{ReasonSkippedCommand, ReasonCacheDisabled, ReasonDevelopmentBuild, ReasonAlreadyRan, ReasonRetriesExhausted} + notDue := []Reason{ReasonSkippedCommand, ReasonOnRuntime, ReasonCacheDisabled, ReasonDevelopmentBuild, ReasonAlreadyRan, ReasonRetriesExhausted} due := []Reason{ReasonSkippedCI, ReasonSkippedOptOut, ReasonGateOff, ReasonSkippedHoldout, ReasonRun} for _, r := range notDue { assert.False(t, r.Due(), r) diff --git a/libs/aitools/autoinstall/run.go b/libs/aitools/autoinstall/run.go index cd6d323d2bf..38856ce660a 100644 --- a/libs/aitools/autoinstall/run.go +++ b/libs/aitools/autoinstall/run.go @@ -59,14 +59,19 @@ func CacheDisabled(ctx context.Context) bool { } // GateOn reports whether the compatibility manifest turns auto-install on for -// version. A manifest that cannot be resolved reads as off. -func GateOn(ctx context.Context, version string) bool { - on, err := clicompat.ResolveAutoInstall(ctx, version) - if err != nil { - log.Debugf(ctx, "auto-install: could not resolve the gate: %v", err) - return false +// version. It fails when the manifest could not be fetched: the gate is then +// unknown, not off. +func GateOn(ctx context.Context, version string) (bool, error) { + return clicompat.ResolveAutoInstall(ctx, version) +} + +// NextAttempt returns the number of the attempt a run for version makes after +// prior: 1 for a version the state has not failed for. +func NextAttempt(prior State, version string) int { + if prior.LastVersion == version { + return prior.Attempts + 1 } - return on + return 1 } // NewID returns a random machine identifier for State.ID. @@ -90,12 +95,20 @@ func Record(ctx context.Context, version string, attempts int, id string) error return updateState(ctx, func(s *State) { s.LastVersion = version s.Attempts = attempts + if attempts == 0 { + s.Installing = false + } if s.ID == "" { s.ID = id } }) } +// MarkInstalling sets State.Installing before a first install starts. +func MarkInstalling(ctx context.Context) error { + return updateState(ctx, func(s *State) { s.Installing = true }) +} + // TryLock claims the machine-wide auto-install lock so that concurrent CLI // processes (agents often run several at once) do not install at the same // time. ok is false when another process holds it or it cannot be taken; that diff --git a/libs/aitools/autoinstall/run_test.go b/libs/aitools/autoinstall/run_test.go index 98597b9234b..b73b51d6eb5 100644 --- a/libs/aitools/autoinstall/run_test.go +++ b/libs/aitools/autoinstall/run_test.go @@ -96,13 +96,54 @@ func TestLockReturnsOtherErrorsAtOnce(t *testing.T) { assert.Equal(t, 1, tries) } +func TestSetOptedOutWithoutLocks(t *testing.T) { + ctx := env.WithUserHomeDir(t.Context(), t.TempDir()) + + // An explicit opt-out must work where the filesystem has no locks. + orig := tryLock + t.Cleanup(func() { tryLock = orig }) + tryLock = func(*os.File) error { return errors.New("no locks here") } + require.NoError(t, SetOptedOut(ctx, true)) + + s, err := load(ctx) + require.NoError(t, err) + assert.True(t, s.OptedOut) + + // A lock another process holds is waited for, not skipped. + tryLock = func(*os.File) error { return errLockHeld } + assert.ErrorIs(t, SetOptedOut(ctx, false), errLockHeld) +} + +func TestRecordClearsInstallingWhenSettled(t *testing.T) { + ctx := env.WithUserHomeDir(t.Context(), t.TempDir()) + + require.NoError(t, MarkInstalling(ctx)) + // A failed attempt leaves the marker for the next run. + require.NoError(t, Record(ctx, "1.30.0", 1, "machine")) + s, err := Load(ctx) + require.NoError(t, err) + assert.True(t, s.Installing) + + require.NoError(t, Record(ctx, "1.30.0", 0, "machine")) + s, err = Load(ctx) + require.NoError(t, err) + assert.False(t, s.Installing) +} + +func TestNextAttempt(t *testing.T) { + assert.Equal(t, 1, NextAttempt(State{}, "1.30.0")) + assert.Equal(t, 1, NextAttempt(State{LastVersion: "1.29.0", Attempts: 4}, "1.30.0")) + assert.Equal(t, 5, NextAttempt(State{LastVersion: "1.30.0", Attempts: 4}, "1.30.0")) +} + func TestChildEnvCarriesTheRefusal(t *testing.T) { - t.Cleanup(func() { declined.Store(false) }) + t.Cleanup(ResetDeclined) - assert.NotContains(t, ChildEnv(), EnvVar+"=false") + environ := []string{"HOME=/home/user"} + assert.Equal(t, environ, ChildEnv(environ)) // A CLI child started after a "no" must not install either, even when // the opt-out could not be stored. declined.Store(true) - assert.Equal(t, EnvVar+"=false", ChildEnv()[len(ChildEnv())-1]) + assert.Equal(t, []string{"HOME=/home/user", EnvVar + "=false"}, ChildEnv(environ)) } diff --git a/libs/aitools/autoinstall/state.go b/libs/aitools/autoinstall/state.go index f48c09d1e83..c332b98497c 100644 --- a/libs/aitools/autoinstall/state.go +++ b/libs/aitools/autoinstall/state.go @@ -37,6 +37,12 @@ type State struct { // `databricks apps init`. OptedOut bool `json:"opted_out,omitempty"` + // Installing is set while auto-install makes a first install, and stays + // set when the install did not finish. The skills such an install left + // have no install state, like the ones the user put there, so a later run + // uses it to finish the install instead of leaving the skills alone. + Installing bool `json:"installing,omitempty"` + // ID is a random per-machine identifier, created on the first run that // decides anything. Only its Bucket is used, and it never leaves the // machine. The CLI has no other stable machine identifier. @@ -99,13 +105,19 @@ func save(ctx context.Context, s State) error { // SetOptedOut turns auto-install off (true) or back on (false), keeping the // rest of the state. A state file that cannot be read is replaced: the user's -// explicit choice matters more than a broken record. +// explicit choice matters more than a broken record. Where the state lock +// cannot be taken at all (a filesystem without locks), it writes without it: +// auto-install cannot run there, so nothing else writes the state. func SetOptedOut(ctx context.Context, optedOut bool) error { unlock, err := lock(ctx, stateLockFileName, stateLockWait) - if err != nil { + switch { + case errors.Is(err, errLockHeld): return err + case err != nil: + log.Debugf(ctx, "Saving the auto-install state without its lock: %v", err) + default: + defer unlock() } - defer unlock() s, err := load(ctx) if err != nil { log.Debugf(ctx, "Replacing unreadable auto-install state: %v", err) @@ -132,11 +144,16 @@ func Declined() bool { return declined.Load() } -// ChildEnv returns the environment for a CLI process this one starts: the -// current environment, with auto-install turned off when the user declined in -// this process, so the child honors the answer even if storing it failed. -func ChildEnv() []string { - environ := os.Environ() +// ResetDeclined forgets a Decline in this process. Tests that decline use it +// to clean up. +func ResetDeclined() { + declined.Store(false) +} + +// ChildEnv returns environ for a CLI process this one starts, with +// auto-install turned off when the user declined in this process, so the child +// honors the answer even if storing it failed. +func ChildEnv(environ []string) []string { if Declined() { environ = append(environ, EnvVar+"=false") } diff --git a/libs/aitools/installer/plugin.go b/libs/aitools/installer/plugin.go index 99d4b8fb32f..75ad7e5edd5 100644 --- a/libs/aitools/installer/plugin.go +++ b/libs/aitools/installer/plugin.go @@ -83,7 +83,7 @@ func runAgentCmd(ctx context.Context, timeout time.Duration, argv []string) (str defer cancel() // A timed-out agent CLI is stopped with its descendants: one left running // would hold the output pipes open and block past the timeout. - out, err := process.Background(cctx, argv, process.WithProcessGroup()) + out, err := process.BackgroundInGroup(cctx, argv) if err != nil && errors.Is(cctx.Err(), context.DeadlineExceeded) { return out, &agentCmdTimeoutError{timeout: timeout, err: err} } diff --git a/libs/clicompat/clicompat.go b/libs/clicompat/clicompat.go index 7d9f6f746fa..b59a0e2222f 100644 --- a/libs/clicompat/clicompat.go +++ b/libs/clicompat/clicompat.go @@ -97,9 +97,18 @@ var httpClient = &http.Client{Timeout: fetchTimeout} // Set DATABRICKS_CACHE_ENABLED=false to bypass the local cache (tiers 1 and 3a), // which is useful to recover from a bad cached manifest. func FetchManifest(ctx context.Context) (Manifest, error) { + m, _, err := fetchManifest(ctx) + return m, err +} + +// fetchManifest is FetchManifest. fallback is set when the remote and the +// local cache both failed and the manifest is the embedded one, which may be +// older than either. +func fetchManifest(ctx context.Context) (m Manifest, fallback bool, err error) { if force, ok := env.GetBool(ctx, "DATABRICKS_FORCE_EMBEDDED_COMPAT"); ok && force { log.Debugf(ctx, "Using embedded manifest (DATABRICKS_FORCE_EMBEDDED_COMPAT=true)") - return parseEmbeddedManifest() + m, err := parseEmbeddedManifest() + return m, false, err } cacheEnabled := true @@ -121,7 +130,7 @@ func FetchManifest(ctx context.Context) (Manifest, error) { // Tier 1: local file is fresh. if localErr == nil && local.isFresh(cacheTTL) { log.Debugf(ctx, "Using cached manifest from %s", localPath) - return local.manifest, nil + return local.manifest, false, nil } // Tier 2: fetch from remote (local file missing or stale). @@ -130,23 +139,23 @@ func FetchManifest(ctx context.Context) (Manifest, error) { if cacheEnabled { writeLocalManifest(ctx, localPath, m) } - return m, nil + return m, false, nil } // Tier 3a: local file exists but stale — use it anyway. if localErr == nil { log.Debugf(ctx, "Using stale cached manifest (remote failed: %v)", fetchErr) - return local.manifest, nil + return local.manifest, false, nil } // Tier 3b: embedded manifest. m, embeddedErr := parseEmbeddedManifest() if embeddedErr == nil { log.Debugf(ctx, "Using embedded manifest (remote and local cache failed)") - return m, nil + return m, true, nil } - return nil, fmt.Errorf("all manifest sources failed (remote: %w, embedded: %w)", fetchErr, embeddedErr) + return nil, false, fmt.Errorf("all manifest sources failed (remote: %w, embedded: %w)", fetchErr, embeddedErr) } // parseEmbeddedManifest parses the embedded manifest once and caches the result. @@ -295,9 +304,18 @@ func ResolveAgentSkillsVersion(ctx context.Context) (string, error) { } // ResolveAutoInstall reports whether the manifest entry for cliVersion turns on -// the automatic installation of Databricks AI tools. +// the automatic installation of Databricks AI tools. It fails when only the +// embedded manifest is available: that one predates any remote change of the +// gate, so it cannot tell off from not known yet. func ResolveAutoInstall(ctx context.Context, cliVersion string) (bool, error) { - entry, err := resolveEntry(ctx, cliVersion) + m, fallback, err := fetchManifest(ctx) + if err != nil { + return false, err + } + if fallback { + return false, errors.New("the compatibility manifest could not be fetched") + } + entry, err := Resolve(m, cliVersion) if err != nil { return false, err } diff --git a/libs/clicompat/clicompat_test.go b/libs/clicompat/clicompat_test.go index 1fda6a53979..388abd02262 100644 --- a/libs/clicompat/clicompat_test.go +++ b/libs/clicompat/clicompat_test.go @@ -353,6 +353,20 @@ func TestResolveAutoInstall(t *testing.T) { } } +func TestResolveAutoInstallFailsOnTheEmbeddedFallback(t *testing.T) { + ctx := testContext(t) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + })) + defer srv.Close() + redirectToServer(t, srv) + + // Neither the remote nor a cached manifest is there: the gate is unknown, + // not off. + _, err := ResolveAutoInstall(ctx, "1.30.0") + assert.Error(t, err) +} + // The remote manifest is this same file on main, so turning auto_install on // anywhere is a launch decision: change this test in the PR that makes it. func TestManifestLeavesAutoInstallOff(t *testing.T) { diff --git a/libs/process/interrupt_other.go b/libs/process/interrupt_other.go new file mode 100644 index 00000000000..73625fa5060 --- /dev/null +++ b/libs/process/interrupt_other.go @@ -0,0 +1,12 @@ +//go:build !unix + +package process + +import "context" + +// BackgroundInGroup runs args as Background does, with WithProcessGroup. On +// these platforms the child stays in the console's process group, so Ctrl-C +// reaches it directly and nothing is forwarded. +func BackgroundInGroup(ctx context.Context, args []string, opts ...execOption) (string, error) { + return Background(ctx, args, append(opts, WithProcessGroup())...) +} diff --git a/libs/process/interrupt_unix.go b/libs/process/interrupt_unix.go new file mode 100644 index 00000000000..26fd8ffdbd3 --- /dev/null +++ b/libs/process/interrupt_unix.go @@ -0,0 +1,47 @@ +//go:build unix + +package process + +import ( + "context" + "os" + "os/signal" + "syscall" +) + +// BackgroundInGroup runs args as Background does, with WithProcessGroup, so a +// cancelled ctx stops the child's descendants too. +// +// The new process group takes the child out of the terminal's foreground +// group, so Ctrl-C no longer reaches it. While the child runs, the first SIGINT +// or SIGTERM is caught instead: it stops the group, and is then raised again +// in this process, which ends as it would have without the group. The default +// disposition is restored before the group is stopped, so a second signal ends +// this process at once. +func BackgroundInGroup(ctx context.Context, args []string, opts ...execOption) (string, error) { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + sigCh := make(chan os.Signal, 1) + signal.Notify(sigCh, os.Interrupt, syscall.SIGTERM) + caught := make(chan os.Signal, 1) + done := make(chan struct{}) + go func() { + defer close(caught) + select { + case sig := <-sigCh: + signal.Stop(sigCh) + caught <- sig + cancel() + case <-done: + } + }() + + out, err := Background(ctx, args, append(opts, WithProcessGroup())...) + close(done) + signal.Stop(sigCh) + if sig, ok := <-caught; ok { + _ = syscall.Kill(os.Getpid(), sig.(syscall.Signal)) + } + return out, err +} diff --git a/libs/process/interrupt_unix_test.go b/libs/process/interrupt_unix_test.go new file mode 100644 index 00000000000..090a3477b1b --- /dev/null +++ b/libs/process/interrupt_unix_test.go @@ -0,0 +1,62 @@ +//go:build unix + +package process + +import ( + "errors" + "os" + "os/exec" + "path/filepath" + "syscall" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// interruptHelperEnv makes TestBackgroundInGroupInterruptHelper run +// BackgroundInGroup in a helper process, which the test interrupts. Raising +// the signal again ends the process, so it cannot run in the test process. +const interruptHelperEnv = "PROCESS_TEST_INTERRUPT_HELPER_PIDFILE" + +func TestBackgroundInGroupInterruptHelper(t *testing.T) { + pidFile := os.Getenv(interruptHelperEnv) + if pidFile == "" { + t.Skip("run by TestBackgroundInGroupForwardsInterrupt") + } + _, _ = BackgroundInGroup(t.Context(), []string{"sh", "-c", `sleep 300 & echo $! > "$1"; wait`, "sh", pidFile}) + // The raised signal ends the process asynchronously. + time.Sleep(5 * time.Second) + t.Fatal("the interrupt did not end the process") +} + +func TestBackgroundInGroupForwardsInterrupt(t *testing.T) { + pidFile := filepath.Join(t.TempDir(), "grandchild.pid") + helper := exec.Command(os.Args[0], "-test.run=^TestBackgroundInGroupInterruptHelper$") + helper.Env = append(os.Environ(), interruptHelperEnv+"="+pidFile) + require.NoError(t, helper.Start()) + + grandchildPid := waitForPid(t, pidFile) + // A Ctrl-C in the terminal signals the helper, which is in the foreground + // group, but not the child, which has its own group. + require.NoError(t, helper.Process.Signal(os.Interrupt)) + + waitErr := make(chan error, 1) + go func() { waitErr <- helper.Wait() }() + select { + case err := <-waitErr: + exitErr, ok := errors.AsType[*exec.ExitError](err) + require.True(t, ok, "helper exited with %v", err) + status := exitErr.Sys().(syscall.WaitStatus) + assert.True(t, status.Signaled(), "the helper must end as the interrupt would have, got %v", err) + assert.Equal(t, syscall.SIGINT, status.Signal()) + case <-time.After(processGroupGracePeriod + 5*time.Second): + _ = helper.Process.Kill() + t.Fatal("the interrupt did not end the helper") + } + + assert.Eventually(t, func() bool { + return errors.Is(syscall.Kill(grandchildPid, 0), syscall.ESRCH) + }, 5*time.Second, 20*time.Millisecond, "grandchild %d outlived the interrupt", grandchildPid) +} diff --git a/libs/telemetry/protos/README.md b/libs/telemetry/protos/README.md index 5da4cc71908..45805ab0c70 100644 --- a/libs/telemetry/protos/README.md +++ b/libs/telemetry/protos/README.md @@ -31,12 +31,13 @@ within the upload timeout, when auth has already resolved. On the first run of a new CLI version, the CLI can install Databricks AI tools for the coding agents on the machine (see `databricks aitools auto-install --help`). It logs an `aitools_auto_install_event` -with the outcome (`INSTALLED`, `SKIPPED_CI`, `SKIPPED_OPTOUT`, `SKIPPED_HOLDOUT`, `SKIPPED_NO_AGENTS`, -`FAILED`, `RETRIES_EXHAUSTED`), the machine's holdout arm and, for a failure, its error -category: once per version and machine, plus once per retry of a failed attempt. Nothing is logged -for a version whose outcome is settled, or while the feature is turned off remotely (the CI and -opt-out checks come first and are logged either way). An install with no tools installed yet also -logs an `aitools_install_event` with `trigger` `AUTO_FIRST_RUN`; installs by `databricks aitools -install` leave `trigger` unset. The holdout arm comes from a random machine id kept in +with the outcome (`INSTALLED`, `UP_TO_DATE`, `KEPT`, `SKIPPED_CI`, `SKIPPED_OPTOUT`, +`SKIPPED_HOLDOUT`, `SKIPPED_NO_AGENTS`, `SKIPPED_PLATFORM`, `FAILED`, `RETRIES_EXHAUSTED`), the +machine's holdout arm and, for a failure, its error category: once per version and machine, plus +once per retry of a failed attempt. Nothing is logged for a version whose outcome is settled, on +Databricks Runtime, or while the feature is turned off remotely (the CI and opt-out checks come +first and are logged either way). An install with no tools installed yet also logs an +`aitools_install_event` with `trigger` `AUTO_FIRST_RUN`; installs by `databricks aitools install` +leave `trigger` unset. The holdout arm comes from a random machine id kept in `~/.databricks/aitools/auto-install.json`; the id itself is never sent. Like the other aitools events, these upload at exit only when the command resolved auth. diff --git a/libs/telemetry/protos/aitools_events.go b/libs/telemetry/protos/aitools_events.go index 7095064d049..eb67e1cce93 100644 --- a/libs/telemetry/protos/aitools_events.go +++ b/libs/telemetry/protos/aitools_events.go @@ -168,9 +168,21 @@ type AitoolsAutoInstallOutcome string const ( AitoolsAutoInstallOutcomeUnspecified AitoolsAutoInstallOutcome = "TYPE_UNSPECIFIED" - // The tools were installed, or an existing installation was updated. + // The tools were installed, or an existing installation was updated and + // changed. AitoolsAutoInstallOutcomeInstalled AitoolsAutoInstallOutcome = "INSTALLED" - // The run was in CI (the CI env var is set). + // An existing installation was updated and nothing changed: it was current, + // or every outdated skill was kept because the user modified it. + AitoolsAutoInstallOutcomeUpToDate AitoolsAutoInstallOutcome = "UP_TO_DATE" + // The installed tools were left as they are: skills without install state + // (installed by hand, or by an older CLI), or skills newer than the release + // this CLI installs. + AitoolsAutoInstallOutcomeKept AitoolsAutoInstallOutcome = "KEPT" + // Every change would have run an agent CLI, which auto-install does not do + // on Windows: a plan of only plugin installs, or an update of an + // installation with plugins. + AitoolsAutoInstallOutcomeSkippedPlatform AitoolsAutoInstallOutcome = "SKIPPED_PLATFORM" + // The run was in CI (the CI env var, or a CI provider's own). AitoolsAutoInstallOutcomeSkippedCI AitoolsAutoInstallOutcome = "SKIPPED_CI" // The user turned auto-install off: DATABRICKS_AITOOLS_AUTO_INSTALL=false, // `aitools auto-install disable`, a global `aitools uninstall`, or "no" diff --git a/libs/versioncheck/notice.go b/libs/versioncheck/notice.go index 14dc0a62b41..9c57136722b 100644 --- a/libs/versioncheck/notice.go +++ b/libs/versioncheck/notice.go @@ -243,7 +243,7 @@ func notifyEnabled(ctx context.Context, cmd *cobra.Command) bool { func gatherConditions(ctx context.Context, cmd *cobra.Command) notifyConditions { optedOut, _ := env.GetBool(ctx, disableEnv) - ci := InCI(ctx) + ci, _ := env.GetBool(ctx, ciEnv) cacheDisabled := false if enabled, ok := env.GetBool(ctx, cacheEnabledEnv); ok && !enabled { cacheDisabled = true @@ -263,25 +263,19 @@ func gatherConditions(ctx context.Context, cmd *cobra.Command) notifyConditions ci: ci, nonInteractive: nonInteractive, jsonOutput: isJSONOutput(cmd), - exemptCommand: isExemptCommand(cmd), + exemptCommand: IsExemptCommand(cmd), } } -// InCI reports whether the CLI runs in continuous integration. -func InCI(ctx context.Context) bool { - ci, _ := env.GetBool(ctx, ciEnv) - return ci -} - func isJSONOutput(cmd *cobra.Command) bool { f := cmd.Flag("output") return f != nil && f.Value.String() == "json" } -// isExemptCommand suppresses the notice for commands where it would be noise or +// IsExemptCommand reports whether cmd gets no notice, as it for commands where it would be noise or // would corrupt machine-readable output: the version commands themselves, shell // completion, and help. -func isExemptCommand(cmd *cobra.Command) bool { +func IsExemptCommand(cmd *cobra.Command) bool { for c := cmd; c != nil; c = c.Parent() { switch c.Name() { case "version", "completion", "help", cobra.ShellCompRequestCmd, cobra.ShellCompNoDescRequestCmd: diff --git a/libs/versioncheck/notice_test.go b/libs/versioncheck/notice_test.go index eadd50c707c..c8643e860c2 100644 --- a/libs/versioncheck/notice_test.go +++ b/libs/versioncheck/notice_test.go @@ -148,7 +148,7 @@ func TestIsExemptCommand(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - assert.Equal(t, tt.want, isExemptCommand(tt.cmd)) + assert.Equal(t, tt.want, IsExemptCommand(tt.cmd)) }) } } From eafb2514f717d87ea0e149670911a4862321f793 Mon Sep 17 00:00:00 2001 From: Anton Nekipelov <226657+anton-107@users.noreply.github.com> Date: Tue, 17 Nov 2026 15:07:11 +0000 Subject: [PATCH 7/7] aitools: unset CI provider variables in the auto-install acceptance tests agents.InCI also detects providers such as GITHUB_ACTIONS, so on CI runners every case reported SKIPPED_CI. Co-authored-by: Isaac --- acceptance/cmd/aitools/auto-install/script.prepare | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/acceptance/cmd/aitools/auto-install/script.prepare b/acceptance/cmd/aitools/auto-install/script.prepare index 5f77180c74b..8a130b71809 100644 --- a/acceptance/cmd/aitools/auto-install/script.prepare +++ b/acceptance/cmd/aitools/auto-install/script.prepare @@ -4,8 +4,9 @@ auto_install_state() { if [ -f "$path" ]; then cat "$path"; else echo "(no auto-install state)"; fi } -# The host's own CI flag or opt-out would change every case. -unset CI DATABRICKS_AITOOLS_AUTO_INSTALL +# The host's own CI flag or opt-out would change every case. CI detection also +# reads the providers' own variables (the SDK's useragent.CiCdProvider list). +unset CI DATABRICKS_AITOOLS_AUTO_INSTALL GITHUB_ACTIONS GITLAB_CI JENKINS_URL TF_BUILD CIRCLECI TRAVIS BITBUCKET_BUILD_NUMBER PROJECT_ID BUILD_ID PROJECT_NUMBER LOCATION CODEBUILD_BUILD_ARN TFC_RUN_ID # Telemetry is printed by print_telemetry. unset DATABRICKS_CLI_DISABLE_TELEMETRY DATABRICKS_CLI_TELEMETRY_TIMEOUT