From 2423089167da2a9e1b1d8f6dd0ec69062d28658b Mon Sep 17 00:00:00 2001 From: Artemiy Kolesnikov Date: Fri, 13 Nov 2026 04:41:39 +0000 Subject: [PATCH 1/5] aitools: Hint the AI Runtime skill only on air subcommands The Claude Code install hint fired on every non-aitools command with a generic line ("Databricks skills are not installed..."). Coding agents treat it as unrelated noise on commands like `auth profiles`, and it never named the skill that helps with the task at hand. MaybeHint now fires only on runnable `databricks air` subcommands and names the databricks-ai-runtime skill, what it covers, the install command, and a no-install preview (`aitools install --path`). Detection looks for that skill by exact name in the home stores and the project store (cwd up to the repository root), and counts the databricks plugin only when it applies to cwd. Bare `air` and `air --help` render help without the pre-run hook, so the air help footer now carries the same benefit line and the preview command. Telemetry keeps INSTALL_TEXT and the per-session throttle is unchanged. Co-authored-by: Isaac --- .nextchanges/cli/air-skill-hint.md | 1 + acceptance/air/help/output.txt | 4 + .../cmd/aitools/claude-config-dir/output.txt | 18 +- .../cmd/aitools/claude-config-dir/script | 10 +- .../telemetry/apps-init-tip/output.txt | 2 - .../cmd/aitools/telemetry/hint/output.txt | 38 ++- acceptance/cmd/aitools/telemetry/hint/script | 19 +- acceptance/cmd/version/output.txt | 3 +- acceptance/cmd/version/script | 5 +- cmd/air/air.go | 4 + cmd/air/air_test.go | 5 + cmd/air/hint_test.go | 89 ++++++ libs/aitools/agents/calling.go | 5 +- libs/aitools/agents/hint.go | 114 ++++--- libs/aitools/agents/hint_test.go | 299 +++++++++++++----- libs/aitools/agents/recommend.go | 4 +- 16 files changed, 452 insertions(+), 168 deletions(-) create mode 100644 .nextchanges/cli/air-skill-hint.md create mode 100644 cmd/air/hint_test.go diff --git a/.nextchanges/cli/air-skill-hint.md b/.nextchanges/cli/air-skill-hint.md new file mode 100644 index 00000000000..65eeb49e63b --- /dev/null +++ b/.nextchanges/cli/air-skill-hint.md @@ -0,0 +1 @@ +* Claude Code gets the Databricks skills install hint only on `databricks air` subcommands, where it names the `databricks-ai-runtime` skill, what it covers, and how to preview it with `databricks aitools install --path`; other commands no longer print the hint, and `databricks air --help` lists the preview too. diff --git a/acceptance/air/help/output.txt b/acceptance/air/help/output.txt index 7b37a4c83ac..f782fa5cf89 100644 --- a/acceptance/air/help/output.txt +++ b/acceptance/air/help/output.txt @@ -43,6 +43,10 @@ Learn more: Quickstart (AWS) https://docs.databricks.com/aws/en/machine-learning/ai-runtime/quickstart Quickstart (Azure) https://learn.microsoft.com/azure/databricks/machine-learning/ai-runtime/quickstart Coding agent skill databricks aitools install --skills-only --skills databricks-ai-runtime + Teaches coding agents the workload YAML, GPU types, secrets, + and how to follow and debug runs. + Preview the skill databricks aitools install --path --skills databricks-ai-runtime + Writes the skill files to without installing anything. === list help diff --git a/acceptance/cmd/aitools/claude-config-dir/output.txt b/acceptance/cmd/aitools/claude-config-dir/output.txt index 508651018c5..32646afffb2 100644 --- a/acceptance/cmd/aitools/claude-config-dir/output.txt +++ b/acceptance/cmd/aitools/claude-config-dir/output.txt @@ -1,21 +1,21 @@ === Plugin installed only in the custom config directory ->>> CLAUDE_CONFIG_DIR=custom [CLI] version -Databricks CLI v[CLI_VERSION] +>>> CLAUDE_CONFIG_DIR=custom [CLI] air list +No active runs found. Use --all-status to include runs that are no longer active. === Custom directory takes precedence over the default installation ->>> CLAUDE_CONFIG_DIR=missing [CLI] version -Databricks skills are not installed. To work with Databricks reliably, first run: databricks aitools install -Databricks CLI v[CLI_VERSION] +>>> CLAUDE_CONFIG_DIR=missing [CLI] air list +Tip: the databricks-ai-runtime skill is not installed. It documents the air workload YAML, GPU types, secrets, and how to follow and debug runs. Install it: databricks aitools install --skills-only --skills databricks-ai-runtime (to read it first without installing: databricks aitools install --path --skills databricks-ai-runtime) +No active runs found. Use --all-status to include runs that are no longer active. === Empty override uses the default installation ->>> CLAUDE_CONFIG_DIR= [CLI] version -Databricks CLI v[CLI_VERSION] +>>> CLAUDE_CONFIG_DIR= [CLI] air list +No active runs found. Use --all-status to include runs that are no longer active. === Unset override uses the default installation ->>> [CLI] version -Databricks CLI v[CLI_VERSION] +>>> [CLI] air list +No active runs found. Use --all-status to include runs that are no longer active. diff --git a/acceptance/cmd/aitools/claude-config-dir/script b/acceptance/cmd/aitools/claude-config-dir/script index 26f51369813..5ab0a15e0b3 100644 --- a/acceptance/cmd/aitools/claude-config-dir/script +++ b/acceptance/cmd/aitools/claude-config-dir/script @@ -1,18 +1,18 @@ export CLAUDECODE=1 unset CLAUDE_CODE_SESSION_ID -# aitools commands skip the install hint; version exercises it without auth or network. +# Only air subcommands print the install hint; air list against the mock server exercises it. sethome ./empty-home title "Plugin installed only in the custom config directory" -trace CLAUDE_CONFIG_DIR=custom $CLI version +trace CLAUDE_CONFIG_DIR=custom $CLI air list sethome ./installed-home title "Custom directory takes precedence over the default installation" -trace CLAUDE_CONFIG_DIR=missing $CLI version +trace CLAUDE_CONFIG_DIR=missing $CLI air list title "Empty override uses the default installation" -trace CLAUDE_CONFIG_DIR= $CLI version +trace CLAUDE_CONFIG_DIR= $CLI air list title "Unset override uses the default installation" unset CLAUDE_CONFIG_DIR -trace $CLI version +trace $CLI air list diff --git a/acceptance/cmd/aitools/telemetry/apps-init-tip/output.txt b/acceptance/cmd/aitools/telemetry/apps-init-tip/output.txt index 335595f4d75..74c334ebe1c 100644 --- a/acceptance/cmd/aitools/telemetry/apps-init-tip/output.txt +++ b/acceptance/cmd/aitools/telemetry/apps-init-tip/output.txt @@ -2,7 +2,6 @@ === apps init with flags prints the tip: one hint-shown event per hint printed >>> [CLI] apps init --template ./template --skip-install --name app-tip -Databricks skills are not installed. To work with Databricks reliably, first run: databricks aitools install ✔ Creating project ✔ Project created successfully! @@ -13,7 +12,6 @@ Databricks skills are not installed. To work with Databricks reliably, first run Tip: coding agents detected without Databricks skills. Run 'databricks aitools install' to install them. >>> print_telemetry -{"command":"apps_init","exit_code":0,"aitools_hint_event":{"hint_kind":"INSTALL_TEXT"}} {"command":"apps_init","exit_code":0,"aitools_hint_event":{"hint_kind":"APPS_INIT_TIP"}} === plugin installed: no hint, no tip, no event diff --git a/acceptance/cmd/aitools/telemetry/hint/output.txt b/acceptance/cmd/aitools/telemetry/hint/output.txt index 1283eb52109..c3831eaa393 100644 --- a/acceptance/cmd/aitools/telemetry/hint/output.txt +++ b/acceptance/cmd/aitools/telemetry/hint/output.txt @@ -1,22 +1,46 @@ -=== hint printed on an authenticated command: one hint-shown event is uploaded +=== hint printed on an air command: one hint-shown event is uploaded ->>> [CLI] jobs list -Databricks skills are not installed. To work with Databricks reliably, first run: databricks aitools install +>>> [CLI] air list +Tip: the databricks-ai-runtime skill is not installed. It documents the air workload YAML, GPU types, secrets, and how to follow and debug runs. Install it: databricks aitools install --skills-only --skills databricks-ai-runtime (to read it first without installing: databricks aitools install --path --skills databricks-ai-runtime) +No active runs found. Use --all-status to include runs that are no longer active. >>> print_telemetry -{"command":"jobs_list","exit_code":0,"aitools_hint_event":{"hint_kind":"INSTALL_TEXT"}} +{"command":"air_list","exit_code":0,"aitools_hint_event":{"hint_kind":"INSTALL_TEXT"}} + +=== commands outside air print no hint and log no event + +>>> [CLI] jobs list -=== hint printed on a command that resolves no auth: nothing is uploaded +>>> print_telemetry >>> [CLI] version -Databricks skills are not installed. To work with Databricks reliably, first run: databricks aitools install Databricks CLI v[CLI_VERSION] +=== previewing a skill with --path installs nothing, so air still hints + +>>> [CLI] aitools install --path ./preview --skills databricks-test +Using skills version test-ref +Wrote 1 skill to ./preview + >>> print_telemetry +{"command":"aitools_install","exit_code":0,"aitools_command_event":{"command":"INSTALL","stage":"STARTED","interactive":false,"install_mode":"PATH"}} +{"command":"aitools_install","exit_code":0,"aitools_command_event":{"command":"INSTALL","stage":"STEP_COMPLETED","step":"FLAGS_VALIDATED","interactive":false,"install_mode":"PATH"}} +{"command":"aitools_install","exit_code":0,"aitools_command_event":{"command":"INSTALL","stage":"STEP_COMPLETED","step":"SKILLS_DUMPED","interactive":false,"install_mode":"PATH"}} +{"command":"aitools_install","exit_code":0,"aitools_command_event":{"command":"INSTALL","stage":"FINISHED","outcome":"SUCCESS","interactive":false,"install_mode":"PATH"}} +preview/databricks-test/SKILL.md +no aitools state in HOME + +>>> [CLI] air list +Tip: the databricks-ai-runtime skill is not installed. It documents the air workload YAML, GPU types, secrets, and how to follow and debug runs. Install it: databricks aitools install --skills-only --skills databricks-ai-runtime (to read it first without installing: databricks aitools install --path --skills databricks-ai-runtime) +No active runs found. Use --all-status to include runs that are no longer active. + +>>> print_telemetry +{"command":"air_list","exit_code":0,"aitools_hint_event":{"hint_kind":"INSTALL_TEXT"}} === plugin installed: no hint, no event ->>> [CLI] jobs list +>>> [CLI] air list +No active runs found. Use --all-status to include runs that are no longer active. >>> print_telemetry diff --git a/acceptance/cmd/aitools/telemetry/hint/script b/acceptance/cmd/aitools/telemetry/hint/script index 1289a90535e..02c87f03d4a 100644 --- a/acceptance/cmd/aitools/telemetry/hint/script +++ b/acceptance/cmd/aitools/telemetry/hint/script @@ -5,15 +5,26 @@ unset CLAUDE_CODE_SESSION_ID # A host CLAUDE_CONFIG_DIR would move the plugin file away from the isolated HOME. unset CLAUDE_CONFIG_DIR -title "hint printed on an authenticated command: one hint-shown event is uploaded" -trace $CLI jobs list +title "hint printed on an air command: one hint-shown event is uploaded" +trace $CLI air list trace print_telemetry -title "hint printed on a command that resolves no auth: nothing is uploaded" +title "commands outside air print no hint and log no event" +trace $CLI jobs list +trace print_telemetry trace $CLI version + +title "previewing a skill with --path installs nothing, so air still hints" +# The mock manifest serves only databricks-test. +trace $CLI aitools install --path ./preview --skills databricks-test +trace print_telemetry +find preview -type f | sort +rm -rf preview +test -e "$HOME/.databricks/aitools" && echo "unexpected aitools state in HOME" || echo "no aitools state in HOME" +trace $CLI air list trace print_telemetry title "plugin installed: no hint, no event" sethome ./installed-home -trace $CLI jobs list +trace $CLI air list trace print_telemetry diff --git a/acceptance/cmd/version/output.txt b/acceptance/cmd/version/output.txt index 56a4d16ed95..cfb4047c4f5 100644 --- a/acceptance/cmd/version/output.txt +++ b/acceptance/cmd/version/output.txt @@ -35,9 +35,8 @@ Tip: coding agents detected without Databricks skills. Run 'databricks aitools i "development_build": true } -=== version --check from Claude Code prints only the root hint +=== version --check from Claude Code prints no hint >>> [CLI] version --check -Databricks skills are not installed. To work with Databricks reliably, first run: databricks aitools install Databricks CLI v[CLI_VERSION] This is a development build; skipping the update check. diff --git a/acceptance/cmd/version/script b/acceptance/cmd/version/script index 3e72d659289..84067aae808 100644 --- a/acceptance/cmd/version/script +++ b/acceptance/cmd/version/script @@ -23,8 +23,9 @@ trace $CLI version --check title "version --check --output json with a coding agent prints no tip" trace $CLI version --check --output json -# Claude Code callers get only the root hint (agents.MaybeHint), not the tip. -title "version --check from Claude Code prints only the root hint" +# Claude Code callers get no tip: agents.MaybeHint owns their hints and only +# hints on air subcommands. +title "version --check from Claude Code prints no hint" sethome ./claude-home export CLAUDECODE=1 unset CLAUDE_CODE_SESSION_ID diff --git a/cmd/air/air.go b/cmd/air/air.go index 4154bab891c..de1b6ac4ec6 100644 --- a/cmd/air/air.go +++ b/cmd/air/air.go @@ -21,6 +21,10 @@ Learn more: Quickstart (AWS) https://docs.databricks.com/aws/en/machine-learning/ai-runtime/quickstart Quickstart (Azure) https://learn.microsoft.com/azure/databricks/machine-learning/ai-runtime/quickstart Coding agent skill databricks aitools install --skills-only --skills databricks-ai-runtime + Teaches coding agents the workload YAML, GPU types, secrets, + and how to follow and debug runs. + Preview the skill databricks aitools install --path --skills databricks-ai-runtime + Writes the skill files to without installing anything. ` // New returns the root command for the AI Runtime CLI. diff --git a/cmd/air/air_test.go b/cmd/air/air_test.go index 11e4546d21d..b038e3de12c 100644 --- a/cmd/air/air_test.go +++ b/cmd/air/air_test.go @@ -54,6 +54,11 @@ func TestHelpFooterOnlyOnAirCommand(t *testing.T) { cmd.HelpFunc()(cmd, nil) assert.Contains(t, airOut.String(), "Learn more:") assert.Contains(t, airOut.String(), "databricks air run -h config") + // The footer is where `air` and `air --help` recommend the agent skill + // (help never reaches the pre-run hook that prints the install hint). + assert.Contains(t, airOut.String(), "databricks aitools install --skills-only --skills databricks-ai-runtime") + assert.Contains(t, airOut.String(), "Teaches coding agents the workload YAML") + assert.Contains(t, airOut.String(), "databricks aitools install --path --skills databricks-ai-runtime") list := findSubcommand(t, cmd, "list") var listOut bytes.Buffer diff --git a/cmd/air/hint_test.go b/cmd/air/hint_test.go new file mode 100644 index 00000000000..cb14b4e5329 --- /dev/null +++ b/cmd/air/hint_test.go @@ -0,0 +1,89 @@ +package aircmd + +import ( + "bytes" + "testing" + + "github.com/databricks/cli/internal/testutil" + "github.com/databricks/cli/libs/aitools/agents" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/env" + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// airUnderRoot mounts the real air tree under a root whose pre-run hook only +// records that it ran, mirroring where cmd/root calls agents.MaybeHint. +func airUnderRoot(preRunCalled *bool) (*cobra.Command, *cobra.Command) { + root := &cobra.Command{Use: "databricks"} + root.AddGroup(&cobra.Group{ID: "development", Title: "Developer Tools"}) + root.PersistentPreRunE = func(cmd *cobra.Command, args []string) error { + *preRunCalled = true + return nil + } + air := New() + root.AddCommand(air) + return root, air +} + +// TestBareAirAndHelpShowFooterWithoutPreRun pins why the footer, not the +// dynamic hint, is the recommendation surface for `air` and `air --help`. +func TestBareAirAndHelpShowFooterWithoutPreRun(t *testing.T) { + for _, args := range [][]string{{"air"}, {"air", "--help"}} { + t.Run(args[len(args)-1], func(t *testing.T) { + preRunCalled := false + root, _ := airUnderRoot(&preRunCalled) + var out bytes.Buffer + root.SetOut(&out) + root.SetErr(&out) + root.SetArgs(args) + require.NoError(t, root.Execute()) + assert.False(t, preRunCalled, "help must not run the pre-run hook") + assert.Contains(t, out.String(), "Coding agent skill databricks aitools install --skills-only --skills databricks-ai-runtime") + assert.Contains(t, out.String(), "Preview the skill databricks aitools install --path --skills databricks-ai-runtime") + }) + } +} + +// TestRunnableAirCommandsGetSkillHint walks the real air tree, so a new +// subcommand is covered without listing it here. +func TestRunnableAirCommandsGetSkillHint(t *testing.T) { + testutil.SetAgentEnv(t, "CLAUDECODE", "1") + preRunCalled := false + _, air := airUnderRoot(&preRunCalled) + assert.False(t, air.Runnable(), "bare air renders help; the footer covers it") + + var runnable []string + var walk func(c *cobra.Command) + walk = func(c *cobra.Command) { + for _, sub := range c.Commands() { + if sub.Runnable() { + runnable = append(runnable, sub.CommandPath()) + } + walk(sub) + } + } + walk(air) + for _, name := range []string{"run", "get", "list", "logs", "cancel", "convert-to-dabs"} { + assert.Contains(t, runnable, "databricks air "+name) + } + + var visit func(c *cobra.Command) + visit = func(c *cobra.Command) { + for _, sub := range c.Commands() { + if sub.Runnable() { + t.Run(sub.CommandPath(), func(t *testing.T) { + t.Chdir(t.TempDir()) + ctx, stderr := cmdio.NewTestContextWithStderr(t.Context()) + ctx = env.WithUserHomeDir(ctx, t.TempDir()) + ctx = env.Set(ctx, "DATABRICKS_CACHE_DIR", t.TempDir()) + agents.MaybeHint(ctx, sub) + assert.Contains(t, stderr.String(), "the databricks-ai-runtime skill is not installed") + }) + } + visit(sub) + } + } + visit(air) +} diff --git a/libs/aitools/agents/calling.go b/libs/aitools/agents/calling.go index 6bb35c3e14b..0cff90deb85 100644 --- a/libs/aitools/agents/calling.go +++ b/libs/aitools/agents/calling.go @@ -62,8 +62,9 @@ func (t Tooling) String() string { // record. For Codex and Copilot, pluginRecorded (the CLI install state) is the // only plugin source, so it misses plugins installed with the agent's own CLI // and includes plugins later removed with it. The shared skills store is not consulted: it cannot attribute skills. -// The install hint's claudeToolingInstalled differs on purpose: it stays quiet -// for any trace of an install, while this reports what the agent loads. +// The install hint's airSkillAvailable differs on purpose: it looks for the AI +// Runtime skill only, in the shared store too, while this reports what the +// agent loads. func (a *Agent) DatabricksTooling(ctx context.Context, pluginRecorded func() bool) Tooling { // An unknown working directory skips the project lookups below. cwd := "" diff --git a/libs/aitools/agents/hint.go b/libs/aitools/agents/hint.go index b37bcd6060c..3c3d6fd3104 100644 --- a/libs/aitools/agents/hint.go +++ b/libs/aitools/agents/hint.go @@ -2,6 +2,8 @@ package agents import ( "context" + "os" + "path/filepath" "time" "github.com/databricks/cli/libs/cache" @@ -15,10 +17,16 @@ import ( ) const ( - // hintText recommends installing the Databricks AI tooling. The wording - // is deliberately chosen to be effective at helping agents; change it only - // after thorough testing. - hintText = "Databricks skills are not installed. To work with Databricks reliably, first run: databricks aitools install" + // airSkillName is the agent skill for `databricks air`. + airSkillName = "databricks-ai-runtime" + + // airHintText recommends the AI Runtime skill on the commands it helps with. + // It names the skill and what it buys the agent, so a coding agent can tell + // the hint is about the task at hand rather than generic setup, and offers a + // --path preview so the agent can read the skill without installing anything. + airHintText = "Tip: the " + airSkillName + " skill is not installed. It documents the air workload YAML, GPU types, secrets, and how to follow and debug runs. " + + "Install it: databricks aitools install --skills-only --skills " + airSkillName + + " (to read it first without installing: databricks aitools install --path --skills " + airSkillName + ")" // claudeAgentProvider is the useragent.AgentProvider() value for Claude Code. // The SDK maps the CLAUDECODE env var to this product name. The hint is @@ -26,10 +34,10 @@ const ( // https://github.com/databricks/databricks-sdk-go/blob/main/useragent/agent.go claudeAgentProvider = "claude-code" - // aiToolsCommandName is the top-level command whose subtree is never hinted: - // telling the agent to run `databricks aitools install` while it is already - // running an aitools command makes no sense. - aiToolsCommandName = "aitools" + // airCommandName is the top-level command whose runnable subcommands get the + // hint. Bare `databricks air` and `--help` render help without running the + // pre-run hook; the air help footer carries the same recommendation there. + airCommandName = "air" // claudeSessionEnv is Claude Code's per-session identifier. It keys the // per-session throttle (at most one hint per hintSessionTTL per session); @@ -46,18 +54,20 @@ const ( // fire again for the same session. const hintSessionTTL = time.Hour -// MaybeHint prints a one-line recommendation to install the Databricks AI -// tooling when Claude Code drives the CLI without it installed. It is a no-op -// for human callers, other agents, aitools commands themselves, when there is -// no output stream, once the tooling is present, and when the session was -// hinted within the last hintSessionTTL. Best-effort: it never fails the -// command. Besides the stderr line, it logs a telemetry event for each hint -// shown, which makes a command that resolved auth upload telemetry at exit. +// MaybeHint prints a one-line recommendation to install the AI Runtime skill +// when Claude Code runs a `databricks air` subcommand without the skill +// available. Other commands are never hinted: an agent running them has no AI +// Runtime task to apply the skill to, so the line is noise it learns to skip. +// It is also a no-op for human callers, other agents, when there is no output +// stream, and when the session was hinted within the last hintSessionTTL. +// Best-effort: it never fails the command. Besides the stderr line, it logs a +// telemetry event for each hint shown, which makes a command that resolved +// auth upload telemetry at exit. func MaybeHint(ctx context.Context, cmd *cobra.Command) { if useragent.AgentProvider() != claudeAgentProvider { return } - if isAIToolsCommand(cmd) { + if !isAIRSubcommand(cmd) { return } if !cmdio.HasIO(ctx) { @@ -70,50 +80,62 @@ func MaybeHint(ctx context.Context, cmd *cobra.Command) { if alreadyHintedThisSession(ctx) { return } - if claudeToolingInstalled(ctx) { + if airSkillAvailable(ctx) { return } - cmdio.LogString(ctx, hintText) + cmdio.LogString(ctx, airHintText) logHintShown(ctx, protos.AitoolsHintKindInstallText) recordSessionHint(ctx) } -// isAIToolsCommand reports whether cmd is the aitools command or one of its -// subcommands, walking the parent chain (mirrors versioncheck.isExemptCommand). -func isAIToolsCommand(cmd *cobra.Command) bool { - for c := cmd; c != nil; c = c.Parent() { - if c.Name() == aiToolsCommandName { - return true +// isAIRSubcommand reports whether cmd sits under the top-level air command +// (`databricks air list`, `databricks air pools list`, ...), excluding air +// itself. Matching on the top-level ancestor keeps hidden aliases such as +// `databricks experimental air` out. +func isAIRSubcommand(cmd *cobra.Command) bool { + for c := cmd; c.HasParent(); c = c.Parent() { + if !c.Parent().HasParent() { + return c != cmd && c.Name() == airCommandName } } return false } -// claudeToolingInstalled reports whether Databricks AI tooling is present for -// Claude Code. It checks two signals: the databricks plugin in Claude's own -// plugin manifest (written both by `databricks aitools install` and by a direct -// `claude plugin install`), and databricks skill files in the shared canonical -// skills dir (written by `aitools install --skills-only`). Any read/parse -// failure reports false, so an unreadable state errs toward hinting. -// (*Agent).DatabricksTooling, which feeds the User-Agent, differs on purpose: -// it ignores the shared store and out-of-project records, and counts -// .claude/skills dirs, to report what the calling agent loads. -func claudeToolingInstalled(ctx context.Context) bool { - claude := ByName(NameClaudeCode) - if claude == nil { - // claude-code is always in the registry; this guard keeps the hint - // best-effort rather than panicking in a persistent pre-run hook. - return false +// airSkillAvailable reports whether Claude Code has the AI Runtime skill in the +// current working directory: the databricks plugin (which bundles every +// Databricks skill) installed for every project or for this one, or a +// databricks-ai-runtime skill dir in the shared canonical or legacy store or in +// the project store of cwd or a parent up to the repository root. Other +// Databricks skills do not count: they don't cover air. Any read/parse failure +// reports false, so an unreadable state errs toward hinting. +func airSkillAvailable(ctx context.Context) bool { + cwd := "" + if wd, err := os.Getwd(); err == nil { + cwd = resolvePath(wd) } - if version, ok := claude.DatabricksPluginVersion(ctx); ok { - log.Debugf(ctx, "Databricks plugin detected for Claude Code (version %q); skipping install hint", version) - return true + if claude := ByName(NameClaudeCode); claude != nil && claude.pluginInstalls != nil { + if active, _ := claude.DatabricksPluginActiveIn(ctx, cwd); active { + log.Debugf(ctx, "Databricks plugin active for Claude Code in %q; skipping AI Runtime skill hint", cwd) + return true + } } - // The canonical dir always holds a real databricks* subdir after a skills - // install, regardless of whether the agent copy is a symlink or a copy. The - // legacy dir is checked too so users of an older install aren't hinted. - return hasSharedSkills(ctx) + var dirs []string + if homeDir, err := env.UserHomeDir(ctx); err == nil { + dirs = append(dirs, filepath.Join(homeDir, CanonicalSkillsDir), filepath.Join(homeDir, LegacySkillsDir)) + } + if cwd != "" { + for _, dir := range projectDirs(cwd) { + dirs = append(dirs, filepath.Join(dir, CanonicalSkillsDir)) + } + } + for _, dir := range dirs { + // Stat follows the symlinks the installer may leave in place of copies. + if info, err := os.Stat(filepath.Join(dir, airSkillName)); err == nil && info.IsDir() { + return true + } + } + return false } // alreadyHintedThisSession reports whether this Claude session was hinted diff --git a/libs/aitools/agents/hint_test.go b/libs/aitools/agents/hint_test.go index 3add0417795..3f77fdc0294 100644 --- a/libs/aitools/agents/hint_test.go +++ b/libs/aitools/agents/hint_test.go @@ -6,6 +6,7 @@ import ( "log/slog" "os" "path/filepath" + "strings" "testing" "github.com/databricks/cli/internal/testutil" @@ -26,17 +27,32 @@ func setClaudeAgent(t *testing.T) { testutil.SetAgentEnv(t, "CLAUDECODE", "1") } -// claudeCtx returns a context with a captured stderr buffer and an isolated -// HOME + cache dir, so detection and the session throttle read only the test's -// temp state. home is returned so the test can drop fixtures under it. +// claudeCtx returns a context with a captured, non-TTY stderr buffer (what a +// coding agent's shell tool gives the CLI) and an isolated HOME, cache dir and +// working directory, so detection and the session throttle read only the +// test's temp state. home is returned so the test can drop fixtures under it. func claudeCtx(t *testing.T, home string) (context.Context, *bytes.Buffer) { + t.Chdir(t.TempDir()) ctx, stderr := cmdio.NewTestContextWithStderr(t.Context()) ctx = env.WithUserHomeDir(ctx, home) ctx = env.Set(ctx, "DATABRICKS_CACHE_DIR", t.TempDir()) - return ctx, stderr + return telemetry.WithNewLogger(ctx), stderr } -func regularCmd() *cobra.Command { return &cobra.Command{Use: "current-user"} } +// commandAt builds the command tree for a space-separated path under a +// "databricks" root and returns the leaf, so ancestry checks see the same +// shape as the real CLI. +func commandAt(path string) *cobra.Command { + cmd := &cobra.Command{Use: "databricks"} + for name := range strings.FieldsSeq(path) { + child := &cobra.Command{Use: name} + cmd.AddCommand(child) + cmd = child + } + return cmd +} + +func airCmd() *cobra.Command { return commandAt("air list") } func writePluginManifest(t *testing.T, home, body string) { dir := filepath.Join(home, ".claude", "plugins") @@ -59,41 +75,88 @@ func hintKinds(t *testing.T, ctx context.Context) []protos.AitoolsHintKind { return out } +// assertHinted checks for exactly one hint line and one hint-shown event of +// kind INSTALL_TEXT (T8). +func assertHinted(t *testing.T, ctx context.Context, stderr *bytes.Buffer) { + t.Helper() + assert.Equal(t, airHintText+"\n", stderr.String()) + assert.Equal(t, []protos.AitoolsHintKind{protos.AitoolsHintKindInstallText}, hintKinds(t, ctx)) +} + +// assertSilent checks for no hint line and no telemetry event (T8). +func assertSilent(t *testing.T, ctx context.Context, stderr *bytes.Buffer) { + t.Helper() + assert.Empty(t, stderr.String()) + assert.Empty(t, hintKinds(t, ctx)) +} + +func TestAIRHintText(t *testing.T) { + // The hint names the skill, says what it is for, and offers both the install + // and a no-install preview, on one line. + assert.Contains(t, airHintText, "databricks-ai-runtime skill is not installed") + assert.Contains(t, airHintText, "workload YAML") + assert.Contains(t, airHintText, "databricks aitools install --skills-only --skills databricks-ai-runtime") + assert.Contains(t, airHintText, "databricks aitools install --path --skills databricks-ai-runtime") + assert.NotContains(t, airHintText, "\n") +} + func TestMaybeHint_HumanCallerSilent(t *testing.T) { // No agent env: AgentProvider() == "", so no hint even with nothing installed. testutil.SetAgentEnv(t) - ctx, stderr := cmdio.NewTestContextWithStderr(t.Context()) - ctx = env.WithUserHomeDir(ctx, t.TempDir()) - ctx = telemetry.WithNewLogger(ctx) - MaybeHint(ctx, regularCmd()) - assert.Empty(t, stderr.String()) - assert.Empty(t, hintKinds(t, ctx)) + ctx, stderr := claudeCtx(t, t.TempDir()) + MaybeHint(ctx, airCmd()) + assertSilent(t, ctx, stderr) } -func TestMaybeHint_ClaudeNotInstalledHints(t *testing.T) { +// T1: every air subcommand, at any depth, gets the skill-specific hint. +func TestMaybeHint_AIRSubcommandsHint(t *testing.T) { setClaudeAgent(t) - ctx, stderr := claudeCtx(t, t.TempDir()) - ctx = telemetry.WithNewLogger(ctx) - MaybeHint(ctx, regularCmd()) - assert.Contains(t, stderr.String(), hintText) - assert.Equal(t, []protos.AitoolsHintKind{protos.AitoolsHintKindInstallText}, hintKinds(t, ctx)) + for _, path := range []string{ + "air run", "air get", "air list", "air logs", "air cancel", + "air convert-to-dabs", "air images", "air images push", "air pools", "air pools list", + } { + t.Run(path, func(t *testing.T) { + ctx, stderr := claudeCtx(t, t.TempDir()) + MaybeHint(ctx, commandAt(path)) + assertHinted(t, ctx, stderr) + }) + } +} + +// T3: commands outside air are never hinted, including the auth and version +// commands the generic hint used to fire on. +func TestMaybeHint_NonAIRCommandsSilent(t *testing.T) { + setClaudeAgent(t) + for _, path := range []string{ + "auth profiles", "current-user me", "clusters list", "jobs list", "bundle deploy", + "workspace list", "version", "aitools install", "experimental aitools install", + // The bare air node only renders help; its footer carries the tip. + "air", + // The hidden alias that points users at `databricks air`. + "experimental air", + } { + t.Run(path, func(t *testing.T) { + ctx, stderr := claudeCtx(t, t.TempDir()) + MaybeHint(ctx, commandAt(path)) + assertSilent(t, ctx, stderr) + }) + } } func TestMaybeHint_PluginInstalledSilent(t *testing.T) { setClaudeAgent(t) home := t.TempDir() ctx, stderr := claudeCtx(t, home) - // The core regression: a plugin install writes only Claude's manifest, no - // skills subdir. Detection must treat it as installed. + // A plugin install writes only Claude's manifest, no skills subdir, and + // bundles every Databricks skill including databricks-ai-runtime. writePluginManifest(t, home, `{"plugins":{"databricks@claude-plugins-official":[{"scope":"user"}]}}`) - ctx = telemetry.WithNewLogger(ctx) - MaybeHint(ctx, regularCmd()) - assert.Empty(t, stderr.String()) - assert.Empty(t, hintKinds(t, ctx)) + MaybeHint(ctx, airCmd()) + assertSilent(t, ctx, stderr) } -func TestClaudeToolingInstalled_LogsPluginVersion(t *testing.T) { +func TestAIRSkillAvailable_LogsPluginDetection(t *testing.T) { + t.Chdir(t.TempDir()) home := t.TempDir() writePluginManifest(t, home, `{"plugins":{"databricks@claude-plugins-official":[{"version":"0.2.6"}]}}`) @@ -103,14 +166,9 @@ func TestClaudeToolingInstalled_LogsPluginVersion(t *testing.T) { ctx := log.NewContext(t.Context(), slog.New(handler)) ctx = env.WithUserHomeDir(ctx, home) - assert.True(t, claudeToolingInstalled(ctx)) - - // The text handler escapes the inner quotes around the version, so assert on - // the stable, unescaped fragments rather than the full quoted message. - out := buf.String() - assert.Contains(t, out, "Databricks plugin detected for Claude Code") - assert.Contains(t, out, "0.2.6") - assert.Contains(t, out, "skipping install hint") + assert.True(t, airSkillAvailable(ctx)) + assert.Contains(t, buf.String(), "Databricks plugin active for Claude Code") + assert.Contains(t, buf.String(), "skipping AI Runtime skill hint") } func TestMaybeHint_CustomClaudeConfigDir(t *testing.T) { @@ -139,58 +197,90 @@ func TestMaybeHint_CustomClaudeConfigDir(t *testing.T) { writePluginManifest(t, home, manifest) } - MaybeHint(ctx, regularCmd()) + MaybeHint(ctx, airCmd()) if tc.wantHint { - assert.Contains(t, stderr.String(), hintText) + assertHinted(t, ctx, stderr) } else { - assert.Empty(t, stderr.String()) + assertSilent(t, ctx, stderr) } }) } } -func TestMaybeHint_SkillsInstalledSilent(t *testing.T) { - setClaudeAgent(t) - home := t.TempDir() - ctx, stderr := claudeCtx(t, home) - writeCanonicalSkill(t, home, "databricks") - - MaybeHint(ctx, regularCmd()) - assert.Empty(t, stderr.String()) -} - -func TestMaybeHint_LegacySkillsInstalledSilent(t *testing.T) { +// T4: only the databricks-ai-runtime skill itself, in a location Claude loads +// for this working directory, silences the hint. +func TestMaybeHint_AIRSkillDetection(t *testing.T) { setClaudeAgent(t) - home := t.TempDir() - ctx, stderr := claudeCtx(t, home) - require.NoError(t, os.MkdirAll(filepath.Join(home, ".databricks", "agent-skills", "databricks"), 0o755)) + for _, tc := range []struct { + name string + setup func(t *testing.T, home, cwd string) + wantHint bool + }{ + {"nothing installed", func(t *testing.T, home, cwd string) {}, true}, + {"canonical store", func(t *testing.T, home, cwd string) { + writeCanonicalSkill(t, home, airSkillName) + }, false}, + {"legacy store", func(t *testing.T, home, cwd string) { + require.NoError(t, os.MkdirAll(filepath.Join(home, LegacySkillsDir, airSkillName), 0o755)) + }, false}, + {"canonical store via symlink", func(t *testing.T, home, cwd string) { + dir := filepath.Join(home, CanonicalSkillsDir) + require.NoError(t, os.MkdirAll(dir, 0o755)) + require.NoError(t, os.Symlink(t.TempDir(), filepath.Join(dir, airSkillName))) + }, false}, + {"project store in cwd", func(t *testing.T, home, cwd string) { + require.NoError(t, os.MkdirAll(filepath.Join(cwd, CanonicalSkillsDir, airSkillName), 0o755)) + }, false}, + {"other databricks skills only", func(t *testing.T, home, cwd string) { + writeCanonicalSkill(t, home, "databricks") + writeCanonicalSkill(t, home, "databricks-jobs") + require.NoError(t, os.MkdirAll(filepath.Join(cwd, CanonicalSkillsDir, "databricks-apps"), 0o755)) + }, true}, + {"a file named like the skill", func(t *testing.T, home, cwd string) { + dir := filepath.Join(home, CanonicalSkillsDir) + require.NoError(t, os.MkdirAll(dir, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, airSkillName), nil, 0o600)) + }, true}, + {"plugin for this project", func(t *testing.T, home, cwd string) { + writePluginManifest(t, home, `{"plugins":{"databricks@m":[{"scope":"project","projectPath":"`+cwd+`"}]}}`) + }, false}, + {"plugin for an unrelated project", func(t *testing.T, home, cwd string) { + writePluginManifest(t, home, `{"plugins":{"databricks@m":[{"scope":"project","projectPath":"`+t.TempDir()+`"}]}}`) + }, true}, + } { + t.Run(tc.name, func(t *testing.T) { + home := t.TempDir() + ctx, stderr := claudeCtx(t, home) + cwd, err := os.Getwd() + require.NoError(t, err) + tc.setup(t, home, cwd) - MaybeHint(ctx, regularCmd()) - assert.Empty(t, stderr.String()) + MaybeHint(ctx, airCmd()) + if tc.wantHint { + assertHinted(t, ctx, stderr) + } else { + assertSilent(t, ctx, stderr) + } + }) + } } -func TestMaybeHint_NonDatabricksSkillStillHints(t *testing.T) { +// T4: a project-scoped skill at the repository root still counts when the CLI +// runs in a subdirectory, as Claude Code loads it there. +func TestMaybeHint_AIRSkillInRepositoryRoot(t *testing.T) { setClaudeAgent(t) home := t.TempDir() ctx, stderr := claudeCtx(t, home) - writeCanonicalSkill(t, home, "some-other-skill") - - MaybeHint(ctx, regularCmd()) - assert.Contains(t, stderr.String(), hintText) -} - -func TestMaybeHint_AIToolsCommandSilent(t *testing.T) { - setClaudeAgent(t) - ctx, stderr := claudeCtx(t, t.TempDir()) - ctx = telemetry.WithNewLogger(ctx) - - parent := &cobra.Command{Use: aiToolsCommandName} - child := &cobra.Command{Use: "install"} - parent.AddCommand(child) + root, err := os.Getwd() + require.NoError(t, err) + require.NoError(t, os.Mkdir(filepath.Join(root, ".git"), 0o755)) + require.NoError(t, os.MkdirAll(filepath.Join(root, CanonicalSkillsDir, airSkillName), 0o755)) + sub := filepath.Join(root, "training", "configs") + require.NoError(t, os.MkdirAll(sub, 0o755)) + t.Chdir(sub) - MaybeHint(ctx, child) - assert.Empty(t, stderr.String()) - assert.Empty(t, hintKinds(t, ctx)) + MaybeHint(ctx, airCmd()) + assertSilent(t, ctx, stderr) } func TestMaybeHint_NoIONoPanic(t *testing.T) { @@ -199,30 +289,65 @@ func TestMaybeHint_NoIONoPanic(t *testing.T) { // invocations before PersistentPreRunE runs, so HasIO can be false here. ctx := env.WithUserHomeDir(t.Context(), t.TempDir()) ctx = telemetry.WithNewLogger(ctx) - assert.NotPanics(t, func() { MaybeHint(ctx, regularCmd()) }) + assert.NotPanics(t, func() { MaybeHint(ctx, airCmd()) }) assert.Empty(t, hintKinds(t, ctx)) } -func TestMaybeHint_ThrottledPerSession(t *testing.T) { +// T5 and T6: with a session id the first air command in the session is +// hinted and later ones stay quiet until the marker expires; without one +// (older Claude Code) every air command is hinted. Both cases run with a +// non-TTY stderr, which is how a coding agent's shell tool runs the CLI. +func TestMaybeHint_SessionThrottle(t *testing.T) { + setClaudeAgent(t) + for _, tc := range []struct { + name string + session string + wantSecond bool + }{ + {"session id: first air command only", "session-abc", false}, + {"no session id: every air command", "", true}, + } { + t.Run(tc.name, func(t *testing.T) { + home := t.TempDir() + cacheDir := t.TempDir() + newCtx := func() (context.Context, *bytes.Buffer) { + ctx, stderr := claudeCtx(t, home) + ctx = env.Set(ctx, "DATABRICKS_CACHE_DIR", cacheDir) + if tc.session != "" { + ctx = env.Set(ctx, claudeSessionEnv, tc.session) + } + return ctx, stderr + } + + // A non-air command first must not use up the session's hint. + ctx0, stderr0 := newCtx() + MaybeHint(ctx0, commandAt("auth profiles")) + assertSilent(t, ctx0, stderr0) + + ctx1, stderr1 := newCtx() + MaybeHint(ctx1, commandAt("air list")) + assertHinted(t, ctx1, stderr1) + + ctx2, stderr2 := newCtx() + MaybeHint(ctx2, commandAt("air run")) + if tc.wantSecond { + assertHinted(t, ctx2, stderr2) + } else { + assertSilent(t, ctx2, stderr2) + } + }) + } +} + +func TestMaybeHint_OtherSessionStillHinted(t *testing.T) { setClaudeAgent(t) home := t.TempDir() cacheDir := t.TempDir() - newCtx := func() (context.Context, *bytes.Buffer) { - ctx, stderr := cmdio.NewTestContextWithStderr(t.Context()) - ctx = env.WithUserHomeDir(ctx, home) + for _, session := range []string{"session-a", "session-b"} { + ctx, stderr := claudeCtx(t, home) ctx = env.Set(ctx, "DATABRICKS_CACHE_DIR", cacheDir) - ctx = env.Set(ctx, claudeSessionEnv, "session-abc") - return telemetry.WithNewLogger(ctx), stderr + ctx = env.Set(ctx, claudeSessionEnv, session) + MaybeHint(ctx, airCmd()) + assertHinted(t, ctx, stderr) } - - ctx1, stderr1 := newCtx() - MaybeHint(ctx1, regularCmd()) - assert.Contains(t, stderr1.String(), hintText) - assert.Equal(t, []protos.AitoolsHintKind{protos.AitoolsHintKindInstallText}, hintKinds(t, ctx1)) - - // Same session id, second command: must stay quiet and log nothing. - ctx2, stderr2 := newCtx() - MaybeHint(ctx2, regularCmd()) - assert.Empty(t, stderr2.String()) - assert.Empty(t, hintKinds(t, ctx2)) } diff --git a/libs/aitools/agents/recommend.go b/libs/aitools/agents/recommend.go index 14a4da5b00b..7e34de5b517 100644 --- a/libs/aitools/agents/recommend.go +++ b/libs/aitools/agents/recommend.go @@ -23,8 +23,8 @@ func PrintSkillsTip(ctx context.Context) { // MaybeVersionCheckTip prints skillsTipText after `databricks version --check` // when a coding agent is detected without Databricks tooling. Claude Code -// callers are skipped because MaybeHint already covers them, with its -// per-session throttle. installedPlugins is a parameter because this package +// callers are skipped because MaybeHint owns their hints, and it recommends a +// skill only on the commands that skill helps with. installedPlugins is a parameter because this package // cannot import the installer; callers pass installer.InstalledPluginAgents, // which counts a plugin recorded in the CLI install state with or without a // known version. No telemetry is logged: `databricks version` never resolves From 2f16610aaa3290677302e235b091e2bac9952be4 Mon Sep 17 00:00:00 2001 From: Artemiy Kolesnikov Date: Fri, 13 Nov 2026 23:48:32 +0000 Subject: [PATCH 2/5] aitools: link the air skill hint changelog fragment to the pull request Co-authored-by: Isaac --- .nextchanges/cli/air-skill-hint.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nextchanges/cli/air-skill-hint.md b/.nextchanges/cli/air-skill-hint.md index 65eeb49e63b..b36abd2362b 100644 --- a/.nextchanges/cli/air-skill-hint.md +++ b/.nextchanges/cli/air-skill-hint.md @@ -1 +1 @@ -* Claude Code gets the Databricks skills install hint only on `databricks air` subcommands, where it names the `databricks-ai-runtime` skill, what it covers, and how to preview it with `databricks aitools install --path`; other commands no longer print the hint, and `databricks air --help` lists the preview too. +* Claude Code gets the Databricks skills install hint only on `databricks air` subcommands, where it names the `databricks-ai-runtime` skill, what it covers, and how to preview it with `databricks aitools install --path`; other commands no longer print the hint, and `databricks air --help` lists the preview too. ([#7622](https://github.com/databricks/cli/pull/7622)) From f38ee96c0111dd6eb74e9feaee29fddb93d8332c Mon Sep 17 00:00:00 2001 From: Artemiy Kolesnikov Date: Sat, 14 Nov 2026 00:09:28 +0000 Subject: [PATCH 3/5] aitools: escape project paths in the hint test plugin manifest Windows paths contain backslashes, so interpolating them into the manifest string produced invalid JSON escapes and the plugin record was silently dropped, hinting when the test expected silence. Marshal the manifest instead. Co-authored-by: Isaac --- libs/aitools/agents/hint_test.go | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/libs/aitools/agents/hint_test.go b/libs/aitools/agents/hint_test.go index 3f77fdc0294..2de2e7d224f 100644 --- a/libs/aitools/agents/hint_test.go +++ b/libs/aitools/agents/hint_test.go @@ -3,6 +3,7 @@ package agents import ( "bytes" "context" + "encoding/json" "log/slog" "os" "path/filepath" @@ -60,6 +61,22 @@ func writePluginManifest(t *testing.T, home, body string) { require.NoError(t, os.WriteFile(filepath.Join(dir, "installed_plugins.json"), []byte(body), 0o600)) } +// writeProjectPluginManifest writes a manifest with one project-scope install +// covering projectPath. Marshaled, not interpolated, so Windows paths with +// backslashes stay valid JSON. +func writeProjectPluginManifest(t *testing.T, home, projectPath string) { + manifest, err := json.Marshal(map[string]any{ + "plugins": map[string]any{ + "databricks@m": []any{map[string]string{ + "scope": "project", + "projectPath": projectPath, + }}, + }, + }) + require.NoError(t, err) + writePluginManifest(t, home, string(manifest)) +} + func writeCanonicalSkill(t *testing.T, home, name string) { require.NoError(t, os.MkdirAll(filepath.Join(home, ".databricks", "aitools", "skills", name), 0o755)) } @@ -242,10 +259,10 @@ func TestMaybeHint_AIRSkillDetection(t *testing.T) { require.NoError(t, os.WriteFile(filepath.Join(dir, airSkillName), nil, 0o600)) }, true}, {"plugin for this project", func(t *testing.T, home, cwd string) { - writePluginManifest(t, home, `{"plugins":{"databricks@m":[{"scope":"project","projectPath":"`+cwd+`"}]}}`) + writeProjectPluginManifest(t, home, cwd) }, false}, {"plugin for an unrelated project", func(t *testing.T, home, cwd string) { - writePluginManifest(t, home, `{"plugins":{"databricks@m":[{"scope":"project","projectPath":"`+t.TempDir()+`"}]}}`) + writeProjectPluginManifest(t, home, t.TempDir()) }, true}, } { t.Run(tc.name, func(t *testing.T) { From 2548498a062891818d3f2fa6565956bcb6afe574 Mon Sep 17 00:00:00 2001 From: Artemiy Kolesnikov Date: Mon, 16 Nov 2026 18:36:34 +0000 Subject: [PATCH 4/5] aitools: tighten the air skill hint wording Use the reviewer-suggested wording: name the skill and its benefit up front, then the install command, then the --path preview. Co-authored-by: Isaac --- acceptance/cmd/aitools/claude-config-dir/output.txt | 2 +- acceptance/cmd/aitools/telemetry/hint/output.txt | 4 ++-- cmd/air/hint_test.go | 2 +- libs/aitools/agents/hint.go | 5 ++--- libs/aitools/agents/hint_test.go | 6 +++--- 5 files changed, 9 insertions(+), 10 deletions(-) diff --git a/acceptance/cmd/aitools/claude-config-dir/output.txt b/acceptance/cmd/aitools/claude-config-dir/output.txt index 32646afffb2..97eaecc1d0e 100644 --- a/acceptance/cmd/aitools/claude-config-dir/output.txt +++ b/acceptance/cmd/aitools/claude-config-dir/output.txt @@ -7,7 +7,7 @@ No active runs found. Use --all-status to include runs that are no longer active === Custom directory takes precedence over the default installation >>> CLAUDE_CONFIG_DIR=missing [CLI] air list -Tip: the databricks-ai-runtime skill is not installed. It documents the air workload YAML, GPU types, secrets, and how to follow and debug runs. Install it: databricks aitools install --skills-only --skills databricks-ai-runtime (to read it first without installing: databricks aitools install --path --skills databricks-ai-runtime) +Tip: install the databricks-ai-runtime skill for workload configuration and run debugging: databricks aitools install --skills-only --skills databricks-ai-runtime. To preview it first, replace --skills-only with --path . No active runs found. Use --all-status to include runs that are no longer active. === Empty override uses the default installation diff --git a/acceptance/cmd/aitools/telemetry/hint/output.txt b/acceptance/cmd/aitools/telemetry/hint/output.txt index c3831eaa393..79de0691711 100644 --- a/acceptance/cmd/aitools/telemetry/hint/output.txt +++ b/acceptance/cmd/aitools/telemetry/hint/output.txt @@ -2,7 +2,7 @@ === hint printed on an air command: one hint-shown event is uploaded >>> [CLI] air list -Tip: the databricks-ai-runtime skill is not installed. It documents the air workload YAML, GPU types, secrets, and how to follow and debug runs. Install it: databricks aitools install --skills-only --skills databricks-ai-runtime (to read it first without installing: databricks aitools install --path --skills databricks-ai-runtime) +Tip: install the databricks-ai-runtime skill for workload configuration and run debugging: databricks aitools install --skills-only --skills databricks-ai-runtime. To preview it first, replace --skills-only with --path . No active runs found. Use --all-status to include runs that are no longer active. >>> print_telemetry @@ -32,7 +32,7 @@ preview/databricks-test/SKILL.md no aitools state in HOME >>> [CLI] air list -Tip: the databricks-ai-runtime skill is not installed. It documents the air workload YAML, GPU types, secrets, and how to follow and debug runs. Install it: databricks aitools install --skills-only --skills databricks-ai-runtime (to read it first without installing: databricks aitools install --path --skills databricks-ai-runtime) +Tip: install the databricks-ai-runtime skill for workload configuration and run debugging: databricks aitools install --skills-only --skills databricks-ai-runtime. To preview it first, replace --skills-only with --path . No active runs found. Use --all-status to include runs that are no longer active. >>> print_telemetry diff --git a/cmd/air/hint_test.go b/cmd/air/hint_test.go index cb14b4e5329..b168b8c1520 100644 --- a/cmd/air/hint_test.go +++ b/cmd/air/hint_test.go @@ -79,7 +79,7 @@ func TestRunnableAirCommandsGetSkillHint(t *testing.T) { ctx = env.WithUserHomeDir(ctx, t.TempDir()) ctx = env.Set(ctx, "DATABRICKS_CACHE_DIR", t.TempDir()) agents.MaybeHint(ctx, sub) - assert.Contains(t, stderr.String(), "the databricks-ai-runtime skill is not installed") + assert.Contains(t, stderr.String(), "install the databricks-ai-runtime skill") }) } visit(sub) diff --git a/libs/aitools/agents/hint.go b/libs/aitools/agents/hint.go index 3c3d6fd3104..d710e39534e 100644 --- a/libs/aitools/agents/hint.go +++ b/libs/aitools/agents/hint.go @@ -24,9 +24,8 @@ const ( // It names the skill and what it buys the agent, so a coding agent can tell // the hint is about the task at hand rather than generic setup, and offers a // --path preview so the agent can read the skill without installing anything. - airHintText = "Tip: the " + airSkillName + " skill is not installed. It documents the air workload YAML, GPU types, secrets, and how to follow and debug runs. " + - "Install it: databricks aitools install --skills-only --skills " + airSkillName + - " (to read it first without installing: databricks aitools install --path --skills " + airSkillName + ")" + airHintText = "Tip: install the " + airSkillName + " skill for workload configuration and run debugging: databricks aitools install --skills-only --skills " + airSkillName + + ". To preview it first, replace --skills-only with --path ." // claudeAgentProvider is the useragent.AgentProvider() value for Claude Code. // The SDK maps the CLAUDECODE env var to this product name. The hint is diff --git a/libs/aitools/agents/hint_test.go b/libs/aitools/agents/hint_test.go index 2de2e7d224f..e16b6bd205f 100644 --- a/libs/aitools/agents/hint_test.go +++ b/libs/aitools/agents/hint_test.go @@ -110,10 +110,10 @@ func assertSilent(t *testing.T, ctx context.Context, stderr *bytes.Buffer) { func TestAIRHintText(t *testing.T) { // The hint names the skill, says what it is for, and offers both the install // and a no-install preview, on one line. - assert.Contains(t, airHintText, "databricks-ai-runtime skill is not installed") - assert.Contains(t, airHintText, "workload YAML") + assert.Contains(t, airHintText, "install the databricks-ai-runtime skill") + assert.Contains(t, airHintText, "workload configuration and run debugging") assert.Contains(t, airHintText, "databricks aitools install --skills-only --skills databricks-ai-runtime") - assert.Contains(t, airHintText, "databricks aitools install --path --skills databricks-ai-runtime") + assert.Contains(t, airHintText, "replace --skills-only with --path ") assert.NotContains(t, airHintText, "\n") } From ee7cac290681fc1f79720833d58c32905d0da31d Mon Sep 17 00:00:00 2001 From: Artemiy Kolesnikov Date: Mon, 16 Nov 2026 21:55:18 +0000 Subject: [PATCH 5/5] aitools: keep the generic install hint outside air Narrow the change so non-air commands behave exactly as before: Claude Code keeps the generic install hint and the registry agents keep their per-agent install hints on every hinted command. Only on runnable 'databricks air' subcommands does Claude Code now get the AI Runtime skill recommendation instead of the generic hint, with detection keyed to exactly the databricks-ai-runtime skill in the locations Claude Code loads for the working directory. Co-authored-by: Isaac --- .nextchanges/cli/air-skill-hint.md | 2 +- acceptance/cmd/aitools/agent-hints/output.txt | 40 +- acceptance/cmd/aitools/agent-hints/script | 23 +- .../cmd/aitools/claude-config-dir/output.txt | 34 +- .../cmd/aitools/claude-config-dir/script | 17 +- .../telemetry/apps-init-tip/output.txt | 2 + .../cmd/aitools/telemetry/hint/output.txt | 43 +- acceptance/cmd/aitools/telemetry/hint/script | 19 +- acceptance/cmd/version/output.txt | 3 +- acceptance/cmd/version/script | 8 +- cmd/air/hint_test.go | 20 +- libs/aitools/agents/calling.go | 7 +- libs/aitools/agents/hint.go | 120 ++- libs/aitools/agents/hint_agents_test.go | 51 +- libs/aitools/agents/hint_test.go | 693 ++++++++++-------- libs/aitools/agents/recommend.go | 6 +- 16 files changed, 609 insertions(+), 479 deletions(-) diff --git a/.nextchanges/cli/air-skill-hint.md b/.nextchanges/cli/air-skill-hint.md index 9f792dae828..9636ef450a6 100644 --- a/.nextchanges/cli/air-skill-hint.md +++ b/.nextchanges/cli/air-skill-hint.md @@ -1 +1 @@ -* Coding agents get the Databricks skills install hint only on `databricks air` subcommands, where an agent has an AI Runtime task for it to help with. For Claude Code the hint names the `databricks-ai-runtime` skill, what it covers, and how to preview it with `databricks aitools install --path`; other commands no longer print the hint, and `databricks air --help` lists the preview too. ([#7622](https://github.com/databricks/cli/pull/7622)) +* On `databricks air` subcommands, Claude Code now gets an install hint that names the `databricks-ai-runtime` skill, what it covers, and how to preview it with `databricks aitools install --path`, instead of the generic install hint; other commands and other agents' hints are unchanged, and `databricks air --help` lists the skill and the preview too. ([#7622](https://github.com/databricks/cli/pull/7622)) diff --git a/acceptance/cmd/aitools/agent-hints/output.txt b/acceptance/cmd/aitools/agent-hints/output.txt index 758d545d972..ed22089e902 100644 --- a/acceptance/cmd/aitools/agent-hints/output.txt +++ b/acceptance/cmd/aitools/agent-hints/output.txt @@ -1,46 +1,46 @@ === Codex is hinted once per thread ->>> CODEX_CI=1 CODEX_THREAD_ID=thread-1 [CLI] air list +>>> CODEX_CI=1 CODEX_THREAD_ID=thread-1 [CLI] version Databricks skills are not installed. To work with Databricks reliably, first run: databricks aitools install --agents codex --scope global -No active runs found. Use --all-status to include runs that are no longer active. +Databricks CLI v[CLI_VERSION] ->>> CODEX_CI=1 CODEX_THREAD_ID=thread-1 [CLI] air list -No active runs found. Use --all-status to include runs that are no longer active. +>>> CODEX_CI=1 CODEX_THREAD_ID=thread-1 [CLI] version +Databricks CLI v[CLI_VERSION] ->>> CODEX_CI=1 CODEX_THREAD_ID=thread-2 [CLI] air list +>>> CODEX_CI=1 CODEX_THREAD_ID=thread-2 [CLI] version Databricks skills are not installed. To work with Databricks reliably, first run: databricks aitools install --agents codex --scope global -No active runs found. Use --all-status to include runs that are no longer active. +Databricks CLI v[CLI_VERSION] === Cursor exposes no session id, so it is hinted once per machine ->>> CURSOR_AGENT=1 [CLI] air list +>>> CURSOR_AGENT=1 [CLI] version Databricks skills are not installed. To work with Databricks reliably, first run: databricks aitools install --agents cursor --scope global -No active runs found. Use --all-status to include runs that are no longer active. +Databricks CLI v[CLI_VERSION] ->>> CURSOR_AGENT=1 [CLI] air list -No active runs found. Use --all-status to include runs that are no longer active. +>>> CURSOR_AGENT=1 [CLI] version +Databricks CLI v[CLI_VERSION] === Copilot CLI maps to the copilot registry agent ->>> COPILOT_CLI=1 [CLI] air list +>>> COPILOT_CLI=1 [CLI] version Databricks skills are not installed. To work with Databricks reliably, first run: databricks aitools install --agents copilot --scope global -No active runs found. Use --all-status to include runs that are no longer active. +Databricks CLI v[CLI_VERSION] === Gemini CLI with its own Databricks skills is not hinted ->>> GEMINI_CLI=1 [CLI] air list -No active runs found. Use --all-status to include runs that are no longer active. +>>> GEMINI_CLI=1 [CLI] version +Databricks CLI v[CLI_VERSION] === Agents the CLI cannot install for are not hinted ->>> WINDSURF_AGENT=1 [CLI] air list -No active runs found. Use --all-status to include runs that are no longer active. +>>> WINDSURF_AGENT=1 [CLI] version +Databricks CLI v[CLI_VERSION] ->>> VSCODE_AGENT=1 [CLI] air list -No active runs found. Use --all-status to include runs that are no longer active. +>>> VSCODE_AGENT=1 [CLI] version +Databricks CLI v[CLI_VERSION] === Nested agents are not hinted ->>> CODEX_CI=1 CURSOR_AGENT=1 CODEX_THREAD_ID=thread-3 [CLI] air list -No active runs found. Use --all-status to include runs that are no longer active. +>>> CODEX_CI=1 CURSOR_AGENT=1 CODEX_THREAD_ID=thread-3 [CLI] version +Databricks CLI v[CLI_VERSION] diff --git a/acceptance/cmd/aitools/agent-hints/script b/acceptance/cmd/aitools/agent-hints/script index 92c5b9ba11e..405596d991b 100644 --- a/acceptance/cmd/aitools/agent-hints/script +++ b/acceptance/cmd/aitools/agent-hints/script @@ -1,28 +1,27 @@ -# Only air subcommands print the install hint; air list against the mock -# server exercises it without real auth. +# aitools commands skip the install hint; version exercises it without auth or network. sethome ./home unset XDG_CONFIG_HOME GEMINI_CLI_HOME CODEX_THREAD_ID title "Codex is hinted once per thread" -trace CODEX_CI=1 CODEX_THREAD_ID=thread-1 $CLI air list -trace CODEX_CI=1 CODEX_THREAD_ID=thread-1 $CLI air list -trace CODEX_CI=1 CODEX_THREAD_ID=thread-2 $CLI air list +trace CODEX_CI=1 CODEX_THREAD_ID=thread-1 $CLI version +trace CODEX_CI=1 CODEX_THREAD_ID=thread-1 $CLI version +trace CODEX_CI=1 CODEX_THREAD_ID=thread-2 $CLI version title "Cursor exposes no session id, so it is hinted once per machine" -trace CURSOR_AGENT=1 $CLI air list -trace CURSOR_AGENT=1 $CLI air list +trace CURSOR_AGENT=1 $CLI version +trace CURSOR_AGENT=1 $CLI version title "Copilot CLI maps to the copilot registry agent" -trace COPILOT_CLI=1 $CLI air list +trace COPILOT_CLI=1 $CLI version title "Gemini CLI with its own Databricks skills is not hinted" sethome ./gemini-home -trace GEMINI_CLI=1 $CLI air list +trace GEMINI_CLI=1 $CLI version sethome ./home title "Agents the CLI cannot install for are not hinted" -trace WINDSURF_AGENT=1 $CLI air list -trace VSCODE_AGENT=1 $CLI air list +trace WINDSURF_AGENT=1 $CLI version +trace VSCODE_AGENT=1 $CLI version title "Nested agents are not hinted" -trace CODEX_CI=1 CURSOR_AGENT=1 CODEX_THREAD_ID=thread-3 $CLI air list +trace CODEX_CI=1 CURSOR_AGENT=1 CODEX_THREAD_ID=thread-3 $CLI version diff --git a/acceptance/cmd/aitools/claude-config-dir/output.txt b/acceptance/cmd/aitools/claude-config-dir/output.txt index 2e2033c8c7f..e36484dff36 100644 --- a/acceptance/cmd/aitools/claude-config-dir/output.txt +++ b/acceptance/cmd/aitools/claude-config-dir/output.txt @@ -1,38 +1,38 @@ === Plugin installed only in the custom config directory ->>> CLAUDE_CONFIG_DIR=custom [CLI] air list -No active runs found. Use --all-status to include runs that are no longer active. +>>> CLAUDE_CONFIG_DIR=custom [CLI] version +Databricks CLI v[CLI_VERSION] === Custom directory takes precedence over the default installation ->>> CLAUDE_CONFIG_DIR=missing [CLI] air list +>>> CLAUDE_CONFIG_DIR=missing [CLI] version -Tip: install the databricks-ai-runtime skill for workload configuration and run debugging: databricks aitools install --skills-only --skills databricks-ai-runtime. To preview it first, replace --skills-only with --path . -No active runs found. Use --all-status to include runs that are no longer active. +Databricks skills are not installed. To work with Databricks reliably, first run: databricks aitools install +Databricks CLI v[CLI_VERSION] === Empty override uses the default installation ->>> CLAUDE_CONFIG_DIR= [CLI] air list -No active runs found. Use --all-status to include runs that are no longer active. +>>> CLAUDE_CONFIG_DIR= [CLI] version +Databricks CLI v[CLI_VERSION] === Unset override uses the default installation ->>> [CLI] air list -No active runs found. Use --all-status to include runs that are no longer active. +>>> [CLI] version +Databricks CLI v[CLI_VERSION] === Outside a Claude Code subprocess only the text hint prints ->>> CLAUDE_CODE_CHILD_SESSION= [CLI] air list -Tip: install the databricks-ai-runtime skill for workload configuration and run debugging: databricks aitools install --skills-only --skills databricks-ai-runtime. To preview it first, replace --skills-only with --path . -No active runs found. Use --all-status to include runs that are no longer active. +>>> CLAUDE_CODE_CHILD_SESSION= [CLI] version +Databricks skills are not installed. To work with Databricks reliably, first run: databricks aitools install +Databricks CLI v[CLI_VERSION] === Within one session both lines print once, so 2>&1 pipelines see them only on the first command ->>> [CLI] air list +>>> [CLI] version -Tip: install the databricks-ai-runtime skill for workload configuration and run debugging: databricks aitools install --skills-only --skills databricks-ai-runtime. To preview it first, replace --skills-only with --path . -No active runs found. Use --all-status to include runs that are no longer active. +Databricks skills are not installed. To work with Databricks reliably, first run: databricks aitools install +Databricks CLI v[CLI_VERSION] ->>> [CLI] air list -No active runs found. Use --all-status to include runs that are no longer active. +>>> [CLI] version +Databricks CLI v[CLI_VERSION] diff --git a/acceptance/cmd/aitools/claude-config-dir/script b/acceptance/cmd/aitools/claude-config-dir/script index c64a5ec5ed7..ce77b7d518a 100644 --- a/acceptance/cmd/aitools/claude-config-dir/script +++ b/acceptance/cmd/aitools/claude-config-dir/script @@ -6,31 +6,30 @@ unset CLAUDE_CODE_SESSION_ID # CI and the CI/CD provider markers the SDK detects (GitHub Actions sets both). unset CI GITHUB_ACTIONS GITLAB_CI JENKINS_URL TF_BUILD CIRCLECI TRAVIS BITBUCKET_BUILD_NUMBER PROJECT_ID CODEBUILD_BUILD_ARN TFC_RUN_ID -# Only air subcommands print the install hint; air list against the mock -# server exercises it. +# aitools commands skip the install hint; version exercises it without auth or network. sethome ./empty-home title "Plugin installed only in the custom config directory" -trace CLAUDE_CONFIG_DIR=custom $CLI air list +trace CLAUDE_CONFIG_DIR=custom $CLI version sethome ./installed-home title "Custom directory takes precedence over the default installation" -trace CLAUDE_CONFIG_DIR=missing $CLI air list +trace CLAUDE_CONFIG_DIR=missing $CLI version title "Empty override uses the default installation" -trace CLAUDE_CONFIG_DIR= $CLI air list +trace CLAUDE_CONFIG_DIR= $CLI version title "Unset override uses the default installation" unset CLAUDE_CONFIG_DIR -trace $CLI air list +trace $CLI version sethome ./empty-home title "Outside a Claude Code subprocess only the text hint prints" -trace CLAUDE_CODE_CHILD_SESSION= $CLI air list +trace CLAUDE_CODE_CHILD_SESSION= $CLI version # The session throttle lives in the file cache, which a host could disable. export DATABRICKS_CACHE_ENABLED=true title "Within one session both lines print once, so 2>&1 pipelines see them only on the first command" export CLAUDE_CODE_SESSION_ID=session-1 -trace $CLI air list -trace $CLI air list +trace $CLI version +trace $CLI version unset CLAUDE_CODE_SESSION_ID diff --git a/acceptance/cmd/aitools/telemetry/apps-init-tip/output.txt b/acceptance/cmd/aitools/telemetry/apps-init-tip/output.txt index 74c334ebe1c..335595f4d75 100644 --- a/acceptance/cmd/aitools/telemetry/apps-init-tip/output.txt +++ b/acceptance/cmd/aitools/telemetry/apps-init-tip/output.txt @@ -2,6 +2,7 @@ === apps init with flags prints the tip: one hint-shown event per hint printed >>> [CLI] apps init --template ./template --skip-install --name app-tip +Databricks skills are not installed. To work with Databricks reliably, first run: databricks aitools install ✔ Creating project ✔ Project created successfully! @@ -12,6 +13,7 @@ Tip: coding agents detected without Databricks skills. Run 'databricks aitools install' to install them. >>> print_telemetry +{"command":"apps_init","exit_code":0,"aitools_hint_event":{"hint_kind":"INSTALL_TEXT"}} {"command":"apps_init","exit_code":0,"aitools_hint_event":{"hint_kind":"APPS_INIT_TIP"}} === plugin installed: no hint, no tip, no event diff --git a/acceptance/cmd/aitools/telemetry/hint/output.txt b/acceptance/cmd/aitools/telemetry/hint/output.txt index b90eb2a7930..0cf25cd970e 100644 --- a/acceptance/cmd/aitools/telemetry/hint/output.txt +++ b/acceptance/cmd/aitools/telemetry/hint/output.txt @@ -1,50 +1,25 @@ -=== hint printed on an air command: one hint-shown event per printed line is uploaded +=== hint printed on an authenticated command: one hint-shown event per printed line is uploaded ->>> [CLI] air list +>>> [CLI] jobs list -Tip: install the databricks-ai-runtime skill for workload configuration and run debugging: databricks aitools install --skills-only --skills databricks-ai-runtime. To preview it first, replace --skills-only with --path . -No active runs found. Use --all-status to include runs that are no longer active. +Databricks skills are not installed. To work with Databricks reliably, first run: databricks aitools install >>> print_telemetry -{"command":"air_list","exit_code":0,"aitools_hint_event":{"hint_kind":"CLAUDE_CODE_NATIVE"}} -{"command":"air_list","exit_code":0,"aitools_hint_event":{"hint_kind":"INSTALL_TEXT"}} - -=== commands outside air print no hint and log no event - ->>> [CLI] jobs list +{"command":"jobs_list","exit_code":0,"aitools_hint_event":{"hint_kind":"CLAUDE_CODE_NATIVE"}} +{"command":"jobs_list","exit_code":0,"aitools_hint_event":{"hint_kind":"INSTALL_TEXT"}} ->>> print_telemetry +=== hint printed on a command that resolves no auth: nothing is uploaded >>> [CLI] version -Databricks CLI v[CLI_VERSION] - -=== previewing a skill with --path installs nothing, so air still hints - ->>> [CLI] aitools install --path ./preview --skills databricks-test -Using skills version test-ref -Wrote 1 skill to ./preview - ->>> print_telemetry -{"command":"aitools_install","exit_code":0,"aitools_command_event":{"command":"INSTALL","stage":"STARTED","interactive":false,"install_mode":"PATH"}} -{"command":"aitools_install","exit_code":0,"aitools_command_event":{"command":"INSTALL","stage":"STEP_COMPLETED","step":"FLAGS_VALIDATED","interactive":false,"install_mode":"PATH"}} -{"command":"aitools_install","exit_code":0,"aitools_command_event":{"command":"INSTALL","stage":"STEP_COMPLETED","step":"SKILLS_DUMPED","interactive":false,"install_mode":"PATH"}} -{"command":"aitools_install","exit_code":0,"aitools_command_event":{"command":"INSTALL","stage":"FINISHED","outcome":"SUCCESS","interactive":false,"install_mode":"PATH"}} -preview/databricks-test/SKILL.md -no aitools state in HOME - ->>> [CLI] air list -Tip: install the databricks-ai-runtime skill for workload configuration and run debugging: databricks aitools install --skills-only --skills databricks-ai-runtime. To preview it first, replace --skills-only with --path . -No active runs found. Use --all-status to include runs that are no longer active. +Databricks skills are not installed. To work with Databricks reliably, first run: databricks aitools install +Databricks CLI v[CLI_VERSION] >>> print_telemetry -{"command":"air_list","exit_code":0,"aitools_hint_event":{"hint_kind":"CLAUDE_CODE_NATIVE"}} -{"command":"air_list","exit_code":0,"aitools_hint_event":{"hint_kind":"INSTALL_TEXT"}} === plugin installed: no hint, no event ->>> [CLI] air list -No active runs found. Use --all-status to include runs that are no longer active. +>>> [CLI] jobs list >>> print_telemetry diff --git a/acceptance/cmd/aitools/telemetry/hint/script b/acceptance/cmd/aitools/telemetry/hint/script index 8e725509b35..4aa3253b9f1 100644 --- a/acceptance/cmd/aitools/telemetry/hint/script +++ b/acceptance/cmd/aitools/telemetry/hint/script @@ -9,26 +9,15 @@ unset CLAUDE_CODE_SESSION_ID # A host CLAUDE_CONFIG_DIR would move the plugin file away from the isolated HOME. unset CLAUDE_CONFIG_DIR -title "hint printed on an air command: one hint-shown event per printed line is uploaded" -trace $CLI air list -trace print_telemetry - -title "commands outside air print no hint and log no event" +title "hint printed on an authenticated command: one hint-shown event per printed line is uploaded" trace $CLI jobs list trace print_telemetry -trace $CLI version -title "previewing a skill with --path installs nothing, so air still hints" -# The mock manifest serves only databricks-test. -trace $CLI aitools install --path ./preview --skills databricks-test -trace print_telemetry -find preview -type f | sort -rm -rf preview -test -e "$HOME/.databricks/aitools" && echo "unexpected aitools state in HOME" || echo "no aitools state in HOME" -trace $CLI air list +title "hint printed on a command that resolves no auth: nothing is uploaded" +trace $CLI version trace print_telemetry title "plugin installed: no hint, no event" sethome ./installed-home -trace $CLI air list +trace $CLI jobs list trace print_telemetry diff --git a/acceptance/cmd/version/output.txt b/acceptance/cmd/version/output.txt index a3e8c52c542..7fc22af691d 100644 --- a/acceptance/cmd/version/output.txt +++ b/acceptance/cmd/version/output.txt @@ -35,9 +35,10 @@ Tip: coding agents detected without Databricks skills. Run 'databricks aitools i "development_build": true } -=== version --check from Claude Code prints no hint +=== version --check from Claude Code prints only the root hint >>> [CLI] version --check +Databricks skills are not installed. To work with Databricks reliably, first run: databricks aitools install Databricks CLI v[CLI_VERSION] This is a development build; skipping the update check. diff --git a/acceptance/cmd/version/script b/acceptance/cmd/version/script index 7cac7e9b0e3..58a59538f05 100644 --- a/acceptance/cmd/version/script +++ b/acceptance/cmd/version/script @@ -23,17 +23,15 @@ trace $CLI version --check title "version --check --output json with a coding agent prints no tip" trace $CLI version --check --output json -# Callers that agents.MaybeHint hints get no tip here: it owns their install -# guidance and only hints on air subcommands, so `databricks version` prints -# no install guidance to them. -title "version --check from Claude Code prints no hint" +# Callers that agents.MaybeHint hints get only the root hint, not the tip. +title "version --check from Claude Code prints only the root hint" sethome ./claude-home export CLAUDECODE=1 unset CLAUDE_CODE_SESSION_ID trace $CLI version --check unset CLAUDECODE -title "version --check from Codex prints no hint" +title "version --check from Codex prints only the root hint" sethome ./codex-home export CODEX_CI=1 unset CODEX_THREAD_ID diff --git a/cmd/air/hint_test.go b/cmd/air/hint_test.go index 3e2ad540449..65390a4bd8b 100644 --- a/cmd/air/hint_test.go +++ b/cmd/air/hint_test.go @@ -5,6 +5,7 @@ import ( "context" "testing" + "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/internal/testutil" "github.com/databricks/cli/libs/aitools/agents" "github.com/databricks/cli/libs/cmdio" @@ -16,16 +17,16 @@ import ( // airUnderRoot mounts the real air tree under a root whose pre-run hook only // records that it ran, mirroring where cmd/root calls agents.MaybeHint. -func airUnderRoot(preRunCalled *bool) (*cobra.Command, *cobra.Command) { - root := &cobra.Command{Use: "databricks"} - root.AddGroup(&cobra.Group{ID: "development", Title: "Developer Tools"}) - root.PersistentPreRunE = func(cmd *cobra.Command, args []string) error { +func airUnderRoot(ctx context.Context, preRunCalled *bool) (*cobra.Command, *cobra.Command) { + dummy := &cobra.Command{Use: "databricks"} + dummy.AddGroup(&cobra.Group{ID: "development", Title: "Developer Tools"}) + dummy.PersistentPreRunE = func(cmd *cobra.Command, args []string) error { *preRunCalled = true return nil } - air := New() - root.AddCommand(air) - return root, air + air := root.NewCommand(ctx, Spec) + dummy.AddCommand(air) + return dummy, air } // TestBareAirAndHelpShowFooterWithoutPreRun pins why the footer, not the @@ -34,7 +35,7 @@ func TestBareAirAndHelpShowFooterWithoutPreRun(t *testing.T) { for _, args := range [][]string{{"air"}, {"air", "--help"}} { t.Run(args[len(args)-1], func(t *testing.T) { preRunCalled := false - root, _ := airUnderRoot(&preRunCalled) + root, _ := airUnderRoot(t.Context(), &preRunCalled) var out bytes.Buffer root.SetOut(&out) root.SetErr(&out) @@ -51,8 +52,7 @@ func TestBareAirAndHelpShowFooterWithoutPreRun(t *testing.T) { // subcommand is covered without listing it here. func TestRunnableAirCommandsGetSkillHint(t *testing.T) { testutil.SetAgentEnv(t, "CLAUDECODE", "1") - preRunCalled := false - _, air := airUnderRoot(&preRunCalled) + _, air := airUnderRoot(t.Context(), new(bool)) assert.False(t, air.Runnable(), "bare air renders help; the footer covers it") var runnable []string diff --git a/libs/aitools/agents/calling.go b/libs/aitools/agents/calling.go index 4f43ac58ad2..a0bfe24d7af 100644 --- a/libs/aitools/agents/calling.go +++ b/libs/aitools/agents/calling.go @@ -62,9 +62,10 @@ func (t Tooling) String() string { // record. For Codex and Copilot, pluginRecorded (the CLI install state) is the // only plugin source, so it misses plugins installed with the agent's own CLI // and includes plugins later removed with it. The shared skills store is not consulted: it cannot attribute skills. -// The install hint's airSkillAvailable differs on purpose: it looks for the AI -// Runtime skill only (the plugin counts either way), so another Databricks -// skill never silences a hint about this one. +// The install hint's airSkillAvailable (used on `databricks air` subcommands) +// differs on purpose: it looks for the AI Runtime skill only (the plugin +// counts either way), so another Databricks skill never silences a hint about +// that one. func (a *Agent) DatabricksTooling(ctx context.Context, pluginRecorded func() bool) Tooling { // An unknown working directory skips the project lookups below. cwd := "" diff --git a/libs/aitools/agents/hint.go b/libs/aitools/agents/hint.go index 04430f5e75f..ebed64fac44 100644 --- a/libs/aitools/agents/hint.go +++ b/libs/aitools/agents/hint.go @@ -62,15 +62,26 @@ const ( // aiToolsCommandName is the top-level command whose subtree is never hinted: // telling the agent to run `databricks aitools install` while it is already - // running an aitools command makes no sense. (The air scoping below already - // keeps aitools commands silent.) + // running an aitools command makes no sense. aiToolsCommandName = "aitools" - // airCommandName is the top-level command whose runnable subcommands get the - // hint. Bare `databricks air` and `--help` render help without running the - // pre-run hook; the air help footer carries the same recommendation there. + // airCommandName is the top-level command whose runnable subcommands swap + // the generic install hint for the AI Runtime skill recommendation. Bare + // `databricks air` and `--help` render help without running the pre-run + // hook; the air help footer carries the same recommendation there. airCommandName = "air" + // authCommandName and tokenCommandName name `databricks auth token`, the + // credential helper the SDKs and Terraform run. They embed its stderr + // mid-line in their own errors, where Claude Code can't strip the native + // line and neither hint is addressed to anyone, so it is never hinted. + authCommandName = "auth" + tokenCommandName = "token" + + // dockerCommandName names `databricks auth docker token`, which Docker runs + // through docker-credential-databricks and treats the same way. + dockerCommandName = "docker" + // selfExecEnv marks a CLI process that the CLI itself started (see // SelfExecEnv). Its output goes to the parent CLI, which already hinted // if a hint was due and may embed that output mid-line in its own errors, @@ -145,8 +156,8 @@ func hintForAgent(provider string, pluginRecorded PluginRecordedFunc) *agentHint switch provider { case claudeAgentProvider: return &agentHint{ - installed: airSkillAvailable, - text: airHintText, + installed: claudeToolingInstalled, + text: hintText, textKind: protos.AitoolsHintKindInstallText, sessionEnv: claudeSessionEnv, native: claudeNativeHintLine, @@ -160,6 +171,28 @@ func hintForAgent(provider string, pluginRecorded PluginRecordedFunc) *agentHint } } +// hintForCommand returns the hint due for cmd. On a runnable `databricks air` +// subcommand Claude Code gets the AI Runtime skill recommendation instead of +// the generic install hint: the skill is what helps with air work, and naming +// it lets the agent tell the hint is about the task at hand rather than +// generic setup. The plugin suggestion (native) is shared with the generic +// hint. Other agents keep their generic install hint everywhere: the aitools +// installer has no per-agent AI Runtime variant to offer them. +func hintForCommand(cmd *cobra.Command, provider string, pluginRecorded PluginRecordedFunc) *agentHint { + hint := hintForAgent(provider, pluginRecorded) + if hint != nil && provider == claudeAgentProvider && isAIRSubcommand(cmd) { + return &agentHint{ + installed: airSkillAvailable, + text: airHintText, + textKind: protos.AitoolsHintKindInstallText, + sessionEnv: claudeSessionEnv, + native: claudeNativeHintLine, + nativeKind: protos.AitoolsHintKindClaudeCodeNative, + } + } + return hint +} + // isHintedAgent reports whether MaybeHint hints the agent with this // useragent.AgentProvider() value. It defers to hintForAgent so both follow // the same dispatch; pluginRecorded is only read when the hint's installed @@ -177,14 +210,14 @@ func (h *agentHint) nativeLine(ctx context.Context) string { } // MaybeHint recommends installing the Databricks AI tooling when a supported -// agent drives the CLI without it installed, and only on `databricks air` -// subcommands: an agent running another command has no AI Runtime task for the -// skill to help with, so there the line is noise the agent learns to skip. -// For Claude Code the recommendation names the AI Runtime skill and how to -// preview it; other agents get their generic install command. It is a no-op -// for human callers, unsupported agents, commands outside air, a CLI process +// agent drives the CLI without it installed. On a runnable `databricks air` +// subcommand Claude Code instead gets the AI Runtime skill recommendation, +// which names the skill, what it covers, and how to preview it: an agent +// running air has an AI Runtime task for that skill to help with. It is a +// no-op for human callers, unsupported agents, aitools commands themselves, +// the `auth token` and `auth docker token` credential helpers, a CLI process // the CLI started itself, when there is no output stream, and once the -// skill is present. Bare `databricks air` and `--help` render help without +// tooling is present. Bare `databricks air` and `--help` render help without // running the pre-run hook; the air help footer carries the same // recommendation there. // Both lines, the agent-native one (if any) and the model-facing text, and @@ -196,11 +229,14 @@ func (h *agentHint) nativeLine(ctx context.Context) string { // telemetry event for each hint line shown, which makes a command that resolved // auth upload telemetry at exit. func MaybeHint(ctx context.Context, cmd *cobra.Command, pluginRecorded PluginRecordedFunc) { - if !isAIRSubcommand(cmd) { + hint := hintForCommand(cmd, useragent.AgentProvider(), pluginRecorded) + if hint == nil { return } - hint := hintForAgent(useragent.AgentProvider(), pluginRecorded) - if hint == nil { + if isAIToolsCommand(cmd) { + return + } + if isCredentialHelperCommand(cmd) { return } if env.Get(ctx, selfExecEnv) != "" { @@ -275,6 +311,30 @@ func inCI(ctx context.Context) bool { return useragent.CiCdProvider() != "" } +// isCredentialHelperCommand reports whether cmd is `databricks auth token` or +// `databricks auth docker token`. +func isCredentialHelperCommand(cmd *cobra.Command) bool { + if cmd.Name() != tokenCommandName { + return false + } + parent := cmd.Parent() + if parent != nil && parent.Name() == dockerCommandName { + parent = parent.Parent() + } + return parent != nil && parent.Name() == authCommandName +} + +// isAIToolsCommand reports whether cmd is the aitools command or one of its +// subcommands, walking the parent chain (mirrors versioncheck.isExemptCommand). +func isAIToolsCommand(cmd *cobra.Command) bool { + for c := cmd; c != nil; c = c.Parent() { + if c.Name() == aiToolsCommandName { + return true + } + } + return false +} + // isAIRSubcommand reports whether cmd sits under the top-level air command // (`databricks air list`, `databricks air pools list`, ...), excluding air // itself. Matching on the top-level ancestor keeps hidden aliases such as @@ -288,6 +348,32 @@ func isAIRSubcommand(cmd *cobra.Command) bool { return false } +// claudeToolingInstalled reports whether Claude Code has Databricks AI tooling +// in its own locations (see Agent.DatabricksTooling): the databricks plugin, or +// databricks skills in Claude's skills dirs. Skills present only in the shared +// ~/.databricks/aitools/skills store don't count, because Claude Code doesn't +// load them from there. A plugin recorded with project or local scope counts +// only when the CLI's working directory is inside the recorded project: Claude +// Code loads it from the directory it started in, which the CLI can't see, so +// a session started in the project that runs `cd /tmp/x && databricks ...` is +// hinted. Any read/parse failure reports false, so an unreadable state errs +// toward hinting. +func claudeToolingInstalled(ctx context.Context) bool { + claude := ByName(NameClaudeCode) + if claude == nil { + // claude-code is always in the registry; this guard keeps the hint + // best-effort rather than panicking in a persistent pre-run hook. + return false + } + // Claude Code's own manifest decides, so the CLI's install record is never + // consulted for it. + tooling := claude.DatabricksTooling(ctx, noRecordedPlugin) + if tooling == ToolingPlugin { + log.Debugf(ctx, "Databricks plugin detected for Claude Code; skipping install hint") + } + return tooling != ToolingNone +} + // airSkillAvailable reports whether Claude Code has the AI Runtime skill in a // location it loads for the current working directory: the databricks plugin // (which bundles every Databricks skill) installed for every project or for diff --git a/libs/aitools/agents/hint_agents_test.go b/libs/aitools/agents/hint_agents_test.go index 9a8cf384add..046af7cedec 100644 --- a/libs/aitools/agents/hint_agents_test.go +++ b/libs/aitools/agents/hint_agents_test.go @@ -118,7 +118,7 @@ func TestAgentCtxIgnoresProjectSkillsAroundTheTest(t *testing.T) { setAgentEnv(t, agentDetectEnv[NameGemini]) ctx, stderr := agentCtx(t, t.TempDir(), t.TempDir()) - MaybeHint(ctx, airCmd(), noPluginRecorded) + MaybeHint(ctx, regularCmd(), noPluginRecorded) assert.Equal(t, agentInstallHint(gemini, noPluginRecorded).text+"\n", stderr.String()) } @@ -142,7 +142,7 @@ func TestMaybeHint_PerAgent(t *testing.T) { t.Run("not installed hints", func(t *testing.T) { ctx, stderr := agentCtx(t, t.TempDir(), t.TempDir()) ctx = telemetry.WithNewLogger(ctx) - MaybeHint(ctx, airCmd(), noPluginRecorded) + MaybeHint(ctx, regularCmd(), noPluginRecorded) assert.Equal(t, agentInstallHint(a, noPluginRecorded).text+"\n", stderr.String()) events := telemetry.BufferedEvents(ctx) @@ -157,7 +157,7 @@ func TestMaybeHint_PerAgent(t *testing.T) { ctx, stderr := agentCtx(t, home, t.TempDir()) ctx = telemetry.WithNewLogger(ctx) writeAgentSkill(t, ctx, home, a) - MaybeHint(ctx, airCmd(), noPluginRecorded) + MaybeHint(ctx, regularCmd(), noPluginRecorded) assert.Empty(t, stderr.String()) assert.Empty(t, telemetry.BufferedEvents(ctx)) }) @@ -168,18 +168,18 @@ func TestMaybeHint_PerAgent(t *testing.T) { home := t.TempDir() ctx, stderr := agentCtx(t, home, t.TempDir()) writeCanonicalSkill(t, home, "databricks") - MaybeHint(ctx, airCmd(), noPluginRecorded) + MaybeHint(ctx, regularCmd(), noPluginRecorded) assert.Equal(t, agentInstallHint(a, noPluginRecorded).text+"\n", stderr.String()) }) t.Run("throttled per machine without session id", func(t *testing.T) { home, cacheDir := t.TempDir(), t.TempDir() ctx1, stderr1 := agentCtx(t, home, cacheDir) - MaybeHint(ctx1, airCmd(), noPluginRecorded) + MaybeHint(ctx1, regularCmd(), noPluginRecorded) assert.NotEmpty(t, stderr1.String()) ctx2, stderr2 := agentCtx(t, home, cacheDir) - MaybeHint(ctx2, airCmd(), noPluginRecorded) + MaybeHint(ctx2, regularCmd(), noPluginRecorded) assert.Empty(t, stderr2.String()) }) }) @@ -203,7 +203,7 @@ func TestMaybeHint_RecordedPluginSilent(t *testing.T) { } ctx, stderr := agentCtx(t, t.TempDir(), t.TempDir()) ctx = telemetry.WithNewLogger(ctx) - MaybeHint(ctx, airCmd(), recorded) + MaybeHint(ctx, regularCmd(), recorded) assert.Empty(t, stderr.String()) assert.Empty(t, telemetry.BufferedEvents(ctx)) assert.Equal(t, []string{name}, asked) @@ -217,7 +217,7 @@ func TestMaybeHint_CodexThrottledPerThread(t *testing.T) { run := func(thread string) string { ctx, stderr := agentCtx(t, home, cacheDir) ctx = env.Set(ctx, "CODEX_THREAD_ID", thread) - MaybeHint(ctx, airCmd(), noPluginRecorded) + MaybeHint(ctx, regularCmd(), noPluginRecorded) return stderr.String() } @@ -232,7 +232,7 @@ func TestMaybeHint_NoInstallTargetSilent(t *testing.T) { t.Run(envVar, func(t *testing.T) { setAgentEnv(t, envVar) ctx, stderr := agentCtx(t, t.TempDir(), t.TempDir()) - MaybeHint(ctx, airCmd(), noPluginRecorded) + MaybeHint(ctx, regularCmd(), noPluginRecorded) assert.Empty(t, stderr.String()) }) } @@ -242,15 +242,15 @@ func TestMaybeHint_MultipleAgentsSilent(t *testing.T) { setAgentEnv(t, "CODEX_CI") t.Setenv("CURSOR_AGENT", "1") ctx, stderr := agentCtx(t, t.TempDir(), t.TempDir()) - MaybeHint(ctx, airCmd(), noPluginRecorded) + MaybeHint(ctx, regularCmd(), noPluginRecorded) assert.Empty(t, stderr.String()) } func TestMaybeHint_AgentAuthTokenSilent(t *testing.T) { setAgentEnv(t, "CODEX_CI") ctx, stderr := agentCtx(t, t.TempDir(), t.TempDir()) - parent := &cobra.Command{Use: "auth"} - child := &cobra.Command{Use: "token"} + parent := &cobra.Command{Use: authCommandName} + child := &cobra.Command{Use: tokenCommandName} parent.AddCommand(child) MaybeHint(ctx, child, noPluginRecorded) assert.Empty(t, stderr.String()) @@ -274,13 +274,13 @@ func TestMaybeHint_SessionMarkersAreIsolatedPerAgent(t *testing.T) { setClaudeAgent(t) ctx, stderr := agentCtx(t, home, cacheDir) ctx = env.Set(ctx, claudeSessionEnv, "shared-id") - MaybeHint(ctx, airCmd(), noPluginRecorded) - require.Contains(t, stderr.String(), airHintText) + MaybeHint(ctx, regularCmd(), noPluginRecorded) + require.Contains(t, stderr.String(), hintText) setAgentEnv(t, "CODEX_CI") ctx, stderr = agentCtx(t, home, cacheDir) ctx = env.Set(ctx, "CODEX_THREAD_ID", "shared-id") - MaybeHint(ctx, airCmd(), noPluginRecorded) + MaybeHint(ctx, regularCmd(), noPluginRecorded) assert.Equal(t, agentInstallHint(ByName(NameCodex), noPluginRecorded).text+"\n", stderr.String()) } @@ -304,7 +304,7 @@ func TestMaybeHint_UnwritableCacheSkipsHint(t *testing.T) { ctx, stderr := agentCtx(t, t.TempDir(), readOnly) ctx = telemetry.WithNewLogger(ctx) ctx = env.Set(ctx, "CODEX_THREAD_ID", "thread-1") - MaybeHint(ctx, airCmd(), noPluginRecorded) + MaybeHint(ctx, regularCmd(), noPluginRecorded) assert.Empty(t, stderr.String()) assert.Empty(t, telemetry.BufferedEvents(ctx)) } @@ -324,8 +324,25 @@ func TestMaybeHint_DisabledCacheSkipsHint(t *testing.T) { ctx = env.Set(ctx, "DATABRICKS_CACHE_ENABLED", "false") ctx = telemetry.WithNewLogger(ctx) ctx = env.Set(ctx, "CODEX_THREAD_ID", "thread-1") - MaybeHint(ctx, airCmd(), noPluginRecorded) + MaybeHint(ctx, regularCmd(), noPluginRecorded) assert.Empty(t, stderr.String()) assert.Empty(t, telemetry.BufferedEvents(ctx)) } } + +// Non-Claude agents keep their generic install hint on `databricks air` +// subcommands too: the aitools installer has no per-agent AI Runtime variant +// to offer them, so the air scoping only changes Claude Code's hint. +func TestMaybeHint_AgentInstallHintUnchangedOnAIR(t *testing.T) { + for _, a := range Registry { + if a.Name == NameClaudeCode { + continue + } + t.Run(a.Name, func(t *testing.T) { + setAgentEnv(t, agentDetectEnv[a.Name]) + ctx, stderr := agentCtx(t, t.TempDir(), t.TempDir()) + MaybeHint(ctx, airCmd(), noPluginRecorded) + assert.Equal(t, agentInstallHint(a, noPluginRecorded).text+"\n", stderr.String()) + }) + } +} diff --git a/libs/aitools/agents/hint_test.go b/libs/aitools/agents/hint_test.go index 9c7268a8cb0..8e99f64ee75 100644 --- a/libs/aitools/agents/hint_test.go +++ b/libs/aitools/agents/hint_test.go @@ -36,33 +36,19 @@ func setClaudeAgent(t *testing.T) { testutil.SetAgentEnv(t, "CLAUDECODE", "1") } -// claudeCtx returns a context with a captured, non-TTY stderr buffer (what a -// coding agent's shell tool gives the CLI) and an isolated HOME + cache dir, -// so detection and the session throttle read only the test's temp state. home -// is returned so the test can drop fixtures under it. +// claudeCtx returns a context with a captured stderr buffer and an isolated +// HOME + cache dir, so detection and the session throttle read only the test's +// temp state. home is returned so the test can drop fixtures under it. func claudeCtx(t *testing.T, home string) (context.Context, *bytes.Buffer) { ctx, stderr := cmdio.NewTestContextWithStderr(t.Context()) ctx = env.WithUserHomeDir(ctx, home) ctx = env.Set(ctx, "DATABRICKS_CACHE_DIR", t.TempDir()) // Throttle markers live in the file cache, which a host could disable. ctx = env.Set(ctx, "DATABRICKS_CACHE_ENABLED", "true") - return telemetry.WithNewLogger(ctx), stderr + return ctx, stderr } -// commandAt builds the command tree for a space-separated path under a -// "databricks" root and returns the leaf, so ancestry checks see the same -// shape as the real CLI. -func commandAt(path string) *cobra.Command { - cmd := &cobra.Command{Use: "databricks"} - for name := range strings.FieldsSeq(path) { - child := &cobra.Command{Use: name} - cmd.AddCommand(child) - cmd = child - } - return cmd -} - -func airCmd() *cobra.Command { return commandAt("air list") } +func regularCmd() *cobra.Command { return &cobra.Command{Use: "current-user"} } func writePluginManifest(t *testing.T, home, body string) { dir := filepath.Join(home, ".claude", "plugins") @@ -70,29 +56,6 @@ func writePluginManifest(t *testing.T, home, body string) { require.NoError(t, os.WriteFile(filepath.Join(dir, "installed_plugins.json"), []byte(body), 0o600)) } -// writeProjectPluginManifest writes a manifest with one project-scope install -// covering projectPath. Marshaled, not interpolated, so Windows paths with -// backslashes stay valid JSON. -func writeProjectPluginManifest(t *testing.T, home, projectPath string) { - manifest, err := json.Marshal(map[string]any{ - "plugins": map[string]any{ - "databricks@m": []any{map[string]string{ - "scope": "project", - "projectPath": projectPath, - }}, - }, - }) - require.NoError(t, err) - writePluginManifest(t, home, string(manifest)) -} - -// writeClaudeSkill puts a skill dir in Claude Code's own skills dir. It is a -// real dir rather than the installer's symlink so the fixture needs no symlink -// privilege on Windows; skill detection treats both alike. -func writeClaudeSkill(t *testing.T, home, name string) { - require.NoError(t, os.MkdirAll(filepath.Join(home, ".claude", "skills", name), 0o755)) -} - // hintKinds returns the kinds of the hint-shown events buffered on ctx, // failing on any other event so a stray event cannot hide behind the filter. func hintKinds(t *testing.T, ctx context.Context) []protos.AitoolsHintKind { @@ -104,80 +67,30 @@ func hintKinds(t *testing.T, ctx context.Context) []protos.AitoolsHintKind { return out } -// assertHinted checks for exactly one hint line and one hint-shown event of -// kind INSTALL_TEXT. -func assertHinted(t *testing.T, ctx context.Context, stderr *bytes.Buffer) { - t.Helper() - assert.Equal(t, airHintText+"\n", stderr.String()) - assert.Equal(t, []protos.AitoolsHintKind{protos.AitoolsHintKindInstallText}, hintKinds(t, ctx)) -} - -// assertSilent checks for no hint line and no telemetry event. -func assertSilent(t *testing.T, ctx context.Context, stderr *bytes.Buffer) { - t.Helper() - assert.Empty(t, stderr.String()) - assert.Empty(t, hintKinds(t, ctx)) -} - -func TestAIRHintText(t *testing.T) { - // The hint names the skill, says what it is for, and offers both the install - // and a no-install preview, on one line. - assert.Contains(t, airHintText, "install the databricks-ai-runtime skill") - assert.Contains(t, airHintText, "workload configuration and run debugging") - assert.Contains(t, airHintText, "databricks aitools install --skills-only --skills databricks-ai-runtime") - assert.Contains(t, airHintText, "replace --skills-only with --path ") - assert.NotContains(t, airHintText, "\n") +// writeClaudeSkill puts a skill dir in Claude Code's own skills dir. It is a +// real dir rather than the installer's symlink so the fixture needs no symlink +// privilege on Windows; HasDatabricksSkillsIn treats both alike. +func writeClaudeSkill(t *testing.T, home, name string) { + require.NoError(t, os.MkdirAll(filepath.Join(home, ".claude", "skills", name), 0o755)) } func TestMaybeHint_HumanCallerSilent(t *testing.T) { // No agent env: AgentProvider() == "", so no hint even with nothing installed. testutil.SetAgentEnv(t) - ctx, stderr := claudeCtx(t, t.TempDir()) - MaybeHint(ctx, airCmd(), noPluginRecorded) - assertSilent(t, ctx, stderr) -} - -// Every air subcommand, at any depth, gets the skill-specific hint. -func TestMaybeHint_AIRSubcommandsHint(t *testing.T) { - setClaudeAgent(t) - for _, path := range []string{ - "air run", "air get", "air list", "air logs", "air cancel", - "air convert-to-dabs", "air images", "air images push", "air pools", "air pools list", - } { - t.Run(path, func(t *testing.T) { - ctx, stderr := claudeCtx(t, t.TempDir()) - MaybeHint(ctx, commandAt(path), noPluginRecorded) - assertHinted(t, ctx, stderr) - }) - } -} - -// Commands outside air are never hinted, including the auth and version -// commands the generic hint used to fire on. -func TestMaybeHint_NonAIRCommandsSilent(t *testing.T) { - setClaudeAgent(t) - for _, path := range []string{ - "auth profiles", "auth token", "current-user me", "clusters list", "jobs list", - "bundle deploy", "workspace list", "version", "aitools install", - "experimental aitools install", - // The bare air node only renders help; its footer carries the tip. - "air", - // The hidden alias that points users at `databricks air`. - "experimental air", - } { - t.Run(path, func(t *testing.T) { - ctx, stderr := claudeCtx(t, t.TempDir()) - MaybeHint(ctx, commandAt(path), noPluginRecorded) - assertSilent(t, ctx, stderr) - }) - } + ctx, stderr := cmdio.NewTestContextWithStderr(t.Context()) + ctx = env.WithUserHomeDir(ctx, t.TempDir()) + ctx = telemetry.WithNewLogger(ctx) + MaybeHint(ctx, regularCmd(), noPluginRecorded) + assert.Empty(t, stderr.String()) + assert.Empty(t, hintKinds(t, ctx)) } func TestMaybeHint_ClaudeNotInstalledHints(t *testing.T) { setClaudeAgent(t) ctx, stderr := claudeCtx(t, t.TempDir()) - MaybeHint(ctx, airCmd(), noPluginRecorded) - assert.Contains(t, stderr.String(), airHintText) + ctx = telemetry.WithNewLogger(ctx) + MaybeHint(ctx, regularCmd(), noPluginRecorded) + assert.Contains(t, stderr.String(), hintText) assert.Equal(t, []protos.AitoolsHintKind{protos.AitoolsHintKindInstallText}, hintKinds(t, ctx)) } @@ -185,33 +98,18 @@ func TestMaybeHint_PluginInstalledSilent(t *testing.T) { setClaudeAgent(t) home := t.TempDir() ctx, stderr := claudeCtx(t, home) - // A plugin install writes only Claude's manifest, no skills subdir, and - // bundles every Databricks skill including databricks-ai-runtime. + // The core regression: a plugin install writes only Claude's manifest, no + // skills subdir. Detection must treat it as installed. writePluginManifest(t, home, `{"plugins":{"databricks@claude-plugins-official":[{"scope":"user"}]}}`) + ctx = telemetry.WithNewLogger(ctx) - MaybeHint(ctx, airCmd(), noPluginRecorded) - assertSilent(t, ctx, stderr) -} - -func TestAIRSkillAvailable_LogsPluginDetection(t *testing.T) { - t.Chdir(t.TempDir()) - home := t.TempDir() - writePluginManifest(t, home, `{"plugins":{"databricks@claude-plugins-official":[{"version":"0.2.6"}]}}`) - - // Capture debug output by installing a text logger on the context. - var buf bytes.Buffer - handler := slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: log.LevelDebug}) - ctx := log.NewContext(t.Context(), slog.New(handler)) - ctx = env.WithUserHomeDir(ctx, home) - - assert.True(t, airSkillAvailable(ctx)) - - assert.Contains(t, buf.String(), "Databricks plugin active for Claude Code") - assert.Contains(t, buf.String(), "skipping AI Runtime skill hint") + MaybeHint(ctx, regularCmd(), noPluginRecorded) + assert.Empty(t, stderr.String()) + assert.Empty(t, hintKinds(t, ctx)) } // A project or local plugin only loads inside its own project, so elsewhere -// Claude Code still lacks the skill and gets the hint. +// Claude Code still lacks the tooling and gets the hint. func TestMaybeHint_ProjectPluginCountsOnlyInItsProject(t *testing.T) { project, other := t.TempDir(), t.TempDir() for _, tc := range []struct { @@ -233,16 +131,33 @@ func TestMaybeHint_ProjectPluginCountsOnlyInItsProject(t *testing.T) { writePluginManifest(t, home, string(manifest)) t.Chdir(project) - MaybeHint(ctx, airCmd(), noPluginRecorded) + MaybeHint(ctx, regularCmd(), noPluginRecorded) if tc.wantHint { - assertHinted(t, ctx, stderr) + assert.Contains(t, stderr.String(), hintText) } else { - assertSilent(t, ctx, stderr) + assert.Empty(t, stderr.String()) } }) } } +func TestClaudeToolingInstalled_LogsPluginDetected(t *testing.T) { + // Clears the host's CLAUDE_CONFIG_DIR, which would move the manifest. + testutil.SetAgentEnv(t) + home := t.TempDir() + writePluginManifest(t, home, `{"plugins":{"databricks@claude-plugins-official":[{"version":"0.2.6"}]}}`) + + // Capture debug output by installing a text logger on the context. + var buf bytes.Buffer + handler := slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: log.LevelDebug}) + ctx := log.NewContext(t.Context(), slog.New(handler)) + ctx = env.WithUserHomeDir(ctx, home) + + assert.True(t, claudeToolingInstalled(ctx)) + + assert.Contains(t, buf.String(), "Databricks plugin detected for Claude Code; skipping install hint") +} + func TestMaybeHint_CustomClaudeConfigDir(t *testing.T) { setClaudeAgent(t) for _, tc := range []struct { @@ -269,11 +184,11 @@ func TestMaybeHint_CustomClaudeConfigDir(t *testing.T) { writePluginManifest(t, home, manifest) } - MaybeHint(ctx, airCmd(), noPluginRecorded) + MaybeHint(ctx, regularCmd(), noPluginRecorded) if tc.wantHint { - assertHinted(t, ctx, stderr) + assert.Contains(t, stderr.String(), hintText) } else { - assertSilent(t, ctx, stderr) + assert.Empty(t, stderr.String()) } }) } @@ -283,10 +198,10 @@ func TestMaybeHint_SkillsInstalledSilent(t *testing.T) { setClaudeAgent(t) home := t.TempDir() ctx, stderr := claudeCtx(t, home) - writeClaudeSkill(t, home, airSkillName) + writeClaudeSkill(t, home, "databricks") - MaybeHint(ctx, airCmd(), noPluginRecorded) - assertSilent(t, ctx, stderr) + MaybeHint(ctx, regularCmd(), noPluginRecorded) + assert.Empty(t, stderr.String()) } func TestMaybeHint_SharedStoreOnlyStillHints(t *testing.T) { @@ -297,10 +212,10 @@ func TestMaybeHint_SharedStoreOnlyStillHints(t *testing.T) { setClaudeAgent(t) home := t.TempDir() ctx, stderr := claudeCtx(t, home) - require.NoError(t, os.MkdirAll(filepath.Join(home, dir, airSkillName), 0o755)) + require.NoError(t, os.MkdirAll(filepath.Join(home, dir, "databricks"), 0o755)) - MaybeHint(ctx, airCmd(), noPluginRecorded) - assert.Equal(t, airHintText+"\n", stderr.String()) + MaybeHint(ctx, regularCmd(), noPluginRecorded) + assert.Equal(t, hintText+"\n", stderr.String()) }) } } @@ -311,34 +226,22 @@ func TestMaybeHint_NonDatabricksSkillStillHints(t *testing.T) { ctx, stderr := claudeCtx(t, home) writeClaudeSkill(t, home, "some-other-skill") - MaybeHint(ctx, airCmd(), noPluginRecorded) - assert.Contains(t, stderr.String(), airHintText) -} - -// Other Databricks skills, even in Claude Code's own skills dir, do not -// silence the hint: they don't cover air, and the hint recommends installing -// exactly the AI Runtime skill. -func TestMaybeHint_OtherDatabricksSkillStillHints(t *testing.T) { - setClaudeAgent(t) - home := t.TempDir() - ctx, stderr := claudeCtx(t, home) - writeClaudeSkill(t, home, "databricks") - writeClaudeSkill(t, home, "databricks-jobs") - - MaybeHint(ctx, airCmd(), noPluginRecorded) - assertHinted(t, ctx, stderr) + MaybeHint(ctx, regularCmd(), noPluginRecorded) + assert.Contains(t, stderr.String(), hintText) } func TestMaybeHint_AIToolsCommandSilent(t *testing.T) { setClaudeAgent(t) ctx, stderr := claudeCtx(t, t.TempDir()) + ctx = telemetry.WithNewLogger(ctx) parent := &cobra.Command{Use: aiToolsCommandName} child := &cobra.Command{Use: "install"} parent.AddCommand(child) MaybeHint(ctx, child, noPluginRecorded) - assertSilent(t, ctx, stderr) + assert.Empty(t, stderr.String()) + assert.Empty(t, hintKinds(t, ctx)) } func TestMaybeHint_NoIONoPanic(t *testing.T) { @@ -347,141 +250,32 @@ func TestMaybeHint_NoIONoPanic(t *testing.T) { // invocations before PersistentPreRunE runs, so HasIO can be false here. ctx := env.WithUserHomeDir(t.Context(), t.TempDir()) ctx = telemetry.WithNewLogger(ctx) - assert.NotPanics(t, func() { MaybeHint(ctx, airCmd(), noPluginRecorded) }) + assert.NotPanics(t, func() { MaybeHint(ctx, regularCmd(), noPluginRecorded) }) assert.Empty(t, hintKinds(t, ctx)) } -// With a session id the first air command in the session is hinted and later -// ones stay quiet until the marker expires; without one (older Claude Code) -// every air command is hinted. Both cases run with a non-TTY stderr, which is -// how a coding agent's shell tool runs the CLI. -func TestMaybeHint_SessionThrottle(t *testing.T) { - setClaudeAgent(t) - for _, tc := range []struct { - name string - session string - wantSecond bool - }{ - {"session id: first air command only", "session-abc", false}, - {"no session id: every air command", "", true}, - } { - t.Run(tc.name, func(t *testing.T) { - home := t.TempDir() - cacheDir := t.TempDir() - newCtx := func() (context.Context, *bytes.Buffer) { - ctx, stderr := claudeCtx(t, home) - ctx = env.Set(ctx, "DATABRICKS_CACHE_DIR", cacheDir) - if tc.session != "" { - ctx = env.Set(ctx, claudeSessionEnv, tc.session) - } - return ctx, stderr - } - - // A non-air command first must not use up the session's hint. - ctx0, stderr0 := newCtx() - MaybeHint(ctx0, commandAt("auth profiles"), noPluginRecorded) - assertSilent(t, ctx0, stderr0) - - ctx1, stderr1 := newCtx() - MaybeHint(ctx1, commandAt("air list"), noPluginRecorded) - assertHinted(t, ctx1, stderr1) - - ctx2, stderr2 := newCtx() - MaybeHint(ctx2, commandAt("air run"), noPluginRecorded) - if tc.wantSecond { - assertHinted(t, ctx2, stderr2) - } else { - assertSilent(t, ctx2, stderr2) - } - }) - } -} - -func TestMaybeHint_OtherSessionStillHinted(t *testing.T) { +func TestMaybeHint_ThrottledPerSession(t *testing.T) { setClaudeAgent(t) home := t.TempDir() cacheDir := t.TempDir() - for _, session := range []string{"session-a", "session-b"} { - ctx, stderr := claudeCtx(t, home) + newCtx := func() (context.Context, *bytes.Buffer) { + ctx, stderr := cmdio.NewTestContextWithStderr(t.Context()) + ctx = env.WithUserHomeDir(ctx, home) ctx = env.Set(ctx, "DATABRICKS_CACHE_DIR", cacheDir) - ctx = env.Set(ctx, claudeSessionEnv, session) - MaybeHint(ctx, airCmd(), noPluginRecorded) - assertHinted(t, ctx, stderr) - } -} - -// Only the databricks-ai-runtime skill itself, in a location Claude Code loads -// for this working directory, silences the hint. -func TestMaybeHint_AIRSkillDetection(t *testing.T) { - for _, tc := range []struct { - name string - setup func(t *testing.T, home, cwd string) - wantHint bool - }{ - {"nothing installed", func(t *testing.T, home, cwd string) {}, true}, - {"skill in Claude's own skills dir", func(t *testing.T, home, cwd string) { - writeClaudeSkill(t, home, airSkillName) - }, false}, - {"skill in the project skills dir", func(t *testing.T, home, cwd string) { - require.NoError(t, os.MkdirAll(filepath.Join(cwd, ".claude", "skills", airSkillName), 0o755)) - }, false}, - {"other databricks skills only", func(t *testing.T, home, cwd string) { - writeClaudeSkill(t, home, "databricks") - writeClaudeSkill(t, home, "databricks-jobs") - }, true}, - {"a file named like the skill", func(t *testing.T, home, cwd string) { - dir := filepath.Join(home, ".claude", "skills") - require.NoError(t, os.MkdirAll(dir, 0o755)) - require.NoError(t, os.WriteFile(filepath.Join(dir, airSkillName), nil, 0o600)) - }, true}, - {"shared canonical store", func(t *testing.T, home, cwd string) { - writeCanonicalSkill(t, home, airSkillName) - }, true}, - {"shared legacy store", func(t *testing.T, home, cwd string) { - require.NoError(t, os.MkdirAll(filepath.Join(home, LegacySkillsDir, airSkillName), 0o755)) - }, true}, - {"plugin for this project", func(t *testing.T, home, cwd string) { - writeProjectPluginManifest(t, home, cwd) - }, false}, - {"plugin for an unrelated project", func(t *testing.T, home, cwd string) { - writeProjectPluginManifest(t, home, t.TempDir()) - }, true}, - } { - t.Run(tc.name, func(t *testing.T) { - // A fresh project per case: the project-skills fixtures write into - // the working directory and must not leak between cases. - setClaudeAgent(t) - home := t.TempDir() - ctx, stderr := claudeCtx(t, home) - cwd, err := os.Getwd() - require.NoError(t, err) - tc.setup(t, home, cwd) - - MaybeHint(ctx, airCmd(), noPluginRecorded) - if tc.wantHint { - assertHinted(t, ctx, stderr) - } else { - assertSilent(t, ctx, stderr) - } - }) + ctx = env.Set(ctx, claudeSessionEnv, "session-abc") + return telemetry.WithNewLogger(ctx), stderr } -} -// A project-scoped skill at the repository root still counts when the CLI runs -// in a subdirectory, as Claude Code loads it there. -func TestMaybeHint_AIRSkillInRepositoryRoot(t *testing.T) { - setClaudeAgent(t) - home := t.TempDir() - ctx, stderr := claudeCtx(t, home) - root, err := os.Getwd() - require.NoError(t, err) - require.NoError(t, os.MkdirAll(filepath.Join(root, ".claude", "skills", airSkillName), 0o755)) - sub := filepath.Join(root, "training", "configs") - require.NoError(t, os.MkdirAll(sub, 0o755)) - t.Chdir(sub) + ctx1, stderr1 := newCtx() + MaybeHint(ctx1, regularCmd(), noPluginRecorded) + assert.Contains(t, stderr1.String(), hintText) + assert.Equal(t, []protos.AitoolsHintKind{protos.AitoolsHintKindInstallText}, hintKinds(t, ctx1)) - MaybeHint(ctx, airCmd(), noPluginRecorded) - assertSilent(t, ctx, stderr) + // Same session id, second command: must stay quiet and log nothing. + ctx2, stderr2 := newCtx() + MaybeHint(ctx2, regularCmd(), noPluginRecorded) + assert.Empty(t, stderr2.String()) + assert.Empty(t, hintKinds(t, ctx2)) } // wantNativeHint is the exact line Claude Code parses; see @@ -495,7 +289,7 @@ func TestClaudeNativeHint_Format(t *testing.T) { // authDockerCmd returns `auth docker ` as a leaf command. func authDockerCmd(use string) *cobra.Command { - auth := &cobra.Command{Use: "auth"} + auth := &cobra.Command{Use: authCommandName} docker := &cobra.Command{Use: "docker"} leaf := &cobra.Command{Use: use} auth.AddCommand(docker) @@ -518,43 +312,43 @@ func TestMaybeHint_NativeHint(t *testing.T) { { name: "claude child session prints native then text", env: map[string]string{claudeChildSessionEnv: "1"}, - wantStderr: wantNativeHint + "\n" + airHintText + "\n", + wantStderr: wantNativeHint + "\n" + hintText + "\n", }, { name: "no child session prints text only", - wantStderr: airHintText + "\n", + wantStderr: hintText + "\n", }, { name: "CI prints text only", env: map[string]string{claudeChildSessionEnv: "1", ciEnv: "true"}, - wantStderr: airHintText + "\n", + wantStderr: hintText + "\n", }, { name: "CI=false still prints native", env: map[string]string{claudeChildSessionEnv: "1", ciEnv: "false"}, - wantStderr: wantNativeHint + "\n" + airHintText + "\n", + wantStderr: wantNativeHint + "\n" + hintText + "\n", }, { name: "CI=no still prints native", env: map[string]string{claudeChildSessionEnv: "1", ciEnv: "no"}, - wantStderr: wantNativeHint + "\n" + airHintText + "\n", + wantStderr: wantNativeHint + "\n" + hintText + "\n", }, { name: "CI=OFF still prints native", env: map[string]string{claudeChildSessionEnv: "1", ciEnv: "OFF"}, - wantStderr: wantNativeHint + "\n" + airHintText + "\n", + wantStderr: wantNativeHint + "\n" + hintText + "\n", }, { // Some providers set CI to their own name. name: "CI set to a provider name prints text only", env: map[string]string{claudeChildSessionEnv: "1", ciEnv: "woodpecker"}, - wantStderr: airHintText + "\n", + wantStderr: hintText + "\n", }, { // Azure Pipelines sets TF_BUILD but not CI. name: "CI provider without CI var prints text only", env: map[string]string{claudeChildSessionEnv: "1", "TF_BUILD": "True"}, - wantStderr: airHintText + "\n", + wantStderr: hintText + "\n", }, { name: "plugin installed prints nothing", @@ -564,28 +358,28 @@ func TestMaybeHint_NativeHint(t *testing.T) { }, }, { - name: "skill installed prints nothing", + name: "skills installed prints nothing", env: map[string]string{claudeChildSessionEnv: "1"}, setup: func(t *testing.T, home string) { - writeClaudeSkill(t, home, airSkillName) + writeClaudeSkill(t, home, "databricks") }, }, { - // Commands outside air never hint, credential helpers included. name: "auth token prints nothing", env: map[string]string{claudeChildSessionEnv: "1"}, cmd: func() *cobra.Command { - parent := &cobra.Command{Use: "auth"} + parent := &cobra.Command{Use: authCommandName} child := &cobra.Command{Use: "token [PROFILE]"} parent.AddCommand(child) return child }, }, { - name: "other auth command prints nothing", - env: map[string]string{claudeChildSessionEnv: "1"}, + name: "other auth command still hints", + env: map[string]string{claudeChildSessionEnv: "1"}, + wantStderr: wantNativeHint + "\n" + hintText + "\n", cmd: func() *cobra.Command { - parent := &cobra.Command{Use: "auth"} + parent := &cobra.Command{Use: authCommandName} child := &cobra.Command{Use: "profiles"} parent.AddCommand(child) return child @@ -597,9 +391,25 @@ func TestMaybeHint_NativeHint(t *testing.T) { cmd: func() *cobra.Command { return authDockerCmd("token") }, }, { - name: "other auth docker command prints nothing", + name: "other auth docker command still hints", + env: map[string]string{claudeChildSessionEnv: "1"}, + wantStderr: wantNativeHint + "\n" + hintText + "\n", + cmd: func() *cobra.Command { return authDockerCmd("configure [PROFILE]") }, + }, + { + // On air subcommands the text is the AI Runtime skill line. + name: "air command prints native then air text", + env: map[string]string{claudeChildSessionEnv: "1"}, + wantStderr: wantNativeHint + "\n" + airHintText + "\n", + cmd: func() *cobra.Command { return airCmd() }, + }, + { + name: "air command with the skill installed prints nothing", env: map[string]string{claudeChildSessionEnv: "1"}, - cmd: func() *cobra.Command { return authDockerCmd("configure [PROFILE]") }, + cmd: func() *cobra.Command { return airCmd() }, + setup: func(t *testing.T, home string) { + writeClaudeSkill(t, home, airSkillName) + }, }, { // air images push runs `auth docker configure` and embeds its @@ -630,7 +440,7 @@ func TestMaybeHint_NativeHint(t *testing.T) { if tc.setup != nil { tc.setup(t, home) } - cmd := airCmd() + cmd := regularCmd() if tc.cmd != nil { cmd = tc.cmd() } @@ -659,7 +469,7 @@ func TestMaybeHint_NativeHintSilentForOtherCallers(t *testing.T) { // Claude Code caller gets the native hint. ctx = env.Set(ctx, claudeChildSessionEnv, "1") - MaybeHint(ctx, airCmd(), noPluginRecorded) + MaybeHint(ctx, regularCmd(), noPluginRecorded) assert.Equal(t, tc.wantText, stderr.String()) }) } @@ -676,24 +486,24 @@ func TestMaybeHint_NativeHintThrottledPerSession(t *testing.T) { ctx = env.Set(ctx, "DATABRICKS_CACHE_ENABLED", "true") ctx = env.Set(ctx, claudeSessionEnv, "session-abc") ctx = env.Set(ctx, claudeChildSessionEnv, "1") - return telemetry.WithNewLogger(ctx), stderr + return ctx, stderr } ctx1, stderr1 := newCtx() - MaybeHint(ctx1, airCmd(), noPluginRecorded) - assert.Equal(t, wantNativeHint+"\n"+airHintText+"\n", stderr1.String()) + MaybeHint(ctx1, regularCmd(), noPluginRecorded) + assert.Equal(t, wantNativeHint+"\n"+hintText+"\n", stderr1.String()) // Same session, second command: both lines stay quiet, so the tag doesn't // reach a downstream reader such as `2>&1 | jq` on every command. ctx2, stderr2 := newCtx() - MaybeHint(ctx2, airCmd(), noPluginRecorded) + MaybeHint(ctx2, regularCmd(), noPluginRecorded) assert.Empty(t, stderr2.String()) // Another session: both lines print again. ctx3, stderr3 := newCtx() ctx3 = env.Set(ctx3, claudeSessionEnv, "session-def") - MaybeHint(ctx3, airCmd(), noPluginRecorded) - assert.Equal(t, wantNativeHint+"\n"+airHintText+"\n", stderr3.String()) + MaybeHint(ctx3, regularCmd(), noPluginRecorded) + assert.Equal(t, wantNativeHint+"\n"+hintText+"\n", stderr3.String()) } func TestMaybeHint_NativeHintSilentOnTerminal(t *testing.T) { @@ -729,8 +539,8 @@ func TestMaybeHint_NativeHintSilentOnTerminal(t *testing.T) { ctx = env.Set(ctx, "DATABRICKS_CACHE_DIR", t.TempDir()) ctx = env.Set(ctx, claudeChildSessionEnv, "1") - MaybeHint(ctx, airCmd(), noPluginRecorded) - assert.Equal(t, airHintText+"\n", stderr.String()) + MaybeHint(ctx, regularCmd(), noPluginRecorded) + assert.Equal(t, hintText+"\n", stderr.String()) }) } } @@ -743,7 +553,7 @@ func TestAgentHint_NativeLineNilIsEmpty(t *testing.T) { func TestHintForAgent(t *testing.T) { claude := hintForAgent(claudeAgentProvider, noPluginRecorded) require.NotNil(t, claude) - assert.Equal(t, airHintText, claude.text) + assert.Equal(t, hintText, claude.text) assert.Equal(t, claudeSessionEnv, claude.sessionEnv) assert.Nil(t, hintForAgent("", noPluginRecorded)) @@ -774,7 +584,7 @@ func TestMaybeHint_LogsEventsOncePerSession(t *testing.T) { } ctx1, _ := newCtx(true) - MaybeHint(ctx1, airCmd(), noPluginRecorded) + MaybeHint(ctx1, regularCmd(), noPluginRecorded) assert.Equal(t, []protos.AitoolsHintKind{ protos.AitoolsHintKindClaudeCodeNative, protos.AitoolsHintKindInstallText, @@ -785,13 +595,13 @@ func TestMaybeHint_LogsEventsOncePerSession(t *testing.T) { // Throttled for the session: nothing prints and nothing logs, so the // command doesn't pay for a telemetry upload at exit. ctx2, stderr2 := newCtx(true) - MaybeHint(ctx2, airCmd(), noPluginRecorded) + MaybeHint(ctx2, regularCmd(), noPluginRecorded) assert.Empty(t, stderr2.String()) assert.Empty(t, telemetry.BufferedEvents(ctx2)) // Throttled and not a Claude Code subprocess: nothing prints, nothing logs. ctx3, stderr3 := newCtx(false) - MaybeHint(ctx3, airCmd(), noPluginRecorded) + MaybeHint(ctx3, regularCmd(), noPluginRecorded) assert.Empty(t, stderr3.String()) assert.Empty(t, telemetry.BufferedEvents(ctx3)) } @@ -804,6 +614,259 @@ func TestMaybeHint_InstalledLogsNothing(t *testing.T) { ctx = env.Set(ctx, claudeChildSessionEnv, "1") writePluginManifest(t, home, `{"plugins":{"databricks@claude-plugins-official":[{"scope":"user"}]}}`) - MaybeHint(ctx, airCmd(), noPluginRecorded) + MaybeHint(ctx, regularCmd(), noPluginRecorded) assert.Empty(t, telemetry.BufferedEvents(ctx)) } + +// --- AI Runtime skill hint on air subcommands --- + +// commandAt builds the command tree for a space-separated path under a +// "databricks" root and returns the leaf, so ancestry checks see the same +// shape as the real CLI. +func commandAt(path string) *cobra.Command { + cmd := &cobra.Command{Use: "databricks"} + for name := range strings.FieldsSeq(path) { + child := &cobra.Command{Use: name} + cmd.AddCommand(child) + cmd = child + } + return cmd +} + +func airCmd() *cobra.Command { return commandAt("air list") } + +// writeProjectPluginManifest writes a manifest with one project-scope install +// covering projectPath. Marshaled, not interpolated, so Windows paths with +// backslashes stay valid JSON. +func writeProjectPluginManifest(t *testing.T, home, projectPath string) { + manifest, err := json.Marshal(map[string]any{ + "plugins": map[string]any{ + "databricks@claude-plugins-official": []any{map[string]string{ + "scope": "project", + "projectPath": projectPath, + }}, + }, + }) + require.NoError(t, err) + writePluginManifest(t, home, string(manifest)) +} + +// assertAIRHinted checks for exactly the AI Runtime skill line and one +// hint-shown event of kind INSTALL_TEXT. +func assertAIRHinted(t *testing.T, ctx context.Context, stderr *bytes.Buffer) { + t.Helper() + assert.Equal(t, airHintText+"\n", stderr.String()) + assert.Equal(t, []protos.AitoolsHintKind{protos.AitoolsHintKindInstallText}, hintKinds(t, ctx)) +} + +// assertNothingHinted checks for no hint line and no telemetry event. +func assertNothingHinted(t *testing.T, ctx context.Context, stderr *bytes.Buffer) { + t.Helper() + assert.Empty(t, stderr.String()) + assert.Empty(t, hintKinds(t, ctx)) +} + +func TestAIRHintText(t *testing.T) { + // The hint names the skill, says what it is for, and offers both the install + // and a no-install preview, on one line. + assert.Contains(t, airHintText, "install the databricks-ai-runtime skill") + assert.Contains(t, airHintText, "workload configuration and run debugging") + assert.Contains(t, airHintText, "databricks aitools install --skills-only --skills databricks-ai-runtime") + assert.Contains(t, airHintText, "replace --skills-only with --path ") + assert.NotContains(t, airHintText, "\n") +} + +// TestMaybeHint_NonAIRCommandsKeepGenericHint pins the scoping: outside air, +// Claude Code gets the generic install hint it always got, never the AI +// Runtime skill line. +func TestMaybeHint_NonAIRCommandsKeepGenericHint(t *testing.T) { + setClaudeAgent(t) + for _, path := range []string{ + "auth profiles", "current-user me", "clusters list", "jobs list", + "bundle deploy", "workspace list", "version", + // The bare air node only renders help (the footer carries the tip), and + // the hidden alias points users at `databricks air`; neither is an air + // subcommand, so they keep the generic hint. + "air", + "experimental air", + } { + t.Run(path, func(t *testing.T) { + ctx, stderr := claudeCtx(t, t.TempDir()) + ctx = telemetry.WithNewLogger(ctx) + MaybeHint(ctx, commandAt(path), noPluginRecorded) + assert.Equal(t, hintText+"\n", stderr.String()) + assert.Equal(t, []protos.AitoolsHintKind{protos.AitoolsHintKindInstallText}, hintKinds(t, ctx)) + assert.NotContains(t, stderr.String(), airHintText) + }) + } +} + +// Every runnable air subcommand, at any depth, gets the skill-specific hint. +func TestMaybeHint_AIRSubcommandsHint(t *testing.T) { + setClaudeAgent(t) + for _, path := range []string{ + "air run", "air get", "air list", "air logs", "air cancel", + "air convert-to-dabs", "air images", "air images push", "air pools", "air pools list", + } { + t.Run(path, func(t *testing.T) { + ctx, stderr := claudeCtx(t, t.TempDir()) + ctx = telemetry.WithNewLogger(ctx) + MaybeHint(ctx, commandAt(path), noPluginRecorded) + assertAIRHinted(t, ctx, stderr) + }) + } +} + +// Only the databricks-ai-runtime skill itself, in a location Claude Code loads +// for this working directory, silences the skill hint. +func TestMaybeHint_AIRSkillDetection(t *testing.T) { + for _, tc := range []struct { + name string + setup func(t *testing.T, home, cwd string) + wantHint bool + }{ + {"nothing installed", func(t *testing.T, home, cwd string) {}, true}, + {"skill in Claude's own skills dir", func(t *testing.T, home, cwd string) { + writeClaudeSkill(t, home, airSkillName) + }, false}, + {"skill in the project skills dir", func(t *testing.T, home, cwd string) { + require.NoError(t, os.MkdirAll(filepath.Join(cwd, ".claude", "skills", airSkillName), 0o755)) + }, false}, + {"other databricks skills only", func(t *testing.T, home, cwd string) { + writeClaudeSkill(t, home, "databricks") + writeClaudeSkill(t, home, "databricks-jobs") + }, true}, + {"a file named like the skill", func(t *testing.T, home, cwd string) { + dir := filepath.Join(home, ".claude", "skills") + require.NoError(t, os.MkdirAll(dir, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, airSkillName), nil, 0o600)) + }, true}, + {"shared canonical store", func(t *testing.T, home, cwd string) { + writeCanonicalSkill(t, home, airSkillName) + }, true}, + {"shared legacy store", func(t *testing.T, home, cwd string) { + require.NoError(t, os.MkdirAll(filepath.Join(home, LegacySkillsDir, airSkillName), 0o755)) + }, true}, + {"plugin for this project", func(t *testing.T, home, cwd string) { + writeProjectPluginManifest(t, home, cwd) + }, false}, + {"plugin for an unrelated project", func(t *testing.T, home, cwd string) { + writeProjectPluginManifest(t, home, t.TempDir()) + }, true}, + } { + t.Run(tc.name, func(t *testing.T) { + // A fresh project per case: the project-skills fixtures write into + // the working directory and must not leak between cases. + setClaudeAgent(t) + home := t.TempDir() + ctx, stderr := claudeCtx(t, home) + ctx = telemetry.WithNewLogger(ctx) + cwd, err := os.Getwd() + require.NoError(t, err) + tc.setup(t, home, cwd) + + MaybeHint(ctx, airCmd(), noPluginRecorded) + if tc.wantHint { + assertAIRHinted(t, ctx, stderr) + } else { + assertNothingHinted(t, ctx, stderr) + } + }) + } +} + +// A project-scoped skill at the repository root still counts when the CLI runs +// in a subdirectory, as Claude Code loads it there. +func TestMaybeHint_AIRSkillInRepositoryRoot(t *testing.T) { + setClaudeAgent(t) + home := t.TempDir() + ctx, stderr := claudeCtx(t, home) + ctx = telemetry.WithNewLogger(ctx) + root, err := os.Getwd() + require.NoError(t, err) + require.NoError(t, os.MkdirAll(filepath.Join(root, ".claude", "skills", airSkillName), 0o755)) + sub := filepath.Join(root, "training", "configs") + require.NoError(t, os.MkdirAll(sub, 0o755)) + t.Chdir(sub) + + MaybeHint(ctx, airCmd(), noPluginRecorded) + assertNothingHinted(t, ctx, stderr) +} + +func TestAIRSkillAvailable_LogsPluginDetection(t *testing.T) { + // Clears the host's CLAUDE_CONFIG_DIR, which would move the manifest. + testutil.SetAgentEnv(t) + t.Chdir(t.TempDir()) + home := t.TempDir() + writePluginManifest(t, home, `{"plugins":{"databricks@claude-plugins-official":[{"version":"0.2.6"}]}}`) + + // Capture debug output by installing a text logger on the context. + var buf bytes.Buffer + handler := slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: log.LevelDebug}) + ctx := log.NewContext(t.Context(), slog.New(handler)) + ctx = env.WithUserHomeDir(ctx, home) + + assert.True(t, airSkillAvailable(ctx)) + + assert.Contains(t, buf.String(), "Databricks plugin active for Claude Code") + assert.Contains(t, buf.String(), "skipping AI Runtime skill hint") +} + +// With a session id the first hinted command is the only one; without one +// (older Claude Code) every air command is hinted. The throttle is shared with +// the generic hint, so a hint already shown on a non-air command also silences +// the air one for the rest of the session. +func TestMaybeHint_AIRSessionThrottle(t *testing.T) { + setClaudeAgent(t) + home := t.TempDir() + cacheDir := t.TempDir() + newCtx := func() (context.Context, *bytes.Buffer) { + ctx, stderr := claudeCtx(t, home) + ctx = env.Set(ctx, "DATABRICKS_CACHE_DIR", cacheDir) + ctx = env.Set(ctx, claudeSessionEnv, "session-abc") + ctx = telemetry.WithNewLogger(ctx) + return ctx, stderr + } + + ctx1, stderr1 := newCtx() + MaybeHint(ctx1, airCmd(), noPluginRecorded) + assertAIRHinted(t, ctx1, stderr1) + + // Same session, second air command: stays quiet. + ctx2, stderr2 := newCtx() + MaybeHint(ctx2, commandAt("air run"), noPluginRecorded) + assertNothingHinted(t, ctx2, stderr2) + + // Another session: the air hint prints again. + ctx3, stderr3 := newCtx() + ctx3 = env.Set(ctx3, claudeSessionEnv, "session-def") + MaybeHint(ctx3, commandAt("air run"), noPluginRecorded) + assertAIRHinted(t, ctx3, stderr3) + + // A generic hint shown on a non-air command consumes the same session's + // marker, so a later air command stays quiet. + ctx4, stderr4 := newCtx() + ctx4 = env.Set(ctx4, claudeSessionEnv, "session-ghi") + MaybeHint(ctx4, commandAt("auth profiles"), noPluginRecorded) + assert.Equal(t, hintText+"\n", stderr4.String()) + ctx5, stderr5 := newCtx() + ctx5 = env.Set(ctx5, claudeSessionEnv, "session-ghi") + MaybeHint(ctx5, airCmd(), noPluginRecorded) + assertNothingHinted(t, ctx5, stderr5) +} + +func TestHintForCommand(t *testing.T) { + air := hintForCommand(airCmd(), claudeAgentProvider, noPluginRecorded) + require.NotNil(t, air) + assert.Equal(t, airHintText, air.text) + + // Outside air Claude Code keeps the generic hint. + regular := hintForCommand(regularCmd(), claudeAgentProvider, noPluginRecorded) + require.NotNil(t, regular) + assert.Equal(t, hintText, regular.text) + + // Other agents are untouched on air subcommands. + cursor := hintForCommand(airCmd(), "cursor", noPluginRecorded) + require.NotNil(t, cursor) + assert.Equal(t, agentInstallHint(ByName(NameCursor), noPluginRecorded).text, cursor.text) +} diff --git a/libs/aitools/agents/recommend.go b/libs/aitools/agents/recommend.go index bfe9bcf641a..dfbefa435f8 100644 --- a/libs/aitools/agents/recommend.go +++ b/libs/aitools/agents/recommend.go @@ -24,9 +24,9 @@ func PrintSkillsTip(ctx context.Context) { // MaybeVersionCheckTip prints skillsTipText after `databricks version --check` // when a coding agent is detected without Databricks tooling. Callers that // MaybeHint hints (Claude Code and the registry agents, see isHintedAgent) are -// skipped: MaybeHint owns their install guidance, scoped to `databricks air` -// subcommands with its own throttle, and a second, different command in the -// same run would compete with it. installedPlugins is a parameter because this package +// skipped: the root hook already names the install command for them, with its +// own throttle, and a second, different command in the same run would compete +// with it. installedPlugins is a parameter because this package // cannot import the installer; callers pass installer.InstalledPluginAgents, // which counts a plugin recorded in the CLI install state with or without a // known version. No telemetry is logged: `databricks version` never resolves