From 2ca4742c5be6a3617726cc3edc512287886f08c5 Mon Sep 17 00:00:00 2001 From: Chris Stephens Date: Wed, 18 Nov 2026 18:06:29 +0000 Subject: [PATCH 1/2] Add a hidden --skills-invoked flag for experimental skill tracking Agents prefixing CLI calls with DATABRICKS_SKILLS_INVOKED=... no longer match permission allowlists such as Claude Code's Bash(databricks:*), so the command needs approval or is denied in headless runs. A flag keeps the command starting with "databricks". The flag is hidden from help and completion, wins over the env var when set, and is ignored when telemetry is disabled. Co-authored-by: Isaac --- acceptance/cmd/skills-invoked/output.txt | 25 ++++++++++++++++ acceptance/cmd/skills-invoked/script | 19 ++++++++++++ cmd/root/export_test.go | 1 + cmd/root/root.go | 3 +- cmd/root/user_agent_skills.go | 38 +++++++++++++++++------- cmd/root/user_agent_skills_test.go | 23 ++++++++++---- 6 files changed, 91 insertions(+), 18 deletions(-) diff --git a/acceptance/cmd/skills-invoked/output.txt b/acceptance/cmd/skills-invoked/output.txt index 6f17ed00b8a..50d0025864a 100644 --- a/acceptance/cmd/skills-invoked/output.txt +++ b/acceptance/cmd/skills-invoked/output.txt @@ -34,3 +34,28 @@ >>> [CLI] current-user me >>> print_requests.py --get //scim/v2/Me + +=== Hidden flag adds the skills, so commands still start with databricks + +>>> [CLI] current-user me --skills-invoked databricks-core,databricks-jobs + +>>> print_requests.py --get //scim/v2/Me + +=== Flag wins over env var + +>>> [CLI] current-user me --skills-invoked from-flag + +>>> print_requests.py --get //scim/v2/Me + +=== No skills from the flag when telemetry is disabled + +>>> [CLI] current-user me --skills-invoked databricks-core + +>>> print_requests.py --get //scim/v2/Me + +=== Flag is hidden from help and completion + +>>> [CLI] current-user me --help + +>>> [CLI] __complete current-user me --sk +Completion ended with directive: ShellCompDirectiveNoFileComp diff --git a/acceptance/cmd/skills-invoked/script b/acceptance/cmd/skills-invoked/script index 12c530226a1..918f4141abd 100644 --- a/acceptance/cmd/skills-invoked/script +++ b/acceptance/cmd/skills-invoked/script @@ -31,3 +31,22 @@ title "No skills when telemetry is disabled" DATABRICKS_SKILLS_INVOKED=databricks-core DATABRICKS_CLI_DISABLE_TELEMETRY=1 trace $CLI current-user me > /dev/null trace print_requests.py --get //scim/v2/Me > LOG.disabled contains.py "!skill/" < LOG.disabled > /dev/null + +title "Hidden flag adds the skills, so commands still start with databricks" +trace $CLI current-user me --skills-invoked databricks-core,databricks-jobs > /dev/null +trace print_requests.py --get //scim/v2/Me > LOG.flag +contains.py " skill/databricks-core skill/databricks-jobs" < LOG.flag > /dev/null + +title "Flag wins over env var" +DATABRICKS_SKILLS_INVOKED=from-env trace $CLI current-user me --skills-invoked from-flag > /dev/null +trace print_requests.py --get //scim/v2/Me > LOG.flag-wins +contains.py " skill/from-flag" "!skill/from-env" < LOG.flag-wins > /dev/null + +title "No skills from the flag when telemetry is disabled" +DATABRICKS_CLI_DISABLE_TELEMETRY=1 trace $CLI current-user me --skills-invoked databricks-core > /dev/null +trace print_requests.py --get //scim/v2/Me > LOG.flag-disabled +contains.py "!skill/" < LOG.flag-disabled > /dev/null + +title "Flag is hidden from help and completion" +trace $CLI current-user me --help | contains.py "!skills-invoked" > /dev/null +trace $CLI __complete current-user me --sk | contains.py "!skills-invoked" > /dev/null diff --git a/cmd/root/export_test.go b/cmd/root/export_test.go index b220e086b90..efb693072cf 100644 --- a/cmd/root/export_test.go +++ b/cmd/root/export_test.go @@ -11,6 +11,7 @@ var ( ParseSkills = parseSkills WithSkillsInUserAgent = withSkillsInUserAgent SkillsInvokedEnvVar = skillsInvokedEnvVar + InitSkillsInvokedFlag = initSkillsInvokedFlag MaxSkillsUserAgentLength = maxSkillsUserAgentLength ) diff --git a/cmd/root/root.go b/cmd/root/root.go index 0f1c9a90515..1a9ca7c62e6 100644 --- a/cmd/root/root.go +++ b/cmd/root/root.go @@ -51,6 +51,7 @@ func newRootCommand(ctx context.Context) *cobra.Command { initProfileFlag(cmd) initEnvironmentFlag(cmd) initTargetFlag(cmd) + initSkillsInvokedFlag(cmd) // Deprecated flag. Warn if it is specified. initProgressLoggerFlag(cmd, logFlags) @@ -88,7 +89,7 @@ func newRootCommand(ctx context.Context) *cobra.Command { ctx = installer.WithAiDevKitInUserAgent(ctx) ctx = installer.WithCallingAgentToolingInUserAgent(ctx) ctx = InjectTestPidToUserAgent(ctx) - ctx = withSkillsInUserAgent(ctx) + ctx = withSkillsInUserAgent(ctx, cmd) cmd.SetContext(ctx) // Recommend installing Databricks AI tooling to the calling coding agent diff --git a/cmd/root/user_agent_skills.go b/cmd/root/user_agent_skills.go index 4892d07295b..6f7447db6d0 100644 --- a/cmd/root/user_agent_skills.go +++ b/cmd/root/user_agent_skills.go @@ -11,16 +11,23 @@ import ( "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/telemetry" "github.com/databricks/databricks-sdk-go/useragent" + "github.com/spf13/cobra" ) // Experimental skill-usage tracking: the databricks-core agent skill // (https://github.com/databricks/databricks-agent-skills/tree/main/skills/databricks-core) -// tells agents to prefix CLI calls with this env var, set to the comma-separated -// bare names of the skills they loaded. It may change or be removed without -// notice. It is an env var rather than a flag so removing it doesn't change the -// CLI's interface, and because older CLIs ignore unknown env vars but reject -// unknown flags. -const skillsInvokedEnvVar = "DATABRICKS_SKILLS_INVOKED" +// tells agents to pass the comma-separated bare names of the skills they loaded. +// It may change or be removed without notice. +// +// skillsInvokedFlag is what agents use: a command that still starts with +// "databricks" keeps matching agent permission allowlists such as Claude Code's +// Bash(databricks:*), which an env var prefix does not. It is hidden from help +// and completion so it isn't part of the advertised interface. The env var is +// kept for callers that set it in the environment. +const ( + skillsInvokedFlag = "skills-invoked" + skillsInvokedEnvVar = "DATABRICKS_SKILLS_INVOKED" +) // skillUserAgentKey is repeated once per skill: "skill/databricks-core skill/databricks-jobs". const skillUserAgentKey = "skill" @@ -67,14 +74,23 @@ func parseSkills(ctx context.Context, raw string) []string { return skills } -// withSkillsInUserAgent adds one skill/ pair per skill in -// skillsInvokedEnvVar (experimental tracking), unless the user opted out of -// telemetry. -func withSkillsInUserAgent(ctx context.Context) context.Context { +func initSkillsInvokedFlag(cmd *cobra.Command) { + cmd.PersistentFlags().String(skillsInvokedFlag, "", "comma-separated agent skills that invoked this command") + cmd.PersistentFlags().MarkHidden(skillsInvokedFlag) +} + +// withSkillsInUserAgent adds one skill/ pair per skill from the +// skillsInvokedFlag flag, or skillsInvokedEnvVar when the flag isn't set +// (experimental tracking), unless the user opted out of telemetry. +func withSkillsInUserAgent(ctx context.Context, cmd *cobra.Command) context.Context { if telemetry.Disabled(ctx) { return ctx } - for _, skill := range parseSkills(ctx, env.Get(ctx, skillsInvokedEnvVar)) { + raw := env.Get(ctx, skillsInvokedEnvVar) + if f := cmd.Flag(skillsInvokedFlag); f.Changed { + raw = f.Value.String() + } + for _, skill := range parseSkills(ctx, raw) { // Each skill adds its own pair to the same context; this is accumulation, // not the accidental nesting fatcontext flags. ctx = useragent.InContext(ctx, skillUserAgentKey, skill) //nolint:fatcontext diff --git a/cmd/root/user_agent_skills_test.go b/cmd/root/user_agent_skills_test.go index fee8a9ae864..f1b64a050a2 100644 --- a/cmd/root/user_agent_skills_test.go +++ b/cmd/root/user_agent_skills_test.go @@ -8,7 +8,9 @@ import ( "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/env" "github.com/databricks/databricks-sdk-go/useragent" + "github.com/spf13/cobra" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) // skillNames returns n distinct skill names of exactly length bytes. @@ -72,20 +74,29 @@ func TestParseSkills(t *testing.T) { func TestWithSkillsInUserAgent(t *testing.T) { tests := []struct { name string - skills string + env string + args []string disableTelemetry string want string }{ - {"unset", "", "", ""}, - {"set", "databricks-core,databricks-jobs", "", "skill/databricks-core skill/databricks-jobs"}, - {"telemetry disabled", "databricks-core", "1", ""}, + {"unset", "", nil, "", ""}, + {"env var", "databricks-core,databricks-jobs", nil, "", "skill/databricks-core skill/databricks-jobs"}, + {"flag", "", []string{"--skills-invoked", "databricks-core,databricks-jobs"}, "", "skill/databricks-core skill/databricks-jobs"}, + {"flag wins over env var", "from-env", []string{"--skills-invoked", "from-flag"}, "", "skill/from-flag"}, + {"empty flag still wins over env var", "from-env", []string{"--skills-invoked="}, "", ""}, + {"telemetry disabled with env var", "databricks-core", nil, "1", ""}, + {"telemetry disabled with flag", "", []string{"--skills-invoked", "databricks-core"}, "1", ""}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + cmd := &cobra.Command{Use: "test"} + root.InitSkillsInvokedFlag(cmd) + require.NoError(t, cmd.ParseFlags(tt.args)) + // Always set both: an empty value in the context overrides the host environment. - ctx := env.Set(t.Context(), root.SkillsInvokedEnvVar, tt.skills) + ctx := env.Set(t.Context(), root.SkillsInvokedEnvVar, tt.env) ctx = env.Set(ctx, "DATABRICKS_CLI_DISABLE_TELEMETRY", tt.disableTelemetry) - ua := useragent.FromContext(root.WithSkillsInUserAgent(ctx)) + ua := useragent.FromContext(root.WithSkillsInUserAgent(ctx, cmd)) if tt.want == "" { assert.NotContains(t, ua, "skill/") } else { From e4d58d00c2008f2a63fcbb58259109dd487a72eb Mon Sep 17 00:00:00 2001 From: Chris Stephens Date: Wed, 18 Nov 2026 18:38:25 +0000 Subject: [PATCH 2/2] Address review nits on --skills-invoked Debug messages no longer name DATABRICKS_SKILLS_INVOKED, since the value can come from the flag too. Add a doc comment to initSkillsInvokedFlag. Co-authored-by: Isaac --- cmd/root/user_agent_skills.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cmd/root/user_agent_skills.go b/cmd/root/user_agent_skills.go index 6f7447db6d0..6d86cb5445e 100644 --- a/cmd/root/user_agent_skills.go +++ b/cmd/root/user_agent_skills.go @@ -62,10 +62,10 @@ func parseSkills(ctx context.Context, raw string) []string { switch { case name == "": case len(name) > maxSkillLength || !skillNamePattern().MatchString(name): - log.Debugf(ctx, "dropping invalid %s entry %q", skillsInvokedEnvVar, entry) + log.Debugf(ctx, "dropping invalid skill entry %q", entry) case slices.Contains(skills, name): case size+pairSize > maxSkillsUserAgentLength: - log.Debugf(ctx, "dropping %s entry %q: skills exceed %d user agent bytes", skillsInvokedEnvVar, name, maxSkillsUserAgentLength) + log.Debugf(ctx, "dropping skill entry %q: skills exceed %d user agent bytes", name, maxSkillsUserAgentLength) default: skills = append(skills, name) size += pairSize @@ -74,6 +74,7 @@ func parseSkills(ctx context.Context, raw string) []string { return skills } +// initSkillsInvokedFlag registers the hidden skillsInvokedFlag on cmd. func initSkillsInvokedFlag(cmd *cobra.Command) { cmd.PersistentFlags().String(skillsInvokedFlag, "", "comma-separated agent skills that invoked this command") cmd.PersistentFlags().MarkHidden(skillsInvokedFlag)