Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 25 additions & 0 deletions acceptance/cmd/skills-invoked/output.txt
Original file line number Diff line number Diff line change
Expand Up @@ -34,3 +34,28 @@
>>> [CLI] current-user me

>>> print_requests.py --get //scim/v2/Me

=== Hidden flag adds the skills, so commands still start with databricks

>>> [CLI] current-user me --skills-invoked databricks-core,databricks-jobs

>>> print_requests.py --get //scim/v2/Me

=== Flag wins over env var

>>> [CLI] current-user me --skills-invoked from-flag

>>> print_requests.py --get //scim/v2/Me

=== No skills from the flag when telemetry is disabled

>>> [CLI] current-user me --skills-invoked databricks-core

>>> print_requests.py --get //scim/v2/Me

=== Flag is hidden from help and completion

>>> [CLI] current-user me --help

>>> [CLI] __complete current-user me --sk
Completion ended with directive: ShellCompDirectiveNoFileComp
19 changes: 19 additions & 0 deletions acceptance/cmd/skills-invoked/script
Original file line number Diff line number Diff line change
Expand Up @@ -31,3 +31,22 @@ title "No skills when telemetry is disabled"
DATABRICKS_SKILLS_INVOKED=databricks-core DATABRICKS_CLI_DISABLE_TELEMETRY=1 trace $CLI current-user me > /dev/null
trace print_requests.py --get //scim/v2/Me > LOG.disabled
contains.py "!skill/" < LOG.disabled > /dev/null

title "Hidden flag adds the skills, so commands still start with databricks"
trace $CLI current-user me --skills-invoked databricks-core,databricks-jobs > /dev/null
trace print_requests.py --get //scim/v2/Me > LOG.flag
contains.py " skill/databricks-core skill/databricks-jobs" < LOG.flag > /dev/null

title "Flag wins over env var"
DATABRICKS_SKILLS_INVOKED=from-env trace $CLI current-user me --skills-invoked from-flag > /dev/null
trace print_requests.py --get //scim/v2/Me > LOG.flag-wins
contains.py " skill/from-flag" "!skill/from-env" < LOG.flag-wins > /dev/null

title "No skills from the flag when telemetry is disabled"
DATABRICKS_CLI_DISABLE_TELEMETRY=1 trace $CLI current-user me --skills-invoked databricks-core > /dev/null
trace print_requests.py --get //scim/v2/Me > LOG.flag-disabled
contains.py "!skill/" < LOG.flag-disabled > /dev/null

title "Flag is hidden from help and completion"
trace $CLI current-user me --help | contains.py "!skills-invoked" > /dev/null
trace $CLI __complete current-user me --sk | contains.py "!skills-invoked" > /dev/null
1 change: 1 addition & 0 deletions cmd/root/export_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ var (
ParseSkills = parseSkills
WithSkillsInUserAgent = withSkillsInUserAgent
SkillsInvokedEnvVar = skillsInvokedEnvVar
InitSkillsInvokedFlag = initSkillsInvokedFlag
MaxSkillsUserAgentLength = maxSkillsUserAgentLength
)

Expand Down
3 changes: 2 additions & 1 deletion cmd/root/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ func newRootCommand(ctx context.Context) *cobra.Command {
initProfileFlag(cmd)
initEnvironmentFlag(cmd)
initTargetFlag(cmd)
initSkillsInvokedFlag(cmd)

// Deprecated flag. Warn if it is specified.
initProgressLoggerFlag(cmd, logFlags)
Expand Down Expand Up @@ -88,7 +89,7 @@ func newRootCommand(ctx context.Context) *cobra.Command {
ctx = installer.WithAiDevKitInUserAgent(ctx)
ctx = installer.WithCallingAgentToolingInUserAgent(ctx)
ctx = InjectTestPidToUserAgent(ctx)
ctx = withSkillsInUserAgent(ctx)
ctx = withSkillsInUserAgent(ctx, cmd)
cmd.SetContext(ctx)

// Recommend installing Databricks AI tooling to the calling coding agent
Expand Down
43 changes: 30 additions & 13 deletions cmd/root/user_agent_skills.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,16 +11,23 @@ import (
"github.com/databricks/cli/libs/log"
"github.com/databricks/cli/libs/telemetry"
"github.com/databricks/databricks-sdk-go/useragent"
"github.com/spf13/cobra"
)

// Experimental skill-usage tracking: the databricks-core agent skill
// (https://github.com/databricks/databricks-agent-skills/tree/main/skills/databricks-core)
// tells agents to prefix CLI calls with this env var, set to the comma-separated
// bare names of the skills they loaded. It may change or be removed without
// notice. It is an env var rather than a flag so removing it doesn't change the
// CLI's interface, and because older CLIs ignore unknown env vars but reject
// unknown flags.
const skillsInvokedEnvVar = "DATABRICKS_SKILLS_INVOKED"
// tells agents to pass the comma-separated bare names of the skills they loaded.
// It may change or be removed without notice.
//
// skillsInvokedFlag is what agents use: a command that still starts with
// "databricks" keeps matching agent permission allowlists such as Claude Code's
// Bash(databricks:*), which an env var prefix does not. It is hidden from help
// and completion so it isn't part of the advertised interface. The env var is
// kept for callers that set it in the environment.
const (
skillsInvokedFlag = "skills-invoked"
skillsInvokedEnvVar = "DATABRICKS_SKILLS_INVOKED"
)

// skillUserAgentKey is repeated once per skill: "skill/databricks-core skill/databricks-jobs".
const skillUserAgentKey = "skill"
Expand Down Expand Up @@ -55,10 +62,10 @@ func parseSkills(ctx context.Context, raw string) []string {
switch {
case name == "":
case len(name) > maxSkillLength || !skillNamePattern().MatchString(name):
log.Debugf(ctx, "dropping invalid %s entry %q", skillsInvokedEnvVar, entry)
log.Debugf(ctx, "dropping invalid skill entry %q", entry)
case slices.Contains(skills, name):
case size+pairSize > maxSkillsUserAgentLength:
log.Debugf(ctx, "dropping %s entry %q: skills exceed %d user agent bytes", skillsInvokedEnvVar, name, maxSkillsUserAgentLength)
log.Debugf(ctx, "dropping skill entry %q: skills exceed %d user agent bytes", name, maxSkillsUserAgentLength)
default:
skills = append(skills, name)
size += pairSize
Expand All @@ -67,14 +74,24 @@ func parseSkills(ctx context.Context, raw string) []string {
return skills
}

// withSkillsInUserAgent adds one skill/<name> pair per skill in
// skillsInvokedEnvVar (experimental tracking), unless the user opted out of
// telemetry.
func withSkillsInUserAgent(ctx context.Context) context.Context {
// initSkillsInvokedFlag registers the hidden skillsInvokedFlag on cmd.
func initSkillsInvokedFlag(cmd *cobra.Command) {

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[nit] Please add a brief doc comment for initSkillsInvokedFlag, as required by .agents/rules/comments.md (starting with the function name and ending with a period).

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Added in e4d58d0.

cmd.PersistentFlags().String(skillsInvokedFlag, "", "comma-separated agent skills that invoked this command")
cmd.PersistentFlags().MarkHidden(skillsInvokedFlag)
}

// withSkillsInUserAgent adds one skill/<name> pair per skill from the
// skillsInvokedFlag flag, or skillsInvokedEnvVar when the flag isn't set
// (experimental tracking), unless the user opted out of telemetry.
func withSkillsInUserAgent(ctx context.Context, cmd *cobra.Command) context.Context {
if telemetry.Disabled(ctx) {
return ctx
}
for _, skill := range parseSkills(ctx, env.Get(ctx, skillsInvokedEnvVar)) {
raw := env.Get(ctx, skillsInvokedEnvVar)
if f := cmd.Flag(skillsInvokedFlag); f.Changed {
raw = f.Value.String()
}
for _, skill := range parseSkills(ctx, raw) {

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[nit] Both debug messages in parseSkills still name DATABRICKS_SKILLS_INVOKED. With --skills-invoked='bad name!' and no env var set, the diagnostic points at the wrong source. Could these say "skill entry" now that the helper parses either input?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed in e4d58d0: both messages now say "skill entry" without naming the source.

// Each skill adds its own pair to the same context; this is accumulation,
// not the accidental nesting fatcontext flags.
ctx = useragent.InContext(ctx, skillUserAgentKey, skill) //nolint:fatcontext
Expand Down
23 changes: 17 additions & 6 deletions cmd/root/user_agent_skills_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,9 @@ import (
"github.com/databricks/cli/cmd/root"
"github.com/databricks/cli/libs/env"
"github.com/databricks/databricks-sdk-go/useragent"
"github.com/spf13/cobra"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)

// skillNames returns n distinct skill names of exactly length bytes.
Expand Down Expand Up @@ -72,20 +74,29 @@ func TestParseSkills(t *testing.T) {
func TestWithSkillsInUserAgent(t *testing.T) {
tests := []struct {
name string
skills string
env string
args []string
disableTelemetry string
want string
}{
{"unset", "", "", ""},
{"set", "databricks-core,databricks-jobs", "", "skill/databricks-core skill/databricks-jobs"},
{"telemetry disabled", "databricks-core", "1", ""},
{"unset", "", nil, "", ""},
{"env var", "databricks-core,databricks-jobs", nil, "", "skill/databricks-core skill/databricks-jobs"},
{"flag", "", []string{"--skills-invoked", "databricks-core,databricks-jobs"}, "", "skill/databricks-core skill/databricks-jobs"},
{"flag wins over env var", "from-env", []string{"--skills-invoked", "from-flag"}, "", "skill/from-flag"},
{"empty flag still wins over env var", "from-env", []string{"--skills-invoked="}, "", ""},
{"telemetry disabled with env var", "databricks-core", nil, "1", ""},
{"telemetry disabled with flag", "", []string{"--skills-invoked", "databricks-core"}, "1", ""},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
cmd := &cobra.Command{Use: "test"}
root.InitSkillsInvokedFlag(cmd)
require.NoError(t, cmd.ParseFlags(tt.args))

// Always set both: an empty value in the context overrides the host environment.
ctx := env.Set(t.Context(), root.SkillsInvokedEnvVar, tt.skills)
ctx := env.Set(t.Context(), root.SkillsInvokedEnvVar, tt.env)
ctx = env.Set(ctx, "DATABRICKS_CLI_DISABLE_TELEMETRY", tt.disableTelemetry)
ua := useragent.FromContext(root.WithSkillsInUserAgent(ctx))
ua := useragent.FromContext(root.WithSkillsInUserAgent(ctx, cmd))
if tt.want == "" {
assert.NotContains(t, ua, "skill/")
} else {
Expand Down
Loading