diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index 661112fef63..e8d3ad4b21f 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -293,6 +293,10 @@ func testAccept(t *testing.T, inprocessMode bool, selectedTests []string, skipTo // from the host shell would move the venv the localenv tests expect. os.Unsetenv("UV_PROJECT_ENVIRONMENT") //nolint:usetesting // t.Setenv cannot unset + // Agent skills set DATABRICKS_SKILLS_INVOKED (experimental skill tracking), + // so a host value would add skill/ pairs to recorded user agents. + os.Unsetenv("DATABRICKS_SKILLS_INVOKED") //nolint:usetesting // t.Setenv cannot unset + if !requirePrerequisites(t) { // Don't run the suite against a stale toolchain; the failed subtest // has already marked the parent test as failed. diff --git a/acceptance/cmd/skills-invoked/databricks.yml b/acceptance/cmd/skills-invoked/databricks.yml new file mode 100644 index 00000000000..a89cdadb769 --- /dev/null +++ b/acceptance/cmd/skills-invoked/databricks.yml @@ -0,0 +1,2 @@ +bundle: + name: skills-invoked diff --git a/acceptance/cmd/skills-invoked/out.test.toml b/acceptance/cmd/skills-invoked/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/cmd/skills-invoked/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/cmd/skills-invoked/output.txt b/acceptance/cmd/skills-invoked/output.txt new file mode 100644 index 00000000000..6f17ed00b8a --- /dev/null +++ b/acceptance/cmd/skills-invoked/output.txt @@ -0,0 +1,36 @@ + +=== Env var adds the skills to the user agent on a workspace GET + +>>> [CLI] current-user me + +>>> print_requests.py --get //scim/v2/Me + +=== Env var adds the skills on databricks api + +>>> [CLI] api get /api/2.1/clusters/list + +>>> print_requests.py --get //clusters/list + +=== Env var adds the skills from bundle commands + +>>> [CLI] bundle validate + +>>> print_requests.py --get //scim/v2/Me + +=== Invalid entries are dropped and the rest is normalized + +>>> [CLI] current-user me + +>>> print_requests.py --get //scim/v2/Me + +=== No skills when unset + +>>> [CLI] current-user me + +>>> print_requests.py --get //scim/v2/Me + +=== No skills when telemetry is disabled + +>>> [CLI] current-user me + +>>> print_requests.py --get //scim/v2/Me diff --git a/acceptance/cmd/skills-invoked/script b/acceptance/cmd/skills-invoked/script new file mode 100644 index 00000000000..12c530226a1 --- /dev/null +++ b/acceptance/cmd/skills-invoked/script @@ -0,0 +1,33 @@ +# Experimental skill-usage tracking: DATABRICKS_SKILLS_INVOKED adds skill/ pairs to the user agent. +# The rest of the user agent varies by machine, so each request dump goes to a LOG.* file (kept for +# diagnosis, excluded from the diff) and only the pairs are asserted. + +title "Env var adds the skills to the user agent on a workspace GET" +DATABRICKS_SKILLS_INVOKED=databricks-core,databricks-jobs trace $CLI current-user me > /dev/null +trace print_requests.py --get //scim/v2/Me > LOG.me +contains.py " skill/databricks-core skill/databricks-jobs" < LOG.me > /dev/null + +title "Env var adds the skills on databricks api" +DATABRICKS_SKILLS_INVOKED=databricks-core MSYS_NO_PATHCONV=1 trace $CLI api get /api/2.1/clusters/list > /dev/null +trace print_requests.py --get //clusters/list > LOG.api +contains.py " skill/databricks-core" < LOG.api > /dev/null + +title "Env var adds the skills from bundle commands" +DATABRICKS_SKILLS_INVOKED=databricks-dabs trace $CLI bundle validate > /dev/null +trace print_requests.py --get //scim/v2/Me > LOG.bundle +contains.py " skill/databricks-dabs" < LOG.bundle > /dev/null + +title "Invalid entries are dropped and the rest is normalized" +DATABRICKS_SKILLS_INVOKED=' Databricks-Core , bad name!,,databricks-core,databricks:databricks-jobs ' trace $CLI current-user me > /dev/null +trace print_requests.py --get //scim/v2/Me > LOG.normalized +contains.py " skill/databricks-core" "!skill/databricks-core skill/" "!bad" "!databricks:" < LOG.normalized > /dev/null + +title "No skills when unset" +trace $CLI current-user me > /dev/null +trace print_requests.py --get //scim/v2/Me > LOG.unset +contains.py "!skill/" < LOG.unset > /dev/null + +title "No skills when telemetry is disabled" +DATABRICKS_SKILLS_INVOKED=databricks-core DATABRICKS_CLI_DISABLE_TELEMETRY=1 trace $CLI current-user me > /dev/null +trace print_requests.py --get //scim/v2/Me > LOG.disabled +contains.py "!skill/" < LOG.disabled > /dev/null diff --git a/acceptance/cmd/skills-invoked/test.toml b/acceptance/cmd/skills-invoked/test.toml new file mode 100644 index 00000000000..20c9a4261c6 --- /dev/null +++ b/acceptance/cmd/skills-invoked/test.toml @@ -0,0 +1,5 @@ +RecordRequests = true +IncludeRequestHeaders = ["User-Agent"] +Ignore = [".databricks"] +# A host opt-out would otherwise remove the skills the positive cases assert. +Env.DATABRICKS_CLI_DISABLE_TELEMETRY = "" diff --git a/cmd/root/export_test.go b/cmd/root/export_test.go index c5142696dde..b220e086b90 100644 --- a/cmd/root/export_test.go +++ b/cmd/root/export_test.go @@ -6,6 +6,14 @@ import ( "github.com/spf13/cobra" ) +// Experimental skill-usage tracking, see user_agent_skills.go. +var ( + ParseSkills = parseSkills + WithSkillsInUserAgent = withSkillsInUserAgent + SkillsInvokedEnvVar = skillsInvokedEnvVar + MaxSkillsUserAgentLength = maxSkillsUserAgentLength +) + // BuildAll builds every command in t and returns the root command, for tests // that walk the whole tree. func BuildAll(ctx context.Context, t *Tree) *cobra.Command { diff --git a/cmd/root/root.go b/cmd/root/root.go index 5fc80ec87a4..0f1c9a90515 100644 --- a/cmd/root/root.go +++ b/cmd/root/root.go @@ -88,6 +88,7 @@ func newRootCommand(ctx context.Context) *cobra.Command { ctx = installer.WithAiDevKitInUserAgent(ctx) ctx = installer.WithCallingAgentToolingInUserAgent(ctx) ctx = InjectTestPidToUserAgent(ctx) + ctx = withSkillsInUserAgent(ctx) cmd.SetContext(ctx) // Recommend installing Databricks AI tooling to the calling coding agent diff --git a/cmd/root/user_agent_skills.go b/cmd/root/user_agent_skills.go new file mode 100644 index 00000000000..4892d07295b --- /dev/null +++ b/cmd/root/user_agent_skills.go @@ -0,0 +1,83 @@ +package root + +import ( + "context" + "regexp" + "slices" + "strings" + "sync" + + "github.com/databricks/cli/libs/env" + "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/telemetry" + "github.com/databricks/databricks-sdk-go/useragent" +) + +// Experimental skill-usage tracking: the databricks-core agent skill +// (https://github.com/databricks/databricks-agent-skills/tree/main/skills/databricks-core) +// tells agents to prefix CLI calls with this env var, set to the comma-separated +// bare names of the skills they loaded. It may change or be removed without +// notice. It is an env var rather than a flag so removing it doesn't change the +// CLI's interface, and because older CLIs ignore unknown env vars but reject +// unknown flags. +const skillsInvokedEnvVar = "DATABRICKS_SKILLS_INVOKED" + +// skillUserAgentKey is repeated once per skill: "skill/databricks-core skill/databricks-jobs". +const skillUserAgentKey = "skill" + +const ( + maxSkillLength = 64 + + // maxSkillsUserAgentLength bounds all " skill/" pairs together. It + // fits five of the longest current skill names (45-byte pairs). The usage + // logs that feed skill analytics cap the user agent at 512 characters, and + // the rest of the CLI's user agent (auth/, agent/ and other pairs, some + // appended after these) takes about 200, so the skills must leave room for it. + maxSkillsUserAgentLength = 225 +) + +// skillNamePattern is a subset of what the SDK accepts in a user agent value, +// so a valid name can never make useragent.InContext panic. +var skillNamePattern = sync.OnceValue(func() *regexp.Regexp { + return regexp.MustCompile(`^[a-z0-9][a-z0-9_.-]*$`) +}) + +// parseSkills normalizes a comma-separated list of skill names: entries are +// trimmed, lowercased and deduplicated, invalid ones are dropped, and names are +// kept in order while their pairs fit in maxSkillsUserAgentLength. Invalid input is only logged at debug level because this +// is telemetry and must never fail or warn a command. +func parseSkills(ctx context.Context, raw string) []string { + var skills []string + size := 0 + for entry := range strings.SplitSeq(raw, ",") { + name := strings.ToLower(strings.TrimSpace(entry)) + pairSize := len(" "+skillUserAgentKey+"/") + len(name) + switch { + case name == "": + case len(name) > maxSkillLength || !skillNamePattern().MatchString(name): + log.Debugf(ctx, "dropping invalid %s entry %q", skillsInvokedEnvVar, entry) + case slices.Contains(skills, name): + case size+pairSize > maxSkillsUserAgentLength: + log.Debugf(ctx, "dropping %s entry %q: skills exceed %d user agent bytes", skillsInvokedEnvVar, name, maxSkillsUserAgentLength) + default: + skills = append(skills, name) + size += pairSize + } + } + return skills +} + +// withSkillsInUserAgent adds one skill/ pair per skill in +// skillsInvokedEnvVar (experimental tracking), unless the user opted out of +// telemetry. +func withSkillsInUserAgent(ctx context.Context) context.Context { + if telemetry.Disabled(ctx) { + return ctx + } + for _, skill := range parseSkills(ctx, env.Get(ctx, skillsInvokedEnvVar)) { + // Each skill adds its own pair to the same context; this is accumulation, + // not the accidental nesting fatcontext flags. + ctx = useragent.InContext(ctx, skillUserAgentKey, skill) //nolint:fatcontext + } + return ctx +} diff --git a/cmd/root/user_agent_skills_test.go b/cmd/root/user_agent_skills_test.go new file mode 100644 index 00000000000..fee8a9ae864 --- /dev/null +++ b/cmd/root/user_agent_skills_test.go @@ -0,0 +1,96 @@ +package root_test + +import ( + "fmt" + "strings" + "testing" + + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/env" + "github.com/databricks/databricks-sdk-go/useragent" + "github.com/stretchr/testify/assert" +) + +// skillNames returns n distinct skill names of exactly length bytes. +func skillNames(n, length int) []string { + names := make([]string, 0, n) + for i := range n { + prefix := fmt.Sprintf("s%02d", i) + names = append(names, prefix+strings.Repeat("x", length-len(prefix))) + } + return names +} + +// pairsSize is the length of " skill/" for every name. +func pairsSize(names []string) int { + size := 0 + for _, name := range names { + size += len(" skill/") + len(name) + } + return size +} + +func TestParseSkills(t *testing.T) { + // 38-byte names (the longest current skill) make 45-byte pairs: five fill + // 225 bytes exactly and a sixth doesn't fit. + fiveFit := skillNames(5, 38) + sixTooMany := skillNames(6, 38) + // 64-, 64-, 64- and 22-byte pairs make 214 bytes, so " skill/abcd" (11) + // lands exactly on 225 and nothing more fits. + exact := []string{strings.Repeat("a", 57), strings.Repeat("b", 57), strings.Repeat("c", 57), "databricks-core", "abcd"} + + tests := []struct { + name string + raw string + want []string + }{ + {"empty", "", nil}, + {"single", "databricks-core", []string{"databricks-core"}}, + {"multiple", "a,b,c", []string{"a", "b", "c"}}, + {"trims and lowercases", " Databricks-Core , JOBS ", []string{"databricks-core", "jobs"}}, + {"drops empty entries", ",,a,,", []string{"a"}}, + {"allows underscore and dot", "a_b.c-d", []string{"a_b.c-d"}}, + {"drops disallowed skill-name characters", "ok,bad name,a:b,a/b,a+b", []string{"ok"}}, + {"drops leading punctuation", "-a,.b,_c", nil}, + {"dedupes preserving first occurrence", "b,a,b,a,c", []string{"b", "a", "c"}}, + {"dedupes after normalization", "Core,core, CORE", []string{"core"}}, + {"drops entry longer than 64 bytes", strings.Repeat("a", 65) + ",ok", []string{"ok"}}, + {"keeps entry of exactly 64 bytes", strings.Repeat("a", 64), []string{strings.Repeat("a", 64)}}, + {"keeps five of the longest skill names", strings.Join(fiveFit, ","), fiveFit}, + {"drops pairs over the budget", strings.Join(sixTooMany, ","), sixTooMany[:5]}, + {"fills the budget exactly", strings.Join(exact, ",") + ",d", exact}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := root.ParseSkills(t.Context(), tt.raw) + assert.Equal(t, tt.want, got) + assert.LessOrEqual(t, pairsSize(got), root.MaxSkillsUserAgentLength) + }) + } +} + +func TestWithSkillsInUserAgent(t *testing.T) { + tests := []struct { + name string + skills string + disableTelemetry string + want string + }{ + {"unset", "", "", ""}, + {"set", "databricks-core,databricks-jobs", "", "skill/databricks-core skill/databricks-jobs"}, + {"telemetry disabled", "databricks-core", "1", ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Always set both: an empty value in the context overrides the host environment. + ctx := env.Set(t.Context(), root.SkillsInvokedEnvVar, tt.skills) + ctx = env.Set(ctx, "DATABRICKS_CLI_DISABLE_TELEMETRY", tt.disableTelemetry) + ua := useragent.FromContext(root.WithSkillsInUserAgent(ctx)) + if tt.want == "" { + assert.NotContains(t, ua, "skill/") + } else { + assert.Contains(t, ua, tt.want) + } + }) + } +}