diff --git a/.nextchanges/cli/claude-code-native-hint-throttle.md b/.nextchanges/cli/claude-code-native-hint-throttle.md new file mode 100644 index 00000000000..bbd64cc7207 --- /dev/null +++ b/.nextchanges/cli/claude-code-native-hint-throttle.md @@ -0,0 +1 @@ +* Throttle Claude Code's native plugin install hint separately from the text hint: it prints at most once per 10 minutes per session, so the install dialog still comes up when a subagent, a hook, or a command outside a Claude Code subprocess runs the session's first `databricks` command. ([#7692](https://github.com/databricks/cli/pull/7692)) diff --git a/acceptance/cmd/aitools/claude-config-dir/output.txt b/acceptance/cmd/aitools/claude-config-dir/output.txt index f191ee55907..360b7386aa2 100644 --- a/acceptance/cmd/aitools/claude-config-dir/output.txt +++ b/acceptance/cmd/aitools/claude-config-dir/output.txt @@ -42,7 +42,7 @@ ID Name State 4321 some-test-cluster 9876 some-other-cluster -=== Within one session both lines print once, so 2>&1 pipelines see them only on the first command +=== Within one session the text line prints at most once per hour and the native line at most once per 10 minutes, so a 2>&1 pipeline on the next command sees neither >>> [CLI] clusters list @@ -55,3 +55,17 @@ ID Name State ID Name State 4321 some-test-cluster 9876 some-other-cluster + +=== A command outside a Claude Code subprocess leaves the native line due, so the next Claude Code command prints only that line + +>>> CLAUDE_CODE_CHILD_SESSION= [CLI] clusters list +Databricks skills are not installed. To work with Databricks reliably, first run: databricks aitools install +ID Name State +4321 some-test-cluster +9876 some-other-cluster + +>>> [CLI] clusters list + +ID Name State +4321 some-test-cluster +9876 some-other-cluster diff --git a/acceptance/cmd/aitools/claude-config-dir/script b/acceptance/cmd/aitools/claude-config-dir/script index aee1021fb8f..ad72ab82556 100644 --- a/acceptance/cmd/aitools/claude-config-dir/script +++ b/acceptance/cmd/aitools/claude-config-dir/script @@ -32,8 +32,14 @@ trace CLAUDE_CODE_CHILD_SESSION= $CLI clusters list # The session throttle lives in the file cache, which a host could disable. export DATABRICKS_CACHE_ENABLED=true -title "Within one session both lines print once, so 2>&1 pipelines see them only on the first command" +title "Within one session the text line prints at most once per hour and the native line at most once per 10 minutes, so a 2>&1 pipeline on the next command sees neither" export CLAUDE_CODE_SESSION_ID=session-1 trace $CLI clusters list trace $CLI clusters list unset CLAUDE_CODE_SESSION_ID + +title "A command outside a Claude Code subprocess leaves the native line due, so the next Claude Code command prints only that line" +export CLAUDE_CODE_SESSION_ID=session-2 +trace CLAUDE_CODE_CHILD_SESSION= $CLI clusters list +trace $CLI clusters list +unset CLAUDE_CODE_SESSION_ID diff --git a/libs/aitools/agents/hint.go b/libs/aitools/agents/hint.go index bee5ed0afaa..9045dddcdf1 100644 --- a/libs/aitools/agents/hint.go +++ b/libs/aitools/agents/hint.go @@ -31,10 +31,11 @@ const ( // the output and decides when to prompt: only in an attended interactive // session, at most once per session, and once ever per plugin for that // Claude Code user config (so in practice once per machine), whatever the - // answer. It is still throttled with the text hint: Claude Code strips it - // only from the tool output, so in a `2>&1 | jq` pipeline the next process - // reads it raw, and after a "No" repeating it can't lead to an install. The - // format is fixed by Claude Code: https://code.claude.com/docs/en/plugins/cli-hints + // answer. It is still throttled per session (see nativeHintThrottle): Claude + // Code strips it only from the tool output, so in a `2>&1 | jq` pipeline the + // next process reads it raw, and after a "No" repeating it can't lead to an + // install. The format is fixed by Claude Code: + // https://code.claude.com/docs/en/plugins/cli-hints claudeNativeHint = `` // claudeChildSessionEnv is set only in subprocesses Claude Code itself @@ -66,20 +67,44 @@ const ( selfExecEnv = "DATABRICKS_CLI_SELF_EXEC" // claudeSessionEnv is Claude Code's per-session identifier. It keys the - // per-session throttle shared by the text and native lines (at most one - // hint per hintSessionTTL per session); when unset (older Claude) the hint - // degrades to per-command. + // per-session throttles of the text and native lines; when unset (older + // Claude) both lines degrade to per-command. claudeSessionEnv = "CLAUDE_CODE_SESSION_ID" - - // hintCacheComponent names the cache bucket holding "already hinted" - // markers (per session, or per agent on this machine). - hintCacheComponent = "aitools-hint" ) -// hintSessionTTL is how long a session's "already hinted" marker lives. Within -// that window a session is hinted at most once; after it expires the hint may -// fire again for the same session. -const hintSessionTTL = time.Hour +// hintThrottle is an "already hinted" marker in the file cache, per throttle +// key (see agentHint.throttleKey): a line it guards prints at most once per +// ttl per key. +type hintThrottle struct { + // component names the cache bucket holding the markers. Each throttle needs + // its own: a cache expires every file in its bucket by its own ttl. + component string + ttl time.Duration +} + +var ( + // textHintThrottle throttles the model-facing text hint, which the model + // reads in every tool output it lands in. + textHintThrottle = hintThrottle{component: "aitools-hint", ttl: time.Hour} + + // nativeHintThrottle throttles the agent-native line on its own, shorter + // clock. The session id doesn't tell the main loop from a subagent or a + // hook (a subagent's commands carry the parent's session id and the + // child-session marker), and Claude Code strips the line there without + // prompting; with the text hint's hour, a subagent making the session's + // first call would hide the dialog from the main loop for that hour. + // Claude Code prompts at most once per session anyway, so repeating the + // line costs only a raw tag in a `2>&1` pipeline every ten minutes, and a + // session waits at most that long for the dialog. + nativeHintThrottle = hintThrottle{component: "aitools-native-hint", ttl: 10 * time.Minute} + + // nativeEventThrottle keeps the native line's hint-shown event on the text + // hint's clock, so a native-only repeat triggers no telemetry upload at + // exit and native-event counts don't grow with session length. Its marker + // holds whether that event was logged in the current text window: each + // text print starts a window and rewrites it. + nativeEventThrottle = hintThrottle{component: "aitools-native-hint-event", ttl: textHintThrottle.ttl} +) // agentHint describes how one calling agent is nudged to install the // Databricks AI tooling. @@ -89,16 +114,16 @@ type agentHint struct { installed func(ctx context.Context) bool // text is the model-facing recommendation, printed at most once per - // hintSessionTTL per session (see sessionEnv), or per agent on this machine - // when there is no session id (see fallbackKey), together with native. + // textHintThrottle per session (see sessionEnv), or per agent on this + // machine when there is no session id (see fallbackKey). text string // textKind tags the hint-shown event logged when text prints. textKind protos.AitoolsHintKind // sessionEnv names the env var carrying the agent's session id, which keys - // the throttle for both lines. When it is empty or unset at runtime, - // fallbackKey keys the throttle instead. + // the throttles of both lines. When it is empty or unset at runtime, + // fallbackKey keys them instead. sessionEnv string // sessionKeyPrefix namespaces the session id in the throttle key, so one @@ -108,7 +133,7 @@ type agentHint struct { sessionKeyPrefix string // fallbackKey keys the throttle when there is no session id, so the hint - // prints at most once per hintSessionTTL on this machine. Empty means no + // prints at most once per textHintThrottle on this machine. Empty means no // throttle without a session id (Claude Code). fallbackKey string @@ -119,8 +144,9 @@ type agentHint struct { durableThrottle bool // native returns a line the agent itself acts on (for Claude Code, its - // plugin install prompt), or "" when none applies to this invocation. It - // shares the text's per-session throttle. Nil for agents without one. + // plugin install prompt), or "" when none applies to this invocation. It is + // throttled per session by nativeHintThrottle, only when it prints. Nil for + // agents without one (all but Claude Code). native func(ctx context.Context) string // nativeKind tags the hint-shown event logged when the native line prints. @@ -170,9 +196,10 @@ func (h *agentHint) nativeLine(ctx context.Context) string { // token` and `auth docker token` credential helpers, a CLI process the CLI // started itself, when there is no output stream, and once the tooling is // present. -// Both lines, the agent-native one (if any) and the model-facing text, and -// their hint-shown events are throttled together per session, or per agent on -// this machine for agents without a session id (fallbackKey). +// The agent-native line (if any) and the model-facing text are throttled +// independently, per session or, for agents without a session id, per agent +// on this machine (fallbackKey). Both hint-shown events follow the text's +// clock (see nativeEventThrottle). // pluginRecorded reads the CLI install record for agents whose plugin has no // manifest reader here (see Agent.DatabricksTooling). // Best-effort: it never fails the command. Besides the stderr lines, it logs a @@ -195,34 +222,52 @@ func MaybeHint(ctx context.Context, cmd *cobra.Command, pluginRecorded PluginRec if !cmdio.HasIO(ctx) { return } - // Check the throttle before the filesystem probe, so a throttled session - // skips the plugin/skills detection. The marker is written only once a hint - // is due, so an already-installed environment never burns it. + // Check the throttles before the filesystem probe, so a throttled session + // skips the plugin/skills detection. A marker is written only once its + // line is due, so an already-installed environment never burns one, and + // neither does a native line withheld from this invocation. key := hint.throttleKey(ctx) - if alreadyHintedThisSession(ctx, key) { + native := hint.nativeLine(ctx) + if native != "" && nativeHintThrottle.seen(ctx, key) { + native = "" + } + textDue := !textHintThrottle.seen(ctx, key) + if native == "" && !textDue { return } if hint.installed(ctx) { return } - if !hint.claim(ctx, key) { + // Only agents without a native line have a durable throttle, so a failed + // claim leaves nothing to print. + if textDue && !hint.claim(ctx, key) { return } - if native := hint.nativeLine(ctx); native != "" { + if native != "" { cmdio.LogString(ctx, native) - logHintShown(ctx, hint.nativeKind) + nativeHintThrottle.record(ctx, key) + // A text print starts a new text window, in which this is the first + // native print. + if textDue || !nativeEventThrottle.seen(ctx, key) { + logHintShown(ctx, hint.nativeKind) + nativeEventThrottle.set(ctx, key, true) + } + } else if textDue && hint.native != nil { + nativeEventThrottle.set(ctx, key, false) + } + if textDue { + cmdio.LogString(ctx, hint.text) + logHintShown(ctx, hint.textKind) } - cmdio.LogString(ctx, hint.text) - logHintShown(ctx, hint.textKind) } -// claim records the throttle key and reports whether the hint may print. A -// durable throttle allows it only once the marker has persisted. +// claim records the text's throttle key and reports whether the text may +// print. A durable throttle allows it only once the marker has persisted. func (h *agentHint) claim(ctx context.Context, key string) bool { if h.durableThrottle { return claimHintMarker(ctx, key) } - recordSessionHint(ctx, key) + textHintThrottle.record(ctx, key) return true } @@ -342,26 +387,34 @@ func (h *agentHint) throttleKey(ctx context.Context) string { return h.fallbackKey } -// alreadyHintedThisSession reports whether the throttle key was hinted within -// the last hintSessionTTL. With no key it returns false so the hint fires per -// command. -func alreadyHintedThisSession(ctx context.Context, key string) bool { +// markers returns the file cache holding t's markers. +func (t hintThrottle) markers(ctx context.Context) *cache.Cache { + return cache.NewCache(ctx, t.component, t.ttl, nil) +} + +// seen reports whether key was marked true within the last t.ttl. With no key +// it returns false so the line prints per command. +func (t hintThrottle) seen(ctx context.Context, key string) bool { if key == "" { return false } - c := cache.NewCache(ctx, hintCacheComponent, hintSessionTTL, nil) - _, seen := cache.Get[bool](ctx, c, key) - return seen + marked, ok := cache.Get[bool](ctx, t.markers(ctx), key) + return ok && marked +} + +// record marks key, so later commands skip the line until the marker expires +// (t.ttl). No-op when the key is "". +func (t hintThrottle) record(ctx context.Context, key string) { + t.set(ctx, key, true) } -// recordSessionHint marks the throttle key as hinted, so later commands stay -// quiet until the marker expires (hintSessionTTL). No-op when the key is "". -func recordSessionHint(ctx context.Context, key string) { +// set writes key's marker, restarting its ttl; seen reports it only when +// marked is true. No-op when the key is "". +func (t hintThrottle) set(ctx context.Context, key string, marked bool) { if key == "" { return } - c := cache.NewCache(ctx, hintCacheComponent, hintSessionTTL, nil) - cache.Put(ctx, c, key, true) + cache.Put(ctx, t.markers(ctx), key, marked) } // sessionID returns the value of sessionEnv, or "" when the agent has no diff --git a/libs/aitools/agents/hint_agents.go b/libs/aitools/agents/hint_agents.go index 78170656580..3d60a984e02 100644 --- a/libs/aitools/agents/hint_agents.go +++ b/libs/aitools/agents/hint_agents.go @@ -56,7 +56,7 @@ func agentInstallHint(a *Agent, pluginRecorded PluginRecordedFunc) *agentHint { // caching turned off (DATABRICKS_CACHE_ENABLED=false): nothing persists, so // these agents get no hint. func claimHintMarker(ctx context.Context, key string) bool { - c := cache.NewCache(ctx, hintCacheComponent, hintSessionTTL, nil) + c := textHintThrottle.markers(ctx) cache.Put(ctx, c, key, true) _, ok := cache.Get[bool](ctx, c, key) return ok diff --git a/libs/aitools/agents/hint_test.go b/libs/aitools/agents/hint_test.go index 558ae25500c..50cdca4afcd 100644 --- a/libs/aitools/agents/hint_test.go +++ b/libs/aitools/agents/hint_test.go @@ -10,6 +10,7 @@ import ( "path/filepath" "strings" "testing" + "time" "github.com/databricks/cli/internal/testutil" "github.com/databricks/cli/libs/cmdio" @@ -290,25 +291,34 @@ func TestMaybeHint_NoIONoPanic(t *testing.T) { assert.Empty(t, hintKinds(t, ctx)) } +// sessionCtx returns a context for one command in session-abc, sharing home +// and cacheDir with the session's other commands. childSession marks it as a +// Claude Code subprocess, which enables the native line. +func sessionCtx(t *testing.T, home, cacheDir string, childSession bool) (context.Context, *bytes.Buffer) { + ctx, stderr := cmdio.NewTestContextWithStderr(t.Context()) + ctx = telemetry.WithNewLogger(ctx) + ctx = env.WithUserHomeDir(ctx, home) + ctx = env.Set(ctx, "DATABRICKS_CACHE_DIR", cacheDir) + // Throttle markers live in the file cache, which a host could disable. + ctx = env.Set(ctx, "DATABRICKS_CACHE_ENABLED", "true") + ctx = env.Set(ctx, claudeSessionEnv, "session-abc") + if childSession { + ctx = env.Set(ctx, claudeChildSessionEnv, "1") + } + return ctx, stderr +} + func TestMaybeHint_ThrottledPerSession(t *testing.T) { setClaudeAgent(t) - home := t.TempDir() - cacheDir := t.TempDir() - newCtx := func() (context.Context, *bytes.Buffer) { - ctx, stderr := cmdio.NewTestContextWithStderr(t.Context()) - ctx = env.WithUserHomeDir(ctx, home) - ctx = env.Set(ctx, "DATABRICKS_CACHE_DIR", cacheDir) - ctx = env.Set(ctx, claudeSessionEnv, "session-abc") - return telemetry.WithNewLogger(ctx), stderr - } + home, cacheDir := t.TempDir(), t.TempDir() - ctx1, stderr1 := newCtx() + ctx1, stderr1 := sessionCtx(t, home, cacheDir, false) MaybeHint(ctx1, regularCmd(), noPluginRecorded) assert.Contains(t, stderr1.String(), hintText) assert.Equal(t, []protos.AitoolsHintKind{protos.AitoolsHintKindInstallText}, hintKinds(t, ctx1)) // Same session id, second command: must stay quiet and log nothing. - ctx2, stderr2 := newCtx() + ctx2, stderr2 := sessionCtx(t, home, cacheDir, false) MaybeHint(ctx2, regularCmd(), noPluginRecorded) assert.Empty(t, stderr2.String()) assert.Empty(t, hintKinds(t, ctx2)) @@ -496,35 +506,170 @@ func TestMaybeHint_NativeHintSilentForOtherCallers(t *testing.T) { func TestMaybeHint_NativeHintThrottledPerSession(t *testing.T) { setClaudeAgent(t) - home := t.TempDir() - cacheDir := t.TempDir() - newCtx := func() (context.Context, *bytes.Buffer) { - ctx, stderr := cmdio.NewTestContextWithStderr(t.Context()) - ctx = env.WithUserHomeDir(ctx, home) - ctx = env.Set(ctx, "DATABRICKS_CACHE_DIR", cacheDir) - ctx = env.Set(ctx, "DATABRICKS_CACHE_ENABLED", "true") - ctx = env.Set(ctx, claudeSessionEnv, "session-abc") - ctx = env.Set(ctx, claudeChildSessionEnv, "1") - return ctx, stderr - } + home, cacheDir := t.TempDir(), t.TempDir() - ctx1, stderr1 := newCtx() + ctx1, stderr1 := sessionCtx(t, home, cacheDir, true) MaybeHint(ctx1, regularCmd(), noPluginRecorded) assert.Equal(t, wantNativeHint+"\n"+hintText+"\n", stderr1.String()) // Same session, second command: both lines stay quiet, so the tag doesn't // reach a downstream reader such as `2>&1 | jq` on every command. - ctx2, stderr2 := newCtx() + ctx2, stderr2 := sessionCtx(t, home, cacheDir, true) MaybeHint(ctx2, regularCmd(), noPluginRecorded) assert.Empty(t, stderr2.String()) // Another session: both lines print again. - ctx3, stderr3 := newCtx() + ctx3, stderr3 := sessionCtx(t, home, cacheDir, true) ctx3 = env.Set(ctx3, claudeSessionEnv, "session-def") MaybeHint(ctx3, regularCmd(), noPluginRecorded) assert.Equal(t, wantNativeHint+"\n"+hintText+"\n", stderr3.String()) } +// ageHintMarkers backdates every marker of throttle in cacheDir by age, as if +// it had been written that long ago. +func ageHintMarkers(t *testing.T, cacheDir string, throttle hintThrottle, age time.Duration) { + markers, err := filepath.Glob(filepath.Join(cacheDir, "*", throttle.component, "*")) + require.NoError(t, err) + require.NotEmpty(t, markers) + aged := time.Now().Add(-age) + for _, m := range markers { + require.NoError(t, os.Chtimes(m, aged, aged)) + } +} + +// A native line withheld from a command (not a Claude Code subprocess, or in +// CI) must not burn the native throttle: the session's next Claude Code +// command still gets it, without the throttled text. That is the first native +// print in the text window, so it logs its event. +func TestMaybeHint_WithheldNativeLineKeepsItsThrottle(t *testing.T) { + for _, tc := range []struct { + name string + childSession bool + ci bool + }{ + {name: "no child session"}, + {name: "CI", childSession: true, ci: true}, + } { + t.Run(tc.name, func(t *testing.T) { + setClaudeAgent(t) + // CiCdProvider caches its first answer process-wide. + useragent.ClearCache() + home, cacheDir := t.TempDir(), t.TempDir() + + ctx1, stderr1 := sessionCtx(t, home, cacheDir, tc.childSession) + if tc.ci { + ctx1 = env.Set(ctx1, ciEnv, "true") + } + MaybeHint(ctx1, regularCmd(), noPluginRecorded) + assert.Equal(t, hintText+"\n", stderr1.String()) + assert.Equal(t, []protos.AitoolsHintKind{protos.AitoolsHintKindInstallText}, hintKinds(t, ctx1)) + + ctx2, stderr2 := sessionCtx(t, home, cacheDir, true) + MaybeHint(ctx2, regularCmd(), noPluginRecorded) + assert.Equal(t, wantNativeHint+"\n", stderr2.String()) + assert.Equal(t, []protos.AitoolsHintKind{protos.AitoolsHintKindClaudeCodeNative}, hintKinds(t, ctx2)) + }) + } +} + +// The native line has its own, shorter throttle: once it expires the line +// prints again while the text hint stays throttled. Its event stays on the +// text's clock, so the repeat logs nothing and the command uploads no +// telemetry for it; a new text window logs both events again. +func TestMaybeHint_NativeOnlyRepeatLogsNoEvent(t *testing.T) { + setClaudeAgent(t) + home, cacheDir := t.TempDir(), t.TempDir() + + ctx1, stderr1 := sessionCtx(t, home, cacheDir, true) + MaybeHint(ctx1, regularCmd(), noPluginRecorded) + assert.Equal(t, wantNativeHint+"\n"+hintText+"\n", stderr1.String()) + assert.Equal(t, []protos.AitoolsHintKind{ + protos.AitoolsHintKindClaudeCodeNative, + protos.AitoolsHintKindInstallText, + }, hintKinds(t, ctx1)) + + // Past the native TTL but not the text's: only the native line prints. + ageHintMarkers(t, cacheDir, nativeHintThrottle, nativeHintThrottle.ttl+time.Minute) + ctx2, stderr2 := sessionCtx(t, home, cacheDir, true) + MaybeHint(ctx2, regularCmd(), noPluginRecorded) + assert.Equal(t, wantNativeHint+"\n", stderr2.String()) + assert.Empty(t, telemetry.BufferedEvents(ctx2)) + + // Both markers fresh again: nothing prints. + ctx3, stderr3 := sessionCtx(t, home, cacheDir, true) + MaybeHint(ctx3, regularCmd(), noPluginRecorded) + assert.Empty(t, stderr3.String()) + assert.Empty(t, telemetry.BufferedEvents(ctx3)) + + // Past the text TTL: a new text window, so both lines and events are back. + for _, throttle := range []hintThrottle{textHintThrottle, nativeHintThrottle, nativeEventThrottle} { + ageHintMarkers(t, cacheDir, throttle, textHintThrottle.ttl+time.Minute) + } + ctx4, stderr4 := sessionCtx(t, home, cacheDir, true) + MaybeHint(ctx4, regularCmd(), noPluginRecorded) + assert.Equal(t, wantNativeHint+"\n"+hintText+"\n", stderr4.String()) + assert.Equal(t, []protos.AitoolsHintKind{ + protos.AitoolsHintKindClaudeCodeNative, + protos.AitoolsHintKindInstallText, + }, hintKinds(t, ctx4)) +} + +// A text print without the native line starts a new text window, so the next +// native print logs its event even when a native event from the previous +// window is younger than the text TTL. +func TestMaybeHint_TextPrintStartsNewNativeEventWindow(t *testing.T) { + setClaudeAgent(t) + home, cacheDir := t.TempDir(), t.TempDir() + + ctx1, _ := sessionCtx(t, home, cacheDir, true) + MaybeHint(ctx1, regularCmd(), noPluginRecorded) + require.Len(t, hintKinds(t, ctx1), 2) + + // The text window expires, but not the native event marker; the next + // command withholds the native line and prints only the text. + ageHintMarkers(t, cacheDir, textHintThrottle, textHintThrottle.ttl+time.Minute) + ageHintMarkers(t, cacheDir, nativeHintThrottle, nativeHintThrottle.ttl+time.Minute) + ctx2, stderr2 := sessionCtx(t, home, cacheDir, false) + MaybeHint(ctx2, regularCmd(), noPluginRecorded) + assert.Equal(t, hintText+"\n", stderr2.String()) + assert.Equal(t, []protos.AitoolsHintKind{protos.AitoolsHintKindInstallText}, hintKinds(t, ctx2)) + + ctx3, stderr3 := sessionCtx(t, home, cacheDir, true) + MaybeHint(ctx3, regularCmd(), noPluginRecorded) + assert.Equal(t, wantNativeHint+"\n", stderr3.String()) + assert.Equal(t, []protos.AitoolsHintKind{protos.AitoolsHintKindClaudeCodeNative}, hintKinds(t, ctx3)) +} + +// A command that prints both lines starts a new text window, so it logs the +// native event even when the previous window's first native line printed late +// and its event marker is still fresh. +func TestMaybeHint_TextAndNativePrintStartsNewNativeEventWindow(t *testing.T) { + setClaudeAgent(t) + home, cacheDir := t.TempDir(), t.TempDir() + + ctx1, stderr1 := sessionCtx(t, home, cacheDir, false) + MaybeHint(ctx1, regularCmd(), noPluginRecorded) + assert.Equal(t, hintText+"\n", stderr1.String()) + + // Late in the text window, the first native line prints and logs its event. + ageHintMarkers(t, cacheDir, textHintThrottle, 55*time.Minute) + ctx2, stderr2 := sessionCtx(t, home, cacheDir, true) + MaybeHint(ctx2, regularCmd(), noPluginRecorded) + assert.Equal(t, wantNativeHint+"\n", stderr2.String()) + assert.Equal(t, []protos.AitoolsHintKind{protos.AitoolsHintKindClaudeCodeNative}, hintKinds(t, ctx2)) + + // The text and native markers expire; the event marker is still fresh. + ageHintMarkers(t, cacheDir, textHintThrottle, textHintThrottle.ttl+time.Minute) + ageHintMarkers(t, cacheDir, nativeHintThrottle, nativeHintThrottle.ttl+time.Minute) + ctx3, stderr3 := sessionCtx(t, home, cacheDir, true) + MaybeHint(ctx3, regularCmd(), noPluginRecorded) + assert.Equal(t, wantNativeHint+"\n"+hintText+"\n", stderr3.String()) + assert.Equal(t, []protos.AitoolsHintKind{ + protos.AitoolsHintKindClaudeCodeNative, + protos.AitoolsHintKindInstallText, + }, hintKinds(t, ctx3)) +} + func TestMaybeHint_NativeHintSilentOnTerminal(t *testing.T) { // The fakes make one stream look like a terminal, as in tmux or screen // started from inside Claude Code, where nothing strips the tag. A person @@ -587,22 +732,9 @@ func TestHintForAgent(t *testing.T) { func TestMaybeHint_LogsEventsOncePerSession(t *testing.T) { setClaudeAgent(t) - home := t.TempDir() - cacheDir := t.TempDir() - newCtx := func(childSession bool) (context.Context, *bytes.Buffer) { - ctx, stderr := cmdio.NewTestContextWithStderr(t.Context()) - ctx = telemetry.WithNewLogger(ctx) - ctx = env.WithUserHomeDir(ctx, home) - ctx = env.Set(ctx, "DATABRICKS_CACHE_DIR", cacheDir) - ctx = env.Set(ctx, "DATABRICKS_CACHE_ENABLED", "true") - ctx = env.Set(ctx, claudeSessionEnv, "session-abc") - if childSession { - ctx = env.Set(ctx, claudeChildSessionEnv, "1") - } - return ctx, stderr - } + home, cacheDir := t.TempDir(), t.TempDir() - ctx1, _ := newCtx(true) + ctx1, _ := sessionCtx(t, home, cacheDir, true) MaybeHint(ctx1, regularCmd(), noPluginRecorded) assert.Equal(t, []protos.AitoolsHintKind{ protos.AitoolsHintKindClaudeCodeNative, @@ -613,13 +745,13 @@ func TestMaybeHint_LogsEventsOncePerSession(t *testing.T) { // Throttled for the session: nothing prints and nothing logs, so the // command doesn't pay for a telemetry upload at exit. - ctx2, stderr2 := newCtx(true) + ctx2, stderr2 := sessionCtx(t, home, cacheDir, true) MaybeHint(ctx2, regularCmd(), noPluginRecorded) assert.Empty(t, stderr2.String()) assert.Empty(t, telemetry.BufferedEvents(ctx2)) // Throttled and not a Claude Code subprocess: nothing prints, nothing logs. - ctx3, stderr3 := newCtx(false) + ctx3, stderr3 := sessionCtx(t, home, cacheDir, false) MaybeHint(ctx3, regularCmd(), noPluginRecorded) assert.Empty(t, stderr3.String()) assert.Empty(t, telemetry.BufferedEvents(ctx3)) diff --git a/libs/telemetry/protos/README.md b/libs/telemetry/protos/README.md index aacdcca2e50..a6ae4c7cbd8 100644 --- a/libs/telemetry/protos/README.md +++ b/libs/telemetry/protos/README.md @@ -13,10 +13,12 @@ hint is never printed on hint-exempt commands (`hintExemptCommands` in `libs/aitools/agents/hint.go`: aitools, `auth`, one-shot lookups, help, completion) or in CLI processes the CLI started itself. It carries only the kind of hint (`hint_kind`); the command it was shown on is the entry's `execution_context.command`. The event uploads at exit only when the command -resolved auth. The install hint prints at most once per hour per agent session (or per agent on the -machine, for agents without a session id), so when the command that printed it resolves no auth -(e.g. `databricks bundle schema`), no install hint event is logged for that hour even though the -agent saw the hint. +resolved auth. The text line prints at most once per hour per agent session (or per agent on the +machine, for agents without a session id). Claude Code's native line prints at most once per 10 +minutes per session, but its event is logged at most once per text-line print: only for the first +native line printed since the text line last printed. So when the command that printed a line +resolves no auth (e.g. `databricks bundle schema`), no event for that line is uploaded until the +text line prints again, even though the agent saw the hint. `databricks aitools` install, update, list, version and uninstall log an `aitools_command_event` funnel per run: STARTED, one STEP_COMPLETED per install step reached (install only), and FINISHED