From 9d0ae5640289ccf026922ab4c3adfdb814cfc45b Mon Sep 17 00:00:00 2001 From: Anton Nekipelov <226657+anton-107@users.noreply.github.com> Date: Tue, 17 Nov 2026 10:55:19 +0000 Subject: [PATCH 1/2] aitools: throttle the native Claude Code hint on its own shorter clock The native line shared the text hint's one-hour session marker, which was written whenever the text printed, including when the native line was withheld (no child-session marker, CI, terminal attached) or printed where Claude Code strips it without prompting (subagents, hooks). The main loop could then go an hour without the install dialog. Throttle the native line under its own cache bucket with a 10-minute TTL, keyed by the hint's throttle key and written only when it prints. The text hint keeps its one-hour marker, and other agents' durable claim now goes through the same text throttle. The CLAUDE_CODE_NATIVE event stays on the text hint's hourly clock: it is logged only for the first native line printed since the text last printed, so a native-only repeat logs no event and triggers no telemetry upload at exit. Co-authored-by: Isaac --- .../cli/claude-code-native-hint-throttle.md | 1 + .../cmd/aitools/claude-config-dir/output.txt | 16 +- .../cmd/aitools/claude-config-dir/script | 8 +- libs/aitools/agents/hint.go | 151 +++++++++----- libs/aitools/agents/hint_agents.go | 2 +- libs/aitools/agents/hint_test.go | 186 ++++++++++++++---- libs/telemetry/protos/README.md | 8 +- 7 files changed, 275 insertions(+), 97 deletions(-) create mode 100644 .nextchanges/cli/claude-code-native-hint-throttle.md diff --git a/.nextchanges/cli/claude-code-native-hint-throttle.md b/.nextchanges/cli/claude-code-native-hint-throttle.md new file mode 100644 index 00000000000..bbd64cc7207 --- /dev/null +++ b/.nextchanges/cli/claude-code-native-hint-throttle.md @@ -0,0 +1 @@ +* Throttle Claude Code's native plugin install hint separately from the text hint: it prints at most once per 10 minutes per session, so the install dialog still comes up when a subagent, a hook, or a command outside a Claude Code subprocess runs the session's first `databricks` command. ([#7692](https://github.com/databricks/cli/pull/7692)) diff --git a/acceptance/cmd/aitools/claude-config-dir/output.txt b/acceptance/cmd/aitools/claude-config-dir/output.txt index f191ee55907..360b7386aa2 100644 --- a/acceptance/cmd/aitools/claude-config-dir/output.txt +++ b/acceptance/cmd/aitools/claude-config-dir/output.txt @@ -42,7 +42,7 @@ ID Name State 4321 some-test-cluster 9876 some-other-cluster -=== Within one session both lines print once, so 2>&1 pipelines see them only on the first command +=== Within one session the text line prints at most once per hour and the native line at most once per 10 minutes, so a 2>&1 pipeline on the next command sees neither >>> [CLI] clusters list @@ -55,3 +55,17 @@ ID Name State ID Name State 4321 some-test-cluster 9876 some-other-cluster + +=== A command outside a Claude Code subprocess leaves the native line due, so the next Claude Code command prints only that line + +>>> CLAUDE_CODE_CHILD_SESSION= [CLI] clusters list +Databricks skills are not installed. To work with Databricks reliably, first run: databricks aitools install +ID Name State +4321 some-test-cluster +9876 some-other-cluster + +>>> [CLI] clusters list + +ID Name State +4321 some-test-cluster +9876 some-other-cluster diff --git a/acceptance/cmd/aitools/claude-config-dir/script b/acceptance/cmd/aitools/claude-config-dir/script index aee1021fb8f..ad72ab82556 100644 --- a/acceptance/cmd/aitools/claude-config-dir/script +++ b/acceptance/cmd/aitools/claude-config-dir/script @@ -32,8 +32,14 @@ trace CLAUDE_CODE_CHILD_SESSION= $CLI clusters list # The session throttle lives in the file cache, which a host could disable. export DATABRICKS_CACHE_ENABLED=true -title "Within one session both lines print once, so 2>&1 pipelines see them only on the first command" +title "Within one session the text line prints at most once per hour and the native line at most once per 10 minutes, so a 2>&1 pipeline on the next command sees neither" export CLAUDE_CODE_SESSION_ID=session-1 trace $CLI clusters list trace $CLI clusters list unset CLAUDE_CODE_SESSION_ID + +title "A command outside a Claude Code subprocess leaves the native line due, so the next Claude Code command prints only that line" +export CLAUDE_CODE_SESSION_ID=session-2 +trace CLAUDE_CODE_CHILD_SESSION= $CLI clusters list +trace $CLI clusters list +unset CLAUDE_CODE_SESSION_ID diff --git a/libs/aitools/agents/hint.go b/libs/aitools/agents/hint.go index bee5ed0afaa..9045dddcdf1 100644 --- a/libs/aitools/agents/hint.go +++ b/libs/aitools/agents/hint.go @@ -31,10 +31,11 @@ const ( // the output and decides when to prompt: only in an attended interactive // session, at most once per session, and once ever per plugin for that // Claude Code user config (so in practice once per machine), whatever the - // answer. It is still throttled with the text hint: Claude Code strips it - // only from the tool output, so in a `2>&1 | jq` pipeline the next process - // reads it raw, and after a "No" repeating it can't lead to an install. The - // format is fixed by Claude Code: https://code.claude.com/docs/en/plugins/cli-hints + // answer. It is still throttled per session (see nativeHintThrottle): Claude + // Code strips it only from the tool output, so in a `2>&1 | jq` pipeline the + // next process reads it raw, and after a "No" repeating it can't lead to an + // install. The format is fixed by Claude Code: + // https://code.claude.com/docs/en/plugins/cli-hints claudeNativeHint = `` // claudeChildSessionEnv is set only in subprocesses Claude Code itself @@ -66,20 +67,44 @@ const ( selfExecEnv = "DATABRICKS_CLI_SELF_EXEC" // claudeSessionEnv is Claude Code's per-session identifier. It keys the - // per-session throttle shared by the text and native lines (at most one - // hint per hintSessionTTL per session); when unset (older Claude) the hint - // degrades to per-command. + // per-session throttles of the text and native lines; when unset (older + // Claude) both lines degrade to per-command. claudeSessionEnv = "CLAUDE_CODE_SESSION_ID" - - // hintCacheComponent names the cache bucket holding "already hinted" - // markers (per session, or per agent on this machine). - hintCacheComponent = "aitools-hint" ) -// hintSessionTTL is how long a session's "already hinted" marker lives. Within -// that window a session is hinted at most once; after it expires the hint may -// fire again for the same session. -const hintSessionTTL = time.Hour +// hintThrottle is an "already hinted" marker in the file cache, per throttle +// key (see agentHint.throttleKey): a line it guards prints at most once per +// ttl per key. +type hintThrottle struct { + // component names the cache bucket holding the markers. Each throttle needs + // its own: a cache expires every file in its bucket by its own ttl. + component string + ttl time.Duration +} + +var ( + // textHintThrottle throttles the model-facing text hint, which the model + // reads in every tool output it lands in. + textHintThrottle = hintThrottle{component: "aitools-hint", ttl: time.Hour} + + // nativeHintThrottle throttles the agent-native line on its own, shorter + // clock. The session id doesn't tell the main loop from a subagent or a + // hook (a subagent's commands carry the parent's session id and the + // child-session marker), and Claude Code strips the line there without + // prompting; with the text hint's hour, a subagent making the session's + // first call would hide the dialog from the main loop for that hour. + // Claude Code prompts at most once per session anyway, so repeating the + // line costs only a raw tag in a `2>&1` pipeline every ten minutes, and a + // session waits at most that long for the dialog. + nativeHintThrottle = hintThrottle{component: "aitools-native-hint", ttl: 10 * time.Minute} + + // nativeEventThrottle keeps the native line's hint-shown event on the text + // hint's clock, so a native-only repeat triggers no telemetry upload at + // exit and native-event counts don't grow with session length. Its marker + // holds whether that event was logged in the current text window: each + // text print starts a window and rewrites it. + nativeEventThrottle = hintThrottle{component: "aitools-native-hint-event", ttl: textHintThrottle.ttl} +) // agentHint describes how one calling agent is nudged to install the // Databricks AI tooling. @@ -89,16 +114,16 @@ type agentHint struct { installed func(ctx context.Context) bool // text is the model-facing recommendation, printed at most once per - // hintSessionTTL per session (see sessionEnv), or per agent on this machine - // when there is no session id (see fallbackKey), together with native. + // textHintThrottle per session (see sessionEnv), or per agent on this + // machine when there is no session id (see fallbackKey). text string // textKind tags the hint-shown event logged when text prints. textKind protos.AitoolsHintKind // sessionEnv names the env var carrying the agent's session id, which keys - // the throttle for both lines. When it is empty or unset at runtime, - // fallbackKey keys the throttle instead. + // the throttles of both lines. When it is empty or unset at runtime, + // fallbackKey keys them instead. sessionEnv string // sessionKeyPrefix namespaces the session id in the throttle key, so one @@ -108,7 +133,7 @@ type agentHint struct { sessionKeyPrefix string // fallbackKey keys the throttle when there is no session id, so the hint - // prints at most once per hintSessionTTL on this machine. Empty means no + // prints at most once per textHintThrottle on this machine. Empty means no // throttle without a session id (Claude Code). fallbackKey string @@ -119,8 +144,9 @@ type agentHint struct { durableThrottle bool // native returns a line the agent itself acts on (for Claude Code, its - // plugin install prompt), or "" when none applies to this invocation. It - // shares the text's per-session throttle. Nil for agents without one. + // plugin install prompt), or "" when none applies to this invocation. It is + // throttled per session by nativeHintThrottle, only when it prints. Nil for + // agents without one (all but Claude Code). native func(ctx context.Context) string // nativeKind tags the hint-shown event logged when the native line prints. @@ -170,9 +196,10 @@ func (h *agentHint) nativeLine(ctx context.Context) string { // token` and `auth docker token` credential helpers, a CLI process the CLI // started itself, when there is no output stream, and once the tooling is // present. -// Both lines, the agent-native one (if any) and the model-facing text, and -// their hint-shown events are throttled together per session, or per agent on -// this machine for agents without a session id (fallbackKey). +// The agent-native line (if any) and the model-facing text are throttled +// independently, per session or, for agents without a session id, per agent +// on this machine (fallbackKey). Both hint-shown events follow the text's +// clock (see nativeEventThrottle). // pluginRecorded reads the CLI install record for agents whose plugin has no // manifest reader here (see Agent.DatabricksTooling). // Best-effort: it never fails the command. Besides the stderr lines, it logs a @@ -195,34 +222,52 @@ func MaybeHint(ctx context.Context, cmd *cobra.Command, pluginRecorded PluginRec if !cmdio.HasIO(ctx) { return } - // Check the throttle before the filesystem probe, so a throttled session - // skips the plugin/skills detection. The marker is written only once a hint - // is due, so an already-installed environment never burns it. + // Check the throttles before the filesystem probe, so a throttled session + // skips the plugin/skills detection. A marker is written only once its + // line is due, so an already-installed environment never burns one, and + // neither does a native line withheld from this invocation. key := hint.throttleKey(ctx) - if alreadyHintedThisSession(ctx, key) { + native := hint.nativeLine(ctx) + if native != "" && nativeHintThrottle.seen(ctx, key) { + native = "" + } + textDue := !textHintThrottle.seen(ctx, key) + if native == "" && !textDue { return } if hint.installed(ctx) { return } - if !hint.claim(ctx, key) { + // Only agents without a native line have a durable throttle, so a failed + // claim leaves nothing to print. + if textDue && !hint.claim(ctx, key) { return } - if native := hint.nativeLine(ctx); native != "" { + if native != "" { cmdio.LogString(ctx, native) - logHintShown(ctx, hint.nativeKind) + nativeHintThrottle.record(ctx, key) + // A text print starts a new text window, in which this is the first + // native print. + if textDue || !nativeEventThrottle.seen(ctx, key) { + logHintShown(ctx, hint.nativeKind) + nativeEventThrottle.set(ctx, key, true) + } + } else if textDue && hint.native != nil { + nativeEventThrottle.set(ctx, key, false) + } + if textDue { + cmdio.LogString(ctx, hint.text) + logHintShown(ctx, hint.textKind) } - cmdio.LogString(ctx, hint.text) - logHintShown(ctx, hint.textKind) } -// claim records the throttle key and reports whether the hint may print. A -// durable throttle allows it only once the marker has persisted. +// claim records the text's throttle key and reports whether the text may +// print. A durable throttle allows it only once the marker has persisted. func (h *agentHint) claim(ctx context.Context, key string) bool { if h.durableThrottle { return claimHintMarker(ctx, key) } - recordSessionHint(ctx, key) + textHintThrottle.record(ctx, key) return true } @@ -342,26 +387,34 @@ func (h *agentHint) throttleKey(ctx context.Context) string { return h.fallbackKey } -// alreadyHintedThisSession reports whether the throttle key was hinted within -// the last hintSessionTTL. With no key it returns false so the hint fires per -// command. -func alreadyHintedThisSession(ctx context.Context, key string) bool { +// markers returns the file cache holding t's markers. +func (t hintThrottle) markers(ctx context.Context) *cache.Cache { + return cache.NewCache(ctx, t.component, t.ttl, nil) +} + +// seen reports whether key was marked true within the last t.ttl. With no key +// it returns false so the line prints per command. +func (t hintThrottle) seen(ctx context.Context, key string) bool { if key == "" { return false } - c := cache.NewCache(ctx, hintCacheComponent, hintSessionTTL, nil) - _, seen := cache.Get[bool](ctx, c, key) - return seen + marked, ok := cache.Get[bool](ctx, t.markers(ctx), key) + return ok && marked +} + +// record marks key, so later commands skip the line until the marker expires +// (t.ttl). No-op when the key is "". +func (t hintThrottle) record(ctx context.Context, key string) { + t.set(ctx, key, true) } -// recordSessionHint marks the throttle key as hinted, so later commands stay -// quiet until the marker expires (hintSessionTTL). No-op when the key is "". -func recordSessionHint(ctx context.Context, key string) { +// set writes key's marker, restarting its ttl; seen reports it only when +// marked is true. No-op when the key is "". +func (t hintThrottle) set(ctx context.Context, key string, marked bool) { if key == "" { return } - c := cache.NewCache(ctx, hintCacheComponent, hintSessionTTL, nil) - cache.Put(ctx, c, key, true) + cache.Put(ctx, t.markers(ctx), key, marked) } // sessionID returns the value of sessionEnv, or "" when the agent has no diff --git a/libs/aitools/agents/hint_agents.go b/libs/aitools/agents/hint_agents.go index 78170656580..3d60a984e02 100644 --- a/libs/aitools/agents/hint_agents.go +++ b/libs/aitools/agents/hint_agents.go @@ -56,7 +56,7 @@ func agentInstallHint(a *Agent, pluginRecorded PluginRecordedFunc) *agentHint { // caching turned off (DATABRICKS_CACHE_ENABLED=false): nothing persists, so // these agents get no hint. func claimHintMarker(ctx context.Context, key string) bool { - c := cache.NewCache(ctx, hintCacheComponent, hintSessionTTL, nil) + c := textHintThrottle.markers(ctx) cache.Put(ctx, c, key, true) _, ok := cache.Get[bool](ctx, c, key) return ok diff --git a/libs/aitools/agents/hint_test.go b/libs/aitools/agents/hint_test.go index 558ae25500c..52d8f4a1eb3 100644 --- a/libs/aitools/agents/hint_test.go +++ b/libs/aitools/agents/hint_test.go @@ -10,6 +10,7 @@ import ( "path/filepath" "strings" "testing" + "time" "github.com/databricks/cli/internal/testutil" "github.com/databricks/cli/libs/cmdio" @@ -290,25 +291,34 @@ func TestMaybeHint_NoIONoPanic(t *testing.T) { assert.Empty(t, hintKinds(t, ctx)) } +// sessionCtx returns a context for one command in session-abc, sharing home +// and cacheDir with the session's other commands. childSession marks it as a +// Claude Code subprocess, which enables the native line. +func sessionCtx(t *testing.T, home, cacheDir string, childSession bool) (context.Context, *bytes.Buffer) { + ctx, stderr := cmdio.NewTestContextWithStderr(t.Context()) + ctx = telemetry.WithNewLogger(ctx) + ctx = env.WithUserHomeDir(ctx, home) + ctx = env.Set(ctx, "DATABRICKS_CACHE_DIR", cacheDir) + // Throttle markers live in the file cache, which a host could disable. + ctx = env.Set(ctx, "DATABRICKS_CACHE_ENABLED", "true") + ctx = env.Set(ctx, claudeSessionEnv, "session-abc") + if childSession { + ctx = env.Set(ctx, claudeChildSessionEnv, "1") + } + return ctx, stderr +} + func TestMaybeHint_ThrottledPerSession(t *testing.T) { setClaudeAgent(t) - home := t.TempDir() - cacheDir := t.TempDir() - newCtx := func() (context.Context, *bytes.Buffer) { - ctx, stderr := cmdio.NewTestContextWithStderr(t.Context()) - ctx = env.WithUserHomeDir(ctx, home) - ctx = env.Set(ctx, "DATABRICKS_CACHE_DIR", cacheDir) - ctx = env.Set(ctx, claudeSessionEnv, "session-abc") - return telemetry.WithNewLogger(ctx), stderr - } + home, cacheDir := t.TempDir(), t.TempDir() - ctx1, stderr1 := newCtx() + ctx1, stderr1 := sessionCtx(t, home, cacheDir, false) MaybeHint(ctx1, regularCmd(), noPluginRecorded) assert.Contains(t, stderr1.String(), hintText) assert.Equal(t, []protos.AitoolsHintKind{protos.AitoolsHintKindInstallText}, hintKinds(t, ctx1)) // Same session id, second command: must stay quiet and log nothing. - ctx2, stderr2 := newCtx() + ctx2, stderr2 := sessionCtx(t, home, cacheDir, false) MaybeHint(ctx2, regularCmd(), noPluginRecorded) assert.Empty(t, stderr2.String()) assert.Empty(t, hintKinds(t, ctx2)) @@ -496,35 +506,140 @@ func TestMaybeHint_NativeHintSilentForOtherCallers(t *testing.T) { func TestMaybeHint_NativeHintThrottledPerSession(t *testing.T) { setClaudeAgent(t) - home := t.TempDir() - cacheDir := t.TempDir() - newCtx := func() (context.Context, *bytes.Buffer) { - ctx, stderr := cmdio.NewTestContextWithStderr(t.Context()) - ctx = env.WithUserHomeDir(ctx, home) - ctx = env.Set(ctx, "DATABRICKS_CACHE_DIR", cacheDir) - ctx = env.Set(ctx, "DATABRICKS_CACHE_ENABLED", "true") - ctx = env.Set(ctx, claudeSessionEnv, "session-abc") - ctx = env.Set(ctx, claudeChildSessionEnv, "1") - return ctx, stderr - } + home, cacheDir := t.TempDir(), t.TempDir() - ctx1, stderr1 := newCtx() + ctx1, stderr1 := sessionCtx(t, home, cacheDir, true) MaybeHint(ctx1, regularCmd(), noPluginRecorded) assert.Equal(t, wantNativeHint+"\n"+hintText+"\n", stderr1.String()) // Same session, second command: both lines stay quiet, so the tag doesn't // reach a downstream reader such as `2>&1 | jq` on every command. - ctx2, stderr2 := newCtx() + ctx2, stderr2 := sessionCtx(t, home, cacheDir, true) MaybeHint(ctx2, regularCmd(), noPluginRecorded) assert.Empty(t, stderr2.String()) // Another session: both lines print again. - ctx3, stderr3 := newCtx() + ctx3, stderr3 := sessionCtx(t, home, cacheDir, true) ctx3 = env.Set(ctx3, claudeSessionEnv, "session-def") MaybeHint(ctx3, regularCmd(), noPluginRecorded) assert.Equal(t, wantNativeHint+"\n"+hintText+"\n", stderr3.String()) } +// ageHintMarkers backdates every marker of throttle in cacheDir by age, as if +// it had been written that long ago. +func ageHintMarkers(t *testing.T, cacheDir string, throttle hintThrottle, age time.Duration) { + markers, err := filepath.Glob(filepath.Join(cacheDir, "*", throttle.component, "*")) + require.NoError(t, err) + require.NotEmpty(t, markers) + aged := time.Now().Add(-age) + for _, m := range markers { + require.NoError(t, os.Chtimes(m, aged, aged)) + } +} + +// A native line withheld from a command (not a Claude Code subprocess, or in +// CI) must not burn the native throttle: the session's next Claude Code +// command still gets it, without the throttled text. That is the first native +// print in the text window, so it logs its event. +func TestMaybeHint_WithheldNativeLineKeepsItsThrottle(t *testing.T) { + for _, tc := range []struct { + name string + childSession bool + ci bool + }{ + {name: "no child session"}, + {name: "CI", childSession: true, ci: true}, + } { + t.Run(tc.name, func(t *testing.T) { + setClaudeAgent(t) + // CiCdProvider caches its first answer process-wide. + useragent.ClearCache() + home, cacheDir := t.TempDir(), t.TempDir() + + ctx1, stderr1 := sessionCtx(t, home, cacheDir, tc.childSession) + if tc.ci { + ctx1 = env.Set(ctx1, ciEnv, "true") + } + MaybeHint(ctx1, regularCmd(), noPluginRecorded) + assert.Equal(t, hintText+"\n", stderr1.String()) + assert.Equal(t, []protos.AitoolsHintKind{protos.AitoolsHintKindInstallText}, hintKinds(t, ctx1)) + + ctx2, stderr2 := sessionCtx(t, home, cacheDir, true) + MaybeHint(ctx2, regularCmd(), noPluginRecorded) + assert.Equal(t, wantNativeHint+"\n", stderr2.String()) + assert.Equal(t, []protos.AitoolsHintKind{protos.AitoolsHintKindClaudeCodeNative}, hintKinds(t, ctx2)) + }) + } +} + +// The native line has its own, shorter throttle: once it expires the line +// prints again while the text hint stays throttled. Its event stays on the +// text's clock, so the repeat logs nothing and the command uploads no +// telemetry for it; a new text window logs both events again. +func TestMaybeHint_NativeOnlyRepeatLogsNoEvent(t *testing.T) { + setClaudeAgent(t) + home, cacheDir := t.TempDir(), t.TempDir() + + ctx1, stderr1 := sessionCtx(t, home, cacheDir, true) + MaybeHint(ctx1, regularCmd(), noPluginRecorded) + assert.Equal(t, wantNativeHint+"\n"+hintText+"\n", stderr1.String()) + assert.Equal(t, []protos.AitoolsHintKind{ + protos.AitoolsHintKindClaudeCodeNative, + protos.AitoolsHintKindInstallText, + }, hintKinds(t, ctx1)) + + // Past the native TTL but not the text's: only the native line prints. + ageHintMarkers(t, cacheDir, nativeHintThrottle, nativeHintThrottle.ttl+time.Minute) + ctx2, stderr2 := sessionCtx(t, home, cacheDir, true) + MaybeHint(ctx2, regularCmd(), noPluginRecorded) + assert.Equal(t, wantNativeHint+"\n", stderr2.String()) + assert.Empty(t, telemetry.BufferedEvents(ctx2)) + + // Both markers fresh again: nothing prints. + ctx3, stderr3 := sessionCtx(t, home, cacheDir, true) + MaybeHint(ctx3, regularCmd(), noPluginRecorded) + assert.Empty(t, stderr3.String()) + assert.Empty(t, telemetry.BufferedEvents(ctx3)) + + // Past the text TTL: a new text window, so both lines and events are back. + for _, throttle := range []hintThrottle{textHintThrottle, nativeHintThrottle, nativeEventThrottle} { + ageHintMarkers(t, cacheDir, throttle, textHintThrottle.ttl+time.Minute) + } + ctx4, stderr4 := sessionCtx(t, home, cacheDir, true) + MaybeHint(ctx4, regularCmd(), noPluginRecorded) + assert.Equal(t, wantNativeHint+"\n"+hintText+"\n", stderr4.String()) + assert.Equal(t, []protos.AitoolsHintKind{ + protos.AitoolsHintKindClaudeCodeNative, + protos.AitoolsHintKindInstallText, + }, hintKinds(t, ctx4)) +} + +// A text print without the native line starts a new text window, so the next +// native print logs its event even when a native event from the previous +// window is younger than the text TTL. +func TestMaybeHint_TextPrintStartsNewNativeEventWindow(t *testing.T) { + setClaudeAgent(t) + home, cacheDir := t.TempDir(), t.TempDir() + + ctx1, _ := sessionCtx(t, home, cacheDir, true) + MaybeHint(ctx1, regularCmd(), noPluginRecorded) + require.Len(t, hintKinds(t, ctx1), 2) + + // The text window expires, but not the native event marker; the next + // command withholds the native line and prints only the text. + ageHintMarkers(t, cacheDir, textHintThrottle, textHintThrottle.ttl+time.Minute) + ageHintMarkers(t, cacheDir, nativeHintThrottle, nativeHintThrottle.ttl+time.Minute) + ctx2, stderr2 := sessionCtx(t, home, cacheDir, false) + MaybeHint(ctx2, regularCmd(), noPluginRecorded) + assert.Equal(t, hintText+"\n", stderr2.String()) + assert.Equal(t, []protos.AitoolsHintKind{protos.AitoolsHintKindInstallText}, hintKinds(t, ctx2)) + + ctx3, stderr3 := sessionCtx(t, home, cacheDir, true) + MaybeHint(ctx3, regularCmd(), noPluginRecorded) + assert.Equal(t, wantNativeHint+"\n", stderr3.String()) + assert.Equal(t, []protos.AitoolsHintKind{protos.AitoolsHintKindClaudeCodeNative}, hintKinds(t, ctx3)) +} + func TestMaybeHint_NativeHintSilentOnTerminal(t *testing.T) { // The fakes make one stream look like a terminal, as in tmux or screen // started from inside Claude Code, where nothing strips the tag. A person @@ -587,22 +702,9 @@ func TestHintForAgent(t *testing.T) { func TestMaybeHint_LogsEventsOncePerSession(t *testing.T) { setClaudeAgent(t) - home := t.TempDir() - cacheDir := t.TempDir() - newCtx := func(childSession bool) (context.Context, *bytes.Buffer) { - ctx, stderr := cmdio.NewTestContextWithStderr(t.Context()) - ctx = telemetry.WithNewLogger(ctx) - ctx = env.WithUserHomeDir(ctx, home) - ctx = env.Set(ctx, "DATABRICKS_CACHE_DIR", cacheDir) - ctx = env.Set(ctx, "DATABRICKS_CACHE_ENABLED", "true") - ctx = env.Set(ctx, claudeSessionEnv, "session-abc") - if childSession { - ctx = env.Set(ctx, claudeChildSessionEnv, "1") - } - return ctx, stderr - } + home, cacheDir := t.TempDir(), t.TempDir() - ctx1, _ := newCtx(true) + ctx1, _ := sessionCtx(t, home, cacheDir, true) MaybeHint(ctx1, regularCmd(), noPluginRecorded) assert.Equal(t, []protos.AitoolsHintKind{ protos.AitoolsHintKindClaudeCodeNative, @@ -613,13 +715,13 @@ func TestMaybeHint_LogsEventsOncePerSession(t *testing.T) { // Throttled for the session: nothing prints and nothing logs, so the // command doesn't pay for a telemetry upload at exit. - ctx2, stderr2 := newCtx(true) + ctx2, stderr2 := sessionCtx(t, home, cacheDir, true) MaybeHint(ctx2, regularCmd(), noPluginRecorded) assert.Empty(t, stderr2.String()) assert.Empty(t, telemetry.BufferedEvents(ctx2)) // Throttled and not a Claude Code subprocess: nothing prints, nothing logs. - ctx3, stderr3 := newCtx(false) + ctx3, stderr3 := sessionCtx(t, home, cacheDir, false) MaybeHint(ctx3, regularCmd(), noPluginRecorded) assert.Empty(t, stderr3.String()) assert.Empty(t, telemetry.BufferedEvents(ctx3)) diff --git a/libs/telemetry/protos/README.md b/libs/telemetry/protos/README.md index aacdcca2e50..3beadb91ab4 100644 --- a/libs/telemetry/protos/README.md +++ b/libs/telemetry/protos/README.md @@ -13,9 +13,11 @@ hint is never printed on hint-exempt commands (`hintExemptCommands` in `libs/aitools/agents/hint.go`: aitools, `auth`, one-shot lookups, help, completion) or in CLI processes the CLI started itself. It carries only the kind of hint (`hint_kind`); the command it was shown on is the entry's `execution_context.command`. The event uploads at exit only when the command -resolved auth. The install hint prints at most once per hour per agent session (or per agent on the -machine, for agents without a session id), so when the command that printed it resolves no auth -(e.g. `databricks bundle schema`), no install hint event is logged for that hour even though the +resolved auth. The text line prints at most once per hour per agent session (or per agent on the +machine, for agents without a session id). Claude Code's native line prints at most once per 10 +minutes per session, but its event is logged at most once per hour: only for the first native line +printed since the text line last printed. So when the command that printed a line resolves no auth +(e.g. `databricks bundle schema`), no event for that line is uploaded for that hour even though the agent saw the hint. `databricks aitools` install, update, list, version and uninstall log an `aitools_command_event` From feeecc54fe8b8a1517484e7b36c4a1e2cd48e877 Mon Sep 17 00:00:00 2001 From: Anton Nekipelov <226657+anton-107@users.noreply.github.com> Date: Tue, 17 Nov 2026 14:27:00 +0000 Subject: [PATCH 2/2] aitools: test the native event on a combined print and fix its README bound Cover the textDue half of the native-event condition: a command that prints both lines starts a new text window and logs the native event, even when the previous window's late native print left a fresh event marker. The README now states the real bound, one native event per text-line print, rather than once per hour. Co-authored-by: Isaac --- libs/aitools/agents/hint_test.go | 30 ++++++++++++++++++++++++++++++ libs/telemetry/protos/README.md | 8 ++++---- 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/libs/aitools/agents/hint_test.go b/libs/aitools/agents/hint_test.go index 52d8f4a1eb3..50cdca4afcd 100644 --- a/libs/aitools/agents/hint_test.go +++ b/libs/aitools/agents/hint_test.go @@ -640,6 +640,36 @@ func TestMaybeHint_TextPrintStartsNewNativeEventWindow(t *testing.T) { assert.Equal(t, []protos.AitoolsHintKind{protos.AitoolsHintKindClaudeCodeNative}, hintKinds(t, ctx3)) } +// A command that prints both lines starts a new text window, so it logs the +// native event even when the previous window's first native line printed late +// and its event marker is still fresh. +func TestMaybeHint_TextAndNativePrintStartsNewNativeEventWindow(t *testing.T) { + setClaudeAgent(t) + home, cacheDir := t.TempDir(), t.TempDir() + + ctx1, stderr1 := sessionCtx(t, home, cacheDir, false) + MaybeHint(ctx1, regularCmd(), noPluginRecorded) + assert.Equal(t, hintText+"\n", stderr1.String()) + + // Late in the text window, the first native line prints and logs its event. + ageHintMarkers(t, cacheDir, textHintThrottle, 55*time.Minute) + ctx2, stderr2 := sessionCtx(t, home, cacheDir, true) + MaybeHint(ctx2, regularCmd(), noPluginRecorded) + assert.Equal(t, wantNativeHint+"\n", stderr2.String()) + assert.Equal(t, []protos.AitoolsHintKind{protos.AitoolsHintKindClaudeCodeNative}, hintKinds(t, ctx2)) + + // The text and native markers expire; the event marker is still fresh. + ageHintMarkers(t, cacheDir, textHintThrottle, textHintThrottle.ttl+time.Minute) + ageHintMarkers(t, cacheDir, nativeHintThrottle, nativeHintThrottle.ttl+time.Minute) + ctx3, stderr3 := sessionCtx(t, home, cacheDir, true) + MaybeHint(ctx3, regularCmd(), noPluginRecorded) + assert.Equal(t, wantNativeHint+"\n"+hintText+"\n", stderr3.String()) + assert.Equal(t, []protos.AitoolsHintKind{ + protos.AitoolsHintKindClaudeCodeNative, + protos.AitoolsHintKindInstallText, + }, hintKinds(t, ctx3)) +} + func TestMaybeHint_NativeHintSilentOnTerminal(t *testing.T) { // The fakes make one stream look like a terminal, as in tmux or screen // started from inside Claude Code, where nothing strips the tag. A person diff --git a/libs/telemetry/protos/README.md b/libs/telemetry/protos/README.md index 3beadb91ab4..a6ae4c7cbd8 100644 --- a/libs/telemetry/protos/README.md +++ b/libs/telemetry/protos/README.md @@ -15,10 +15,10 @@ processes the CLI started itself. It carries only the kind of hint (`hint_kind`) shown on is the entry's `execution_context.command`. The event uploads at exit only when the command resolved auth. The text line prints at most once per hour per agent session (or per agent on the machine, for agents without a session id). Claude Code's native line prints at most once per 10 -minutes per session, but its event is logged at most once per hour: only for the first native line -printed since the text line last printed. So when the command that printed a line resolves no auth -(e.g. `databricks bundle schema`), no event for that line is uploaded for that hour even though the -agent saw the hint. +minutes per session, but its event is logged at most once per text-line print: only for the first +native line printed since the text line last printed. So when the command that printed a line +resolves no auth (e.g. `databricks bundle schema`), no event for that line is uploaded until the +text line prints again, even though the agent saw the hint. `databricks aitools` install, update, list, version and uninstall log an `aitools_command_event` funnel per run: STARTED, one STEP_COMPLETED per install step reached (install only), and FINISHED