diff --git a/acceptance/cmd/aitools/telemetry/mcp-add/.databrickscfg b/acceptance/cmd/aitools/telemetry/mcp-add/.databrickscfg new file mode 100644 index 00000000000..8d80c7ca7da --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-add/.databrickscfg @@ -0,0 +1,3 @@ +[logfood] +host = $DATABRICKS_HOST +token = test-token-logfood-$UNIQUE_NAME diff --git a/acceptance/cmd/aitools/telemetry/mcp-add/codex-config.toml b/acceptance/cmd/aitools/telemetry/mcp-add/codex-config.toml new file mode 100644 index 00000000000..8758ff120ff --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-add/codex-config.toml @@ -0,0 +1 @@ +not = [valid diff --git a/acceptance/cmd/aitools/telemetry/mcp-add/out.test.toml b/acceptance/cmd/aitools/telemetry/mcp-add/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-add/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/cmd/aitools/telemetry/mcp-add/output.txt b/acceptance/cmd/aitools/telemetry/mcp-add/output.txt new file mode 100644 index 00000000000..132249dbdea --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-add/output.txt @@ -0,0 +1,94 @@ + +=== A new install for two agents: CHANGED per (service, agent); github needs sign-in, so its sign-in state is PENDING + +>>> [CLI] aitools mcp add --profile logfood --names system.ai.genie_one_mcp,system.ai.github --agents claude-code,codex --output json +{ + "calling_agent": "TYPE_UNSPECIFIED", + "operation": "ADD", + "servers": [ + { + "service": "GENIE_ONE", + "agent": "CLAUDE_CODE", + "outcome": "CHANGED", + "sign_in_state": "NOT_REQUIRED" + }, + { + "service": "GENIE_ONE", + "agent": "CODEX", + "outcome": "CHANGED", + "sign_in_state": "NOT_REQUIRED" + }, + { + "service": "GITHUB", + "agent": "CLAUDE_CODE", + "outcome": "CHANGED", + "sign_in_state": "PENDING" + }, + { + "service": "GITHUB", + "agent": "CODEX", + "outcome": "CHANGED", + "sign_in_state": "PENDING" + } + ], + "bound_profile_count": 1 +} + +=== The same servers again are re-asserted: UNCHANGED + +>>> [CLI] aitools mcp add --profile logfood --names system.ai.genie_one_mcp,system.ai.github --agents claude-code,codex --output json +{ + "calling_agent": "TYPE_UNSPECIFIED", + "operation": "ADD", + "servers": [ + { + "service": "GENIE_ONE", + "agent": "CLAUDE_CODE", + "outcome": "UNCHANGED", + "sign_in_state": "NOT_REQUIRED" + }, + { + "service": "GENIE_ONE", + "agent": "CODEX", + "outcome": "UNCHANGED", + "sign_in_state": "NOT_REQUIRED" + }, + { + "service": "GITHUB", + "agent": "CLAUDE_CODE", + "outcome": "UNCHANGED", + "sign_in_state": "PENDING" + }, + { + "service": "GITHUB", + "agent": "CODEX", + "outcome": "UNCHANGED", + "sign_in_state": "PENDING" + } + ], + "bound_profile_count": 1 +} + +=== An agent config that does not parse fails that agent only: FAILED with AGENT_CONFIG_INVALID + +>>> musterr [CLI] aitools mcp add --profile logfood --names system.ai.genie_one_mcp --agents claude-code,codex --output json +{ + "calling_agent": "TYPE_UNSPECIFIED", + "operation": "ADD", + "servers": [ + { + "service": "GENIE_ONE", + "agent": "CLAUDE_CODE", + "outcome": "UNCHANGED", + "sign_in_state": "NOT_REQUIRED" + }, + { + "service": "GENIE_ONE", + "agent": "CODEX", + "outcome": "FAILED", + "error_category": "AGENT_CONFIG_INVALID", + "sign_in_state": "NOT_REQUIRED" + } + ], + "bound_profile_count": 1 +} diff --git a/acceptance/cmd/aitools/telemetry/mcp-add/script b/acceptance/cmd/aitools/telemetry/mcp-add/script new file mode 100644 index 00000000000..6cf6f91e92a --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-add/script @@ -0,0 +1,22 @@ +sethome home +mkdir -p home/.claude home/.codex +envsubst < .databrickscfg > out && mv out .databrickscfg +export DATABRICKS_CONFIG_FILE=.databrickscfg + +# Seeding the fake is not an MCP command, so its uploads (if any) are dropped here. +MSYS_NO_PATHCONV=1 $CLI api post "/api/2.1/unity-catalog/mcp-services?parent=schemas/system.ai&mcp_service_id=genie_one_mcp" --profile logfood --json '{"comment":"Ask questions about your data"}' > /dev/null +MSYS_NO_PATHCONV=1 $CLI api post "/api/2.1/unity-catalog/mcp-services?parent=schemas/system.ai&mcp_service_id=github" --profile logfood --json '{"comment":"Repositories, issues, pull requests","config":{"source_connection":{"name":"connections/github","options":{"oauth_provider":"github"}}}}' > /dev/null +rm -f "$OUT_REQUESTS" + +title "A new install for two agents: CHANGED per (service, agent); github needs sign-in, so its sign-in state is PENDING" +trace $CLI aitools mcp add --profile logfood --names system.ai.genie_one_mcp,system.ai.github --agents claude-code,codex --output json > /dev/null +mcp_events + +title "The same servers again are re-asserted: UNCHANGED" +trace $CLI aitools mcp add --profile logfood --names system.ai.genie_one_mcp,system.ai.github --agents claude-code,codex --output json > /dev/null +mcp_events + +title "An agent config that does not parse fails that agent only: FAILED with AGENT_CONFIG_INVALID" +cp codex-config.toml home/.codex/config.toml +trace musterr $CLI aitools mcp add --profile logfood --names system.ai.genie_one_mcp --agents claude-code,codex --output json > /dev/null +mcp_events diff --git a/acceptance/cmd/aitools/telemetry/mcp-remove/.databrickscfg b/acceptance/cmd/aitools/telemetry/mcp-remove/.databrickscfg new file mode 100644 index 00000000000..053534e0c26 --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-remove/.databrickscfg @@ -0,0 +1,7 @@ +[logfood] +host = $DATABRICKS_HOST +token = test-token-logfood-$UNIQUE_NAME + +[dogfood] +host = $DATABRICKS_HOST +token = test-token-logfood-$UNIQUE_NAME diff --git a/acceptance/cmd/aitools/telemetry/mcp-remove/out.test.toml b/acceptance/cmd/aitools/telemetry/mcp-remove/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-remove/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/cmd/aitools/telemetry/mcp-remove/output.txt b/acceptance/cmd/aitools/telemetry/mcp-remove/output.txt new file mode 100644 index 00000000000..da2df1d87f2 --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-remove/output.txt @@ -0,0 +1,95 @@ + +=== Installed for two profiles: the bound profile count is 2 after the second add + +>>> [CLI] aitools mcp add --profile logfood --names system.ai.genie_one_mcp --agents claude-code --output json +{ + "calling_agent": "TYPE_UNSPECIFIED", + "operation": "ADD", + "servers": [ + { + "service": "GENIE_ONE", + "agent": "CLAUDE_CODE", + "outcome": "CHANGED", + "sign_in_state": "NOT_REQUIRED" + } + ], + "bound_profile_count": 1 +} + +>>> [CLI] aitools mcp add --profile dogfood --names system.ai.genie_one_mcp --agents claude-code --output json +{ + "calling_agent": "TYPE_UNSPECIFIED", + "operation": "ADD", + "servers": [ + { + "service": "GENIE_ONE", + "agent": "CLAUDE_CODE", + "outcome": "CHANGED", + "sign_in_state": "NOT_REQUIRED" + } + ], + "bound_profile_count": 2 +} + +=== remove --profile dogfood: CHANGED for its agent entry, and one profile stays bound + +>>> [CLI] aitools mcp remove --profile dogfood --output json +{ + "calling_agent": "TYPE_UNSPECIFIED", + "operation": "REMOVE", + "servers": [ + { + "service": "GENIE_ONE", + "agent": "CLAUDE_CODE", + "outcome": "CHANGED" + } + ], + "bound_profile_count": 1 +} + +=== A Databricks entry changed outside this CLI is left in place: SKIPPED + +>>> [CLI] aitools mcp remove --profile logfood --output json +{ + "calling_agent": "TYPE_UNSPECIFIED", + "operation": "REMOVE", + "servers": [ + { + "service": "GENIE_ONE", + "agent": "CLAUDE_CODE", + "outcome": "SKIPPED" + } + ], + "bound_profile_count": 1 +} + +=== An entry already gone from the agent config is CHANGED, and the state record goes + +>>> [CLI] aitools mcp remove --names genie_one_mcp-logfood --profile logfood --output json +{ + "calling_agent": "TYPE_UNSPECIFIED", + "operation": "REMOVE", + "servers": [ + { + "service": "GENIE_ONE", + "agent": "CLAUDE_CODE", + "outcome": "CHANGED" + } + ], + "bound_profile_count": 0 +} + +=== A record with no agent left goes with a REMOVE of its own: CHANGED with no agent, and no profile bound + +>>> [CLI] aitools mcp remove --names dbsql-logfood --profile logfood --output json +{ + "calling_agent": "TYPE_UNSPECIFIED", + "operation": "REMOVE", + "servers": [ + { + "service": "DBSQL", + "outcome": "CHANGED" + } + ], + "bound_profile_count": 0 +} diff --git a/acceptance/cmd/aitools/telemetry/mcp-remove/record-only.state.json b/acceptance/cmd/aitools/telemetry/mcp-remove/record-only.state.json new file mode 100644 index 00000000000..e40e76fff80 --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-remove/record-only.state.json @@ -0,0 +1,20 @@ +{ + "schema_version": 3, + "mcp": { + "servers": [ + { + "name": "dbsql-logfood", + "service": "system.ai.dbsql", + "profile": "logfood", + "host": "$DATABRICKS_HOST", + "workspace_id": "", + "sign_in_required": false, + "command": "", + "config_file": "$PWD/.databrickscfg", + "agents": {}, + "installed_at": "2026-01-01T00:00:00Z", + "cli_version": "0.0.0" + } + ] + } +} diff --git a/acceptance/cmd/aitools/telemetry/mcp-remove/script b/acceptance/cmd/aitools/telemetry/mcp-remove/script new file mode 100644 index 00000000000..32e8cb12cb9 --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-remove/script @@ -0,0 +1,33 @@ +sethome home +mkdir -p home/.claude +envsubst < .databrickscfg > out && mv out .databrickscfg +export DATABRICKS_CONFIG_FILE=.databrickscfg + +MSYS_NO_PATHCONV=1 $CLI api post "/api/2.1/unity-catalog/mcp-services?parent=schemas/system.ai&mcp_service_id=genie_one_mcp" --profile logfood --json '{"comment":"Ask questions about your data"}' > /dev/null +rm -f "$OUT_REQUESTS" + +title "Installed for two profiles: the bound profile count is 2 after the second add" +trace $CLI aitools mcp add --profile logfood --names system.ai.genie_one_mcp --agents claude-code --output json > /dev/null +mcp_events +trace $CLI aitools mcp add --profile dogfood --names system.ai.genie_one_mcp --agents claude-code --output json > /dev/null +mcp_events + +title "remove --profile dogfood: CHANGED for its agent entry, and one profile stays bound" +trace $CLI aitools mcp remove --profile dogfood --output json > /dev/null +mcp_events + +title "A Databricks entry changed outside this CLI is left in place: SKIPPED" +jq '.mcpServers["genie_one_mcp-logfood"].args |= map(if . == "logfood" then "other" else . end)' home/.claude.json > home/claude.tmp && mv home/claude.tmp home/.claude.json +trace $CLI aitools mcp remove --profile logfood --output json > /dev/null +mcp_events + +title "An entry already gone from the agent config is CHANGED, and the state record goes" +jq 'del(.mcpServers["genie_one_mcp-logfood"])' home/.claude.json > home/claude.tmp && mv home/claude.tmp home/.claude.json +trace $CLI aitools mcp remove --names genie_one_mcp-logfood --profile logfood --output json > /dev/null +mcp_events + +title "A record with no agent left goes with a REMOVE of its own: CHANGED with no agent, and no profile bound" +mkdir -p home/.databricks/aitools/skills +envsubst < record-only.state.json > home/.databricks/aitools/skills/.state.json +trace $CLI aitools mcp remove --names dbsql-logfood --profile logfood --output json > /dev/null +mcp_events diff --git a/acceptance/cmd/aitools/telemetry/mcp-serve/.databrickscfg b/acceptance/cmd/aitools/telemetry/mcp-serve/.databrickscfg new file mode 100644 index 00000000000..8d80c7ca7da --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-serve/.databrickscfg @@ -0,0 +1,3 @@ +[logfood] +host = $DATABRICKS_HOST +token = test-token-logfood-$UNIQUE_NAME diff --git a/acceptance/cmd/aitools/telemetry/mcp-serve/out.test.toml b/acceptance/cmd/aitools/telemetry/mcp-serve/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-serve/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/cmd/aitools/telemetry/mcp-serve/output.txt b/acceptance/cmd/aitools/telemetry/mcp-serve/output.txt new file mode 100644 index 00000000000..b710dbe56d4 --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-serve/output.txt @@ -0,0 +1,44 @@ + +=== serve uploads PROXY_STARTED after the binding checks, while it serves + +>>> [CLI] aitools mcp serve --profile logfood --service system.ai.genie_one_mcp --agent claude-code +No install record for system.ai.genie_one_mcp on profile "logfood"; using its current host [DATABRICKS_URL] +{ + "calling_agent": "CLAUDE_CODE", + "operation": "PROXY_STARTED", + "servers": [ + { + "service": "GENIE_ONE" + } + ], + "bound_profile_count": 0 +} + +=== A binding that fails logs no PROXY_STARTED, so no upload follows + +>>> musterr [CLI] aitools mcp serve --profile gone --service system.ai.genie_one_mcp --agent codex +Error: MCP server system.ai.genie_one_mcp needs Databricks profile "gone", which does not exist in [TEST_TMP_DIR]/.databrickscfg. Set it up with: databricks auth login --profile gone + +=== authenticate opens the sign-in page: PROXY_STARTED, then SIGN_IN_PROMPTED with the sign-in state after the wait +{ + "calling_agent": "CLAUDE_CODE", + "operation": "PROXY_STARTED", + "servers": [ + { + "service": "GITHUB" + } + ], + "bound_profile_count": 0 +} +{ + "calling_agent": "CLAUDE_CODE", + "operation": "SIGN_IN_PROMPTED", + "servers": [ + { + "service": "GITHUB", + "error_category": "SIGN_IN_REQUIRED", + "sign_in_state": "ACTIVE" + } + ], + "bound_profile_count": 0 +} diff --git a/acceptance/cmd/aitools/telemetry/mcp-serve/requests-after.ndjson b/acceptance/cmd/aitools/telemetry/mcp-serve/requests-after.ndjson new file mode 100644 index 00000000000..57d5e4a4066 --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-serve/requests-after.ndjson @@ -0,0 +1,2 @@ +{"jsonrpc":"2.0","id":4,"method":"tools/list"} +{"jsonrpc":"2.0","id":5,"method":"tools/call","params":{"name":"echo","arguments":{"text":"hello"}}} diff --git a/acceptance/cmd/aitools/telemetry/mcp-serve/requests-authenticate.ndjson b/acceptance/cmd/aitools/telemetry/mcp-serve/requests-authenticate.ndjson new file mode 100644 index 00000000000..cc792544a79 --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-serve/requests-authenticate.ndjson @@ -0,0 +1 @@ +{"jsonrpc":"2.0","id":3,"method":"tools/call","params":{"name":"authenticate","arguments":{}}} diff --git a/acceptance/cmd/aitools/telemetry/mcp-serve/requests-list.ndjson b/acceptance/cmd/aitools/telemetry/mcp-serve/requests-list.ndjson new file mode 100644 index 00000000000..84c14278376 --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-serve/requests-list.ndjson @@ -0,0 +1,3 @@ +{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-06-18","capabilities":{},"clientInfo":{"name":"test-client","version":"0.0.1"}}} +{"jsonrpc":"2.0","method":"notifications/initialized"} +{"jsonrpc":"2.0","id":2,"method":"tools/list"} diff --git a/acceptance/cmd/aitools/telemetry/mcp-serve/script b/acceptance/cmd/aitools/telemetry/mcp-serve/script new file mode 100644 index 00000000000..acda2bbd340 --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-serve/script @@ -0,0 +1,32 @@ +sethome home +envsubst < .databrickscfg > out && mv out .databrickscfg +export DATABRICKS_CONFIG_FILE=.databrickscfg + +# No browser: the proxy prints the sign-in URL and does not open anything. +export BROWSER=none +export RETRY_MAX_ATTEMPTS=60 +# Seeding the fake is not an MCP command, so its uploads (if any) are dropped here. +MSYS_NO_PATHCONV=1 $CLI api post "/api/2.1/unity-catalog/mcp-services?parent=schemas/system.ai&mcp_service_id=genie_one_mcp" --profile logfood --json '{"comment":"Ask questions about your data"}' > /dev/null +MSYS_NO_PATHCONV=1 $CLI api post "/api/2.1/unity-catalog/mcp-services?parent=schemas/system.ai&mcp_service_id=github" --profile logfood --json '{"comment":"Repositories, issues, pull requests","config":{"source_connection":{"name":"connections/github","options":{"oauth_provider":"github"}}}}' > /dev/null +rm -f "$OUT_REQUESTS" + +title "serve uploads PROXY_STARTED after the binding checks, while it serves" +trace $CLI aitools mcp serve --profile logfood --service system.ai.genie_one_mcp --agent claude-code < requests-list.ndjson > /dev/null +mcp_events + +title "A binding that fails logs no PROXY_STARTED, so no upload follows" +trace musterr $CLI aitools mcp serve --profile gone --service system.ai.genie_one_mcp --agent codex < /dev/null > /dev/null +mcp_events + +title "authenticate opens the sign-in page: PROXY_STARTED, then SIGN_IN_PROMPTED with the sign-in state after the wait" +{ + cat requests-list.ndjson + retry --until '"id":2' cat serve.stdout.txt > /dev/null + cat requests-authenticate.ndjson + retry --until "Waiting for sign-in" cat serve.stderr.txt > /dev/null + MSYS_NO_PATHCONV=1 $CLI api post /api/2.1/unity-catalog/mcp-services/system.ai.github/user-credentials --profile logfood --json '{}' > /dev/null + retry --until '"id":3' cat serve.stdout.txt > /dev/null + cat requests-after.ndjson +} | $CLI aitools mcp serve --profile logfood --service system.ai.github --agent claude-code > serve.stdout.txt 2> serve.stderr.txt +mcp_events +rm serve.stdout.txt serve.stderr.txt diff --git a/acceptance/cmd/aitools/telemetry/mcp-sign-in/.databrickscfg b/acceptance/cmd/aitools/telemetry/mcp-sign-in/.databrickscfg new file mode 100644 index 00000000000..8d80c7ca7da --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-sign-in/.databrickscfg @@ -0,0 +1,3 @@ +[logfood] +host = $DATABRICKS_HOST +token = test-token-logfood-$UNIQUE_NAME diff --git a/acceptance/cmd/aitools/telemetry/mcp-sign-in/out.test.toml b/acceptance/cmd/aitools/telemetry/mcp-sign-in/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-sign-in/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/cmd/aitools/telemetry/mcp-sign-in/output.txt b/acceptance/cmd/aitools/telemetry/mcp-sign-in/output.txt new file mode 100644 index 00000000000..3137ec011a5 --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-sign-in/output.txt @@ -0,0 +1,106 @@ + +=== login to a service that needs no sign-in: UNCHANGED, sign-in state NOT_REQUIRED + +>>> [CLI] aitools mcp login system.ai.genie_one_mcp --profile logfood +system.ai.genie_one_mcp does not need sign-in. +{ + "calling_agent": "TYPE_UNSPECIFIED", + "operation": "SIGN_IN", + "servers": [ + { + "service": "GENIE_ONE", + "outcome": "UNCHANGED", + "sign_in_state": "NOT_REQUIRED" + } + ], + "bound_profile_count": 0 +} + +=== login completes once the credential is ACTIVE: CHANGED, sign-in state ACTIVE +{ + "calling_agent": "TYPE_UNSPECIFIED", + "operation": "SIGN_IN", + "servers": [ + { + "service": "GITHUB", + "outcome": "CHANGED", + "sign_in_state": "ACTIVE" + } + ], + "bound_profile_count": 0 +} + +=== login while already signed in: UNCHANGED + +>>> [CLI] aitools mcp login system.ai.github --profile logfood +Already signed in to system.ai.github on profile logfood. +Public repositories only until a repository owner installs https://github.com/apps/databricks-github-connector (on AWS GovCloud, https://github.com/apps/databricks-gh-connector-gov-cloud). +{ + "calling_agent": "TYPE_UNSPECIFIED", + "operation": "SIGN_IN", + "servers": [ + { + "service": "GITHUB", + "outcome": "UNCHANGED", + "sign_in_state": "ACTIVE" + } + ], + "bound_profile_count": 0 +} + +=== logout removes the sign-in: CHANGED; a second logout finds nothing: UNCHANGED + +>>> [CLI] aitools mcp logout system.ai.github --profile logfood +Signed out of system.ai.github on profile logfood. +{ + "calling_agent": "TYPE_UNSPECIFIED", + "operation": "SIGN_OUT", + "servers": [ + { + "service": "GITHUB", + "outcome": "CHANGED" + } + ], + "bound_profile_count": 0 +} + +>>> [CLI] aitools mcp logout system.ai.github --profile logfood +Not signed in to system.ai.github on profile logfood. +{ + "calling_agent": "TYPE_UNSPECIFIED", + "operation": "SIGN_OUT", + "servers": [ + { + "service": "GITHUB", + "outcome": "UNCHANGED" + } + ], + "bound_profile_count": 0 +} + +=== add --login signs in during the same run: SIGN_IN first, then ADD with sign-in state ACTIVE +{ + "calling_agent": "TYPE_UNSPECIFIED", + "operation": "SIGN_IN", + "servers": [ + { + "service": "GITHUB", + "outcome": "CHANGED", + "sign_in_state": "ACTIVE" + } + ], + "bound_profile_count": 1 +} +{ + "calling_agent": "TYPE_UNSPECIFIED", + "operation": "ADD", + "servers": [ + { + "service": "GITHUB", + "agent": "CLAUDE_CODE", + "outcome": "CHANGED", + "sign_in_state": "ACTIVE" + } + ], + "bound_profile_count": 1 +} diff --git a/acceptance/cmd/aitools/telemetry/mcp-sign-in/script b/acceptance/cmd/aitools/telemetry/mcp-sign-in/script new file mode 100644 index 00000000000..6ddd1bce499 --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-sign-in/script @@ -0,0 +1,44 @@ +sethome home +mkdir -p home/.claude +envsubst < .databrickscfg > out && mv out .databrickscfg +export DATABRICKS_CONFIG_FILE=.databrickscfg + +# No browser: the CLI prints the sign-in URL and polls. +export BROWSER=none +export RETRY_MAX_ATTEMPTS=60 +# Seeding the fake is not an MCP command, so its uploads (if any) are dropped here. +MSYS_NO_PATHCONV=1 $CLI api post "/api/2.1/unity-catalog/mcp-services?parent=schemas/system.ai&mcp_service_id=genie_one_mcp" --profile logfood --json '{"comment":"Ask questions about your data"}' > /dev/null +MSYS_NO_PATHCONV=1 $CLI api post "/api/2.1/unity-catalog/mcp-services?parent=schemas/system.ai&mcp_service_id=github" --profile logfood --json '{"comment":"Repositories, issues, pull requests","config":{"source_connection":{"name":"connections/github","options":{"oauth_provider":"github"}}}}' > /dev/null +rm -f "$OUT_REQUESTS" + +title "login to a service that needs no sign-in: UNCHANGED, sign-in state NOT_REQUIRED" +trace $CLI aitools mcp login system.ai.genie_one_mcp --profile logfood > /dev/null +mcp_events + +title "login completes once the credential is ACTIVE: CHANGED, sign-in state ACTIVE" +$CLI aitools mcp login system.ai.github --profile logfood > login.txt 2>&1 & +PID=$! +retry --until "Waiting for sign-in" cat login.txt > /dev/null +MSYS_NO_PATHCONV=1 $CLI api post /api/2.1/unity-catalog/mcp-services/system.ai.github/user-credentials --profile logfood --json '{}' > /dev/null +wait $PID +mcp_events +rm login.txt + +title "login while already signed in: UNCHANGED" +trace $CLI aitools mcp login system.ai.github --profile logfood > /dev/null +mcp_events + +title "logout removes the sign-in: CHANGED; a second logout finds nothing: UNCHANGED" +trace $CLI aitools mcp logout system.ai.github --profile logfood > /dev/null +mcp_events +trace $CLI aitools mcp logout system.ai.github --profile logfood > /dev/null +mcp_events + +title "add --login signs in during the same run: SIGN_IN first, then ADD with sign-in state ACTIVE" +$CLI aitools mcp add --profile logfood --names system.ai.github --agents claude-code --login > add.txt 2>&1 & +PID=$! +retry --until "Waiting for sign-in" cat add.txt > /dev/null +MSYS_NO_PATHCONV=1 $CLI api post /api/2.1/unity-catalog/mcp-services/system.ai.github/user-credentials --profile logfood --json '{}' > /dev/null +wait $PID +mcp_events +rm add.txt diff --git a/acceptance/cmd/aitools/telemetry/script.prepare b/acceptance/cmd/aitools/telemetry/script.prepare index 9d3b1d18c4a..c75556a3c1b 100644 --- a/acceptance/cmd/aitools/telemetry/script.prepare +++ b/acceptance/cmd/aitools/telemetry/script.prepare @@ -1,3 +1,11 @@ # A host that disables telemetry or sets a zero upload timeout would upload # nothing, so these cases would record no events. unset DATABRICKS_CLI_DISABLE_TELEMETRY DATABRICKS_CLI_TELEMETRY_TIMEOUT + +# Prints the aitools_mcp_event payloads uploaded since the last call, then forgets the recorded uploads. The uploads are +# filtered by path with print_requests.py; jq reads only the body. +mcp_events() { + if [ -f "$OUT_REQUESTS" ]; then + print_requests.py //telemetry-ext | jq '.body.protoLogs[] | fromjson | .entry.databricks_cli_log.aitools_mcp_event | select(. != null)' + fi +} diff --git a/cmd/aitools/autoinstall_run.go b/cmd/aitools/autoinstall_run.go index c071ab29b96..dfc3079014a 100644 --- a/cmd/aitools/autoinstall_run.go +++ b/cmd/aitools/autoinstall_run.go @@ -22,6 +22,7 @@ import ( "github.com/databricks/cli/libs/dbr" "github.com/databricks/cli/libs/flags" "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/log/handler" "github.com/databricks/cli/libs/telemetry" "github.com/databricks/cli/libs/telemetry/protos" "github.com/databricks/cli/libs/versioncheck" @@ -621,5 +622,5 @@ func changedAgents(ctx context.Context, before, after *installer.InstallState) [ // only in the debug log. func quietContext(ctx context.Context) context.Context { ctx = cmdio.InContext(ctx, cmdio.NewIO(ctx, flags.OutputText, nil, io.Discard, io.Discard, "", "")) - return log.NewContext(ctx, slog.New(debugOnlyHandler{log.GetLogger(ctx).Handler()})) + return log.NewContext(ctx, slog.New(handler.DebugOnlyHandler{Handler: log.GetLogger(ctx).Handler()})) } diff --git a/cmd/aitools/mcp.go b/cmd/aitools/mcp.go index ee8dd76ec85..edc6ed34b4b 100644 --- a/cmd/aitools/mcp.go +++ b/cmd/aitools/mcp.go @@ -17,19 +17,6 @@ import ( "github.com/spf13/cobra" ) -// Error categories that the telemetry enum does not define yet. The JSON output reports them by name. The -// binding categories come from mcp.BindCategory, which has the same names. -const ( - categoryAgentConfigInvalid protos.AitoolsErrorCategory = "AGENT_CONFIG_INVALID" - categoryNameConflict protos.AitoolsErrorCategory = "MCP_NAME_CONFLICT" - categoryServiceNotFound protos.AitoolsErrorCategory = "MCP_SERVICE_NOT_FOUND" - categoryNoProfile protos.AitoolsErrorCategory = "NO_PROFILE" - categoryProfileRequired protos.AitoolsErrorCategory = "PROFILE_REQUIRED" - categoryPermissionDenied protos.AitoolsErrorCategory = "MCP_PERMISSION_DENIED" - categoryLocationNotFound protos.AitoolsErrorCategory = "MCP_LOCATION_NOT_FOUND" - categorySignInTimeout protos.AitoolsErrorCategory = "SIGN_IN_TIMEOUT" -) - // NewMcpCmd returns the `aitools mcp` group. It stays hidden until the whole MCP // stack has landed, so a release never ships `add` without `serve`. func NewMcpCmd() *cobra.Command { @@ -84,9 +71,9 @@ func errorCategory(err error) protos.AitoolsErrorCategory { } switch { case errors.Is(err, agentconfig.ErrConfigInvalid): - return categoryAgentConfigInvalid + return protos.AitoolsErrorCategoryAgentConfigInvalid case errors.Is(err, agentconfig.ErrNotOwned): - return categoryNameConflict + return protos.AitoolsErrorCategoryMCPNameConflict } return classifyInstallError(err) } @@ -116,11 +103,11 @@ func resolveProfile(ctx context.Context, flagValue string) (string, error) { } switch len(profiles) { case 0: - return "", failure(categoryNoProfile, "no workspace profiles in %s. Run: databricks auth login --host --profile ", filepath.ToSlash(path)) + return "", failure(protos.AitoolsErrorCategoryNoProfile, "no workspace profiles in %s. Run: databricks auth login --host --profile ", filepath.ToSlash(path)) case 1: return profiles[0].Name, nil } - return "", failureWithCause(categoryProfileRequired, &profileChoice{profiles: profiles}, "several workspace profiles exist; pass --profile. Found: %s", strings.Join(profiles.Names(), ", ")) + return "", failureWithCause(protos.AitoolsErrorCategoryProfileRequired, &profileChoice{profiles: profiles}, "several workspace profiles exist; pass --profile. Found: %s", strings.Join(profiles.Names(), ", ")) } // mcpWorkspaceProfile reports whether auto-detect may pick p. It accepts the profiles that --profile accepts: a diff --git a/cmd/aitools/mcp_add.go b/cmd/aitools/mcp_add.go index 7e17fbb1252..c751d8a8b45 100644 --- a/cmd/aitools/mcp_add.go +++ b/cmd/aitools/mcp_add.go @@ -188,6 +188,11 @@ Supported agents: ` + strings.Join(mcpAgentNames(), ", "), Servers: []addServerOutput{}, } runErr := runMcpAdd(ctx, cmd, f, jsonMode, out) + logMcpEvent(ctx, protos.AitoolsMcpEvent{ + CallingAgent: callingAgentType(), + Operation: protos.AitoolsMcpOperationAdd, + Servers: addServerResults(out.Servers), + }) if !jsonMode { return runErr } @@ -361,6 +366,9 @@ func runMcpAdd(ctx context.Context, cmd *cobra.Command, f mcpAddFlags, jsonMode if err := planningRefusals(plans, targets, refused); err != nil { return err } + // Every refusal that needs no request has come, so the upload authenticates with the profile this run binds. Resolving + // its config makes the host's metadata request, which is why it waits for the refusals. + setUploadConfig(ctx, profileName, configFile) if !pickingServices { w, workspaceID, err = connectWorkspace(ctx, out, profileName, configFile, pinned) @@ -616,9 +624,9 @@ func describeListError(profileName string, err error) error { var category protos.AitoolsErrorCategory switch listErr.Kind { case mcp.ListErrLocationNotFound: - category = categoryLocationNotFound + category = protos.AitoolsErrorCategoryMCPLocationNotFound case mcp.ListErrPermissionDenied: - category = categoryPermissionDenied + category = protos.AitoolsErrorCategoryMCPPermissionDenied case mcp.ListErrRateLimited: category = protos.AitoolsErrorCategoryRateLimited default: @@ -807,13 +815,13 @@ func pickServices(location string, names []string, services []mcp.Service) ([]mc i := slices.IndexFunc(services, func(s mcp.Service) bool { return strings.EqualFold(s.FullName, name) }) if i < 0 { if len(services) == 0 { - return nil, failure(categoryServiceNotFound, "%s is not an MCP service in %s, which has none", name, location) + return nil, failure(protos.AitoolsErrorCategoryMCPServiceNotFound, "%s is not an MCP service in %s, which has none", name, location) } available := make([]string, 0, len(services)) for _, s := range services { available = append(available, s.FullName) } - return nil, failure(categoryServiceNotFound, "%s is not an MCP service in %s. Available: %s", name, location, strings.Join(available, ", ")) + return nil, failure(protos.AitoolsErrorCategoryMCPServiceNotFound, "%s is not an MCP service in %s. Available: %s", name, location, strings.Join(available, ", ")) } picked = append(picked, services[i]) } @@ -895,7 +903,7 @@ func planAdds(ctx context.Context, in planInput, state *mcp.State, fullNames []s } // A name recovered from an agent entry can repeat across services only through a hand edit; two plans must not share it. if chosen[name] { - return nil, failure(categoryNameConflict, "%s and another requested server both use %s; remove one of the entries first", fullName, name) + return nil, failure(protos.AitoolsErrorCategoryMCPNameConflict, "%s and another requested server both use %s; remove one of the entries first", fullName, name) } chosen[name] = true taken[name] = true @@ -953,7 +961,7 @@ func chooseName(ctx context.Context, in planInput, state *mcp.State, fullName st } // The agents must hold the state's name, or the run would add a second name for the same server beside theirs. if slices.ContainsFunc(owned, func(name string) bool { return name != stored }) { - return "", nil, "", failure(categoryNameConflict, "%s is installed on profile %s as %s, but agents hold it as %s; remove the extra entries", fullName, b.profile, stored, strings.Join(owned, ", ")) + return "", nil, "", failure(protos.AitoolsErrorCategoryMCPNameConflict, "%s is installed on profile %s as %s, but agents hold it as %s; remove the extra entries", fullName, b.profile, stored, strings.Join(owned, ", ")) } } else { switch len(owned) { @@ -961,7 +969,7 @@ func chooseName(ctx context.Context, in planInput, state *mcp.State, fullName st case 1: stored = owned[0] default: - return "", nil, "", failure(categoryNameConflict, "%s is installed on profile %s under more than one name in the agents (%s); remove the extra entries", fullName, b.profile, strings.Join(owned, ", ")) + return "", nil, "", failure(protos.AitoolsErrorCategoryMCPNameConflict, "%s is installed on profile %s under more than one name in the agents (%s); remove the extra entries", fullName, b.profile, strings.Join(owned, ", ")) } } @@ -977,7 +985,7 @@ func chooseName(ctx context.Context, in planInput, state *mcp.State, fullName st return "", nil, "", failure(protos.AitoolsErrorCategoryInvalidFlags, "%s is already installed on profile %s as %s; remove it before using --name %s", fullName, b.profile, stored, in.nameFlag) } if stateHolds(state, stored, entry) { - return "", nil, "", failure(categoryNameConflict, "%s is installed as %s, but another server in the state holds %s; remove that entry first", fullName, stored, stored) + return "", nil, "", failure(protos.AitoolsErrorCategoryMCPNameConflict, "%s is installed as %s, but another server in the state holds %s; remove that entry first", fullName, stored, stored) } failed, err := nameFailures(ctx, in, fullName, stored, true) return stored, failed, "", err @@ -1087,7 +1095,7 @@ func holderConflict(ctx context.Context, in planInput, a *agents.Agent, fullName if in.force { return nil, nil } - return failure(categoryNameConflict, "%s already has a server named %s that this CLI did not write; pass --force to replace it", a.DisplayName, name), nil + return failure(protos.AitoolsErrorCategoryMCPNameConflict, "%s already has a server named %s that this CLI did not write; pass --force to replace it", a.DisplayName, name), nil } serves, err := mcp.EntryServes(ctx, entry, fullName, in.binding.profile, in.binding.configFile) if err != nil { @@ -1097,7 +1105,7 @@ func holderConflict(ctx context.Context, in planInput, a *agents.Agent, fullName case serves: return nil, nil case stored: - return failure(categoryNameConflict, "%s is installed as %s, but %s holds %s for another server; remove that entry first", fullName, name, a.Name, name), nil + return failure(protos.AitoolsErrorCategoryMCPNameConflict, "%s is installed as %s, but %s holds %s for another server; remove that entry first", fullName, name, a.Name, name), nil default: return failure(protos.AitoolsErrorCategoryInvalidFlags, "--name %s is used by another server in %s; pick a different name", name, a.Name), nil } @@ -1250,6 +1258,7 @@ func signInInstalled(ctx context.Context, errOut io.Writer, w *databricks.Worksp if signedIn(ctx, w, plans[i].service.FullName) { server.SignIn = &addSignInOutput{State: signInStateActive} reportSignedIn(errOut, plans[i].service, req.profile, true) + logMcpEvent(ctx, signInEventOf(server.Service, signInStateActive, true, nil)) continue } yes, err := promptSignInNow(server.Service) @@ -1264,11 +1273,16 @@ func signInInstalled(ctx context.Context, errOut io.Writer, w *databricks.Worksp } } var state string + var alreadySignedIn bool + // A sign-in whose URL cannot be built is logged as a failed sign-in. The install record keeps no state for it. + eventState := signInStateFailed url, err := mcp.SignInURL(w.Config.Host, plans[i].service.FullName, req.workspaceID) if err == nil { - state, _, _, err = signIn(ctx, errOut, w, req.pinned, url, plans[i].service, signInOptions{Profile: req.profile, Open: browser.OpenOutputToStderr}) + state, alreadySignedIn, _, err = signIn(ctx, errOut, w, req.pinned, url, plans[i].service, signInOptions{Profile: req.profile, Open: browser.OpenOutputToStderr}) + eventState = state } server.SignIn = &addSignInOutput{State: state} + logMcpEvent(ctx, signInEventOf(server.Service, eventState, alreadySignedIn, err)) if err != nil { server.SignIn.ErrorCategory = errorCategory(err) server.SignIn.Error = err.Error() diff --git a/cmd/aitools/mcp_add_test.go b/cmd/aitools/mcp_add_test.go index 704b6dc0124..ff9649e4ec0 100644 --- a/cmd/aitools/mcp_add_test.go +++ b/cmd/aitools/mcp_add_test.go @@ -207,7 +207,7 @@ func TestPickServices(t *testing.T) { _, err = pickServices("system.ai", []string{"system.ai.foo"}, services) require.Error(t, err) assert.Equal(t, "system.ai.foo is not an MCP service in system.ai. Available: system.ai.genie_one_mcp, system.ai.github", err.Error()) - assert.Equal(t, categoryServiceNotFound, errorCategory(err)) + assert.Equal(t, protos.AitoolsErrorCategoryMCPServiceNotFound, errorCategory(err)) } func TestResolveMCPAgents(t *testing.T) { @@ -257,7 +257,7 @@ func TestDescribeListErrorNamesProfileAndKeepsCause(t *testing.T) { err := describeListError("logfood", listErr) assert.Equal(t, "profile logfood: "+listErr.Error(), err.Error()) - assert.Equal(t, categoryPermissionDenied, errorCategory(err)) + assert.Equal(t, protos.AitoolsErrorCategoryMCPPermissionDenied, errorCategory(err)) got, ok := errors.AsType[*mcp.ListError](err) require.True(t, ok) @@ -268,7 +268,7 @@ func TestDescribeListErrorNamesProfileAndKeepsCause(t *testing.T) { assert.Equal(t, protos.AitoolsErrorCategoryRateLimited, errorCategory(rateLimited)) notFound := describeListError("logfood", &mcp.ListError{Kind: mcp.ListErrLocationNotFound, Location: "main.tools", Err: cause}) - assert.Equal(t, categoryLocationNotFound, errorCategory(notFound)) + assert.Equal(t, protos.AitoolsErrorCategoryMCPLocationNotFound, errorCategory(notFound)) plain := errors.New("network down") assert.Equal(t, plain, describeListError("logfood", plain)) @@ -313,7 +313,8 @@ func withArg0(t *testing.T, arg0 string) { } // noRequestServer returns a server that fails the test on any request. A command that refuses before -// it contacts the profile's host never reaches it. +// it contacts the profile's host never reaches it, with telemetry on or off: the upload's config is resolved only after +// the binding checks have passed, and resolving it is what makes the host metadata request. func noRequestServer(t *testing.T) *httptest.Server { t.Helper() server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -553,7 +554,7 @@ func TestPlanAddsNaming(t *testing.T) { require.NoError(t, os.WriteFile(filepath.Join(openCode, "opencode.json"), []byte(`{"mcp":{"servers":{"two":{"type":"local","command":["databricks","aitools","mcp","serve","--profile","logfood","--service","system.ai.github","--agent","opencode"]}}}}`), 0o600)) _, err := planNames(t, "", defaultBinding(t, "logfood"), &mcp.State{}, []string{"system.ai.github"}, []*agents.Agent{claude, agents.ByName("opencode")}) require.Error(t, err) - assert.Equal(t, categoryNameConflict, errorCategory(err)) + assert.Equal(t, protos.AitoolsErrorCategoryMCPNameConflict, errorCategory(err)) }) t.Run("OpenCode entry with its variables in environment is reused", func(t *testing.T) { @@ -596,7 +597,7 @@ func TestPlanAddsNaming(t *testing.T) { require.Len(t, plans, 1) assert.Equal(t, "github-logfood", plans[0].name) require.Contains(t, plans[0].failed, claude) - assert.Equal(t, categoryNameConflict, errorCategory(plans[0].failed[claude])) + assert.Equal(t, protos.AitoolsErrorCategoryMCPNameConflict, errorCategory(plans[0].failed[claude])) }) t.Run("planning reads the agent entries it is given, not the files", func(t *testing.T) { @@ -885,7 +886,7 @@ func TestAgentFailureErrorWhenAServiceReachedNoAgent(t *testing.T) { } func TestErrorCategoryMapsTypedFailures(t *testing.T) { - assert.Equal(t, categoryNameConflict, + assert.Equal(t, protos.AitoolsErrorCategoryMCPNameConflict, errorCategory(fmt.Errorf("claude-code: server %q: %w", "github", agentconfig.ErrNotOwned))) assert.Equal(t, protos.AitoolsErrorCategoryTimeout, errorCategory(fmt.Errorf("list: %w", context.DeadlineExceeded))) assert.Equal(t, protos.AitoolsErrorCategoryPermissionDenied, errorCategory(&fs.PathError{Op: "open", Path: "x", Err: fs.ErrPermission})) @@ -962,7 +963,7 @@ func TestMcpAddRefusesUnreadableDetectedAgentBeforeAnyRequest(t *testing.T) { err := runAdd(t, "--profile", "logfood", "--names", "github") require.Error(t, err) - assert.Equal(t, categoryAgentConfigInvalid, errorCategory(err)) + assert.Equal(t, protos.AitoolsErrorCategoryAgentConfigInvalid, errorCategory(err)) } // TestValidateMCPAddFlagsRefusesAgentsWithNoName checks that --agents holding no agent name is refused, rather than read as @@ -1077,7 +1078,7 @@ func TestMcpAddRefusesStoredNameHeldByOtherBinding(t *testing.T) { err := runAdd(t, "--profile", "logfood", "--names", "github", "--agents", "claude-code") require.Error(t, err) - assert.Equal(t, categoryNameConflict, errorCategory(err)) + assert.Equal(t, protos.AitoolsErrorCategoryMCPNameConflict, errorCategory(err)) assert.ErrorContains(t, err, "github-logfood") got, err := os.ReadFile(claudeFile) require.NoError(t, err) @@ -1253,7 +1254,7 @@ func TestPlanAddsRefusesRecoveredNameHeldByAnotherServer(t *testing.T) { _, err := planNames(t, "", defaultBinding(t, "logfood"), state, []string{"system.ai.github"}, []*agents.Agent{agents.ByName("claude-code")}) require.Error(t, err) - assert.Equal(t, categoryNameConflict, errorCategory(err)) + assert.Equal(t, protos.AitoolsErrorCategoryMCPNameConflict, errorCategory(err)) assert.Equal(t, "system.ai.github is installed as github-logfood, but another server in the state holds github-logfood; remove that entry first", err.Error()) } @@ -1313,7 +1314,7 @@ func TestMcpAddRefusesUnmanagedEntryBeforeAnyRequest(t *testing.T) { err := runAdd(t, "--profile", "logfood", "--names", "github", "--agents", "claude-code") require.Error(t, err) - assert.Equal(t, categoryNameConflict, errorCategory(err)) + assert.Equal(t, protos.AitoolsErrorCategoryMCPNameConflict, errorCategory(err)) assert.ErrorContains(t, err, "Claude Code already has a server named github-logfood that this CLI did not write; pass --force to replace it") got, err := os.ReadFile(claudeFile) require.NoError(t, err) @@ -1495,14 +1496,14 @@ func TestPlanAddsDetectedAgentConflictFailsOnlyThatAgent(t *testing.T) { require.Len(t, plans, 1) assert.Equal(t, "github-logfood", plans[0].name) require.Contains(t, plans[0].failed, claude) - assert.Equal(t, categoryNameConflict, errorCategory(plans[0].failed[claude])) + assert.Equal(t, protos.AitoolsErrorCategoryMCPNameConflict, errorCategory(plans[0].failed[claude])) assert.NotContains(t, plans[0].failed, codex) assert.NoError(t, planningRefusals(plans, targets, nil)) in.explicit = true _, err = planAdds(ctx, in, state, []string{"system.ai.github"}) require.Error(t, err) - assert.Equal(t, categoryNameConflict, errorCategory(err)) + assert.Equal(t, protos.AitoolsErrorCategoryMCPNameConflict, errorCategory(err)) onlyClaude := []*agents.Agent{claude} in = planInput{binding: defaultBinding(t, "logfood"), targets: onlyClaude, held: readHeld(ctx, onlyClaude)} @@ -1510,7 +1511,7 @@ func TestPlanAddsDetectedAgentConflictFailsOnlyThatAgent(t *testing.T) { require.NoError(t, err) refusal := planningRefusals(plans, onlyClaude, nil) require.Error(t, refusal) - assert.Equal(t, categoryNameConflict, errorCategory(refusal)) + assert.Equal(t, protos.AitoolsErrorCategoryMCPNameConflict, errorCategory(refusal)) } // TestPlanAddsRefusedAgentConflictDoesNotRefuseRun checks that with --agents, a name conflict in an agent that is refused, and so @@ -1547,7 +1548,7 @@ func TestPlanAddsUnmanagedEntryIsReplacedOnlyWithForce(t *testing.T) { in := planInput{binding: defaultBinding(t, "logfood"), targets: targets, held: held, explicit: true} _, err := planAdds(ctx, in, state, []string{"system.ai.github"}) require.Error(t, err) - assert.Equal(t, categoryNameConflict, errorCategory(err)) + assert.Equal(t, protos.AitoolsErrorCategoryMCPNameConflict, errorCategory(err)) in.force = true plans, err := planAdds(ctx, in, state, []string{"system.ai.github"}) @@ -1560,7 +1561,7 @@ func TestPickServicesNamesSchemaWithNoServices(t *testing.T) { _, err := pickServices("system.ai", []string{"system.ai.github"}, nil) require.Error(t, err) assert.Equal(t, "system.ai.github is not an MCP service in system.ai, which has none", err.Error()) - assert.Equal(t, categoryServiceNotFound, errorCategory(err)) + assert.Equal(t, protos.AitoolsErrorCategoryMCPServiceNotFound, errorCategory(err)) } // TestInstallServerWritesOnlyAgentsInstalledForMCP checks that an agent named with --agents is written when it counts as installed for MCP, and that diff --git a/cmd/aitools/mcp_list.go b/cmd/aitools/mcp_list.go index f77c6ac243e..50673533cd2 100644 --- a/cmd/aitools/mcp_list.go +++ b/cmd/aitools/mcp_list.go @@ -134,6 +134,16 @@ type profileKey struct { path string } +// profileGroup returns the group of a server entry in list: its profile in the config file that check resolved, or in the +// file as written when that file does not resolve. +func profileGroup(e mcp.Entry, check mcp.EntryCheck) profileKey { + path := check.Path + if path == "" { + path = e.ConfigFile + } + return profileKey{profile: e.Profile, path: path} +} + // NewMcpListCmd returns `aitools mcp list`. Without --check it reads the state file and the agent configs only, and // makes no network calls. Drift and unreadable entries are reported, not failed: the command exits 0. func NewMcpListCmd() *cobra.Command { @@ -247,12 +257,8 @@ func buildMcpList(ctx context.Context, profileName string) (mcpListOutput, error warnedFiles := map[string]bool{} for i, e := range entries { check := checks[i] - path := check.Path - if path == "" { - // The file cannot be resolved, so the entry is grouped under its file as written. - path = e.ConfigFile - } - key := profileKey{profile: e.Profile, path: path} + key := profileGroup(e, check) + path := key.path p, ok := groups[key] if !ok { p = &mcpListProfile{Profile: e.Profile, Host: e.Host, Status: statusOK, Servers: []mcpListServer{}, configPath: path} diff --git a/cmd/aitools/mcp_login.go b/cmd/aitools/mcp_login.go index c2f2da23d58..99b38f4fbc6 100644 --- a/cmd/aitools/mcp_login.go +++ b/cmd/aitools/mcp_login.go @@ -45,6 +45,9 @@ type mcpSignInOutput struct { URL string `json:"url,omitempty"` ErrorCategory protos.AitoolsErrorCategory `json:"error_category,omitempty"` Error string `json:"error,omitempty"` + + // alreadySignedIn is set by login when the caller was signed in before it ran. + alreadySignedIn bool } // mcpSignInAction runs login or logout against the profile's client and records the result in out. pinnedFile is the @@ -65,7 +68,7 @@ sign-in to complete. Without a browser, open the printed URL yourself. databricks aitools mcp login system.ai.github --profile logfood`, Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - return runMcpSignInCommand(cmd, args[0], mcpLogin) + return runMcpSignInCommand(cmd, args[0], mcpLogin, protos.AitoolsMcpOperationSignIn) }, } } @@ -83,19 +86,23 @@ stays in place. Running it when you are not signed in is not an error. databricks aitools mcp logout system.ai.github --profile logfood`, Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - return runMcpSignInCommand(cmd, args[0], mcpLogout) + return runMcpSignInCommand(cmd, args[0], mcpLogout, protos.AitoolsMcpOperationSignOut) }, } } // runMcpSignInCommand resolves the profile for service and runs action. In JSON mode the document is // the only stdout output; progress goes to stderr. -func runMcpSignInCommand(cmd *cobra.Command, service string, action mcpSignInAction) error { +func runMcpSignInCommand(cmd *cobra.Command, service string, action mcpSignInAction, operation protos.AitoolsMcpOperation) error { ctx := cmd.Context() jsonMode := installOutputIsJSON(cmd) out := &mcpSignInOutput{Service: service} runErr := runMcpSignIn(ctx, cmd.ErrOrStderr(), profileFlag(cmd), service, action, out) + // The host is set once the profile's binding has passed and its client exists, so a run that reached no server logs nothing. + if out.Host != "" { + logMcpEvent(ctx, signInEvent(operation, out, runErr)) + } if !jsonMode { return runErr } @@ -153,6 +160,7 @@ func runMcpSignIn(ctx context.Context, errOut io.Writer, flagValue, service stri // A mistyped --profile gets the existing profiles, as add lists them. return withExistingProfiles(ctx, state, profileName, err) } + setUploadConfig(ctx, profileName, configFile) // A server pinned to a config file names that file in the hints, as add does. pinnedFile := mcp.PinnedFile(entry, configFile) w, err := profileClient(ctx, profileName, configFile, pinnedFile) @@ -163,12 +171,24 @@ func runMcpSignIn(ctx context.Context, errOut io.Writer, flagValue, service stri return action(ctx, errOut, w, out, pinnedFile) } +// signInEvent is the SIGN_IN or SIGN_OUT event of one login or logout. +func signInEvent(operation protos.AitoolsMcpOperation, out *mcpSignInOutput, err error) protos.AitoolsMcpEvent { + if operation == protos.AitoolsMcpOperationSignIn { + return signInEventOf(out.Service, out.State, out.alreadySignedIn, err) + } + return protos.AitoolsMcpEvent{ + CallingAgent: callingAgentType(), + Operation: operation, + Servers: []protos.AitoolsMcpServerResult{signOutServerResult(out.Service, out.State, err)}, + } +} + // serviceFor returns the service with its sign-in requirement, as the profile sees it. A service the profile cannot see // is an error, so that a misspelled name is reported rather than treated as a service with no sign-in. func serviceFor(ctx context.Context, w *databricks.WorkspaceClient, profileName, pinnedFile, service string) (mcp.Service, error) { svc, err := mcp.GetService(ctx, w, service) if mcp.IsNotFound(err) { - return mcp.Service{}, failureWithCause(categoryServiceNotFound, err, "%s is not an MCP service visible to profile %s", service, profileName) + return mcp.Service{}, failureWithCause(protos.AitoolsErrorCategoryMCPServiceNotFound, err, "%s is not an MCP service visible to profile %s", service, profileName) } if err != nil { return mcp.Service{}, describeRequestError(profileName, pinnedFile, err) @@ -199,8 +219,9 @@ func mcpLogin(ctx context.Context, errOut io.Writer, w *databricks.WorkspaceClie if err != nil { return err } - state, _, urlShown, err := signIn(ctx, errOut, w, pinnedFile, url, svc, signInOptions{Profile: out.Profile, Open: browser.OpenOutputToStderr}) + state, alreadySignedIn, urlShown, err := signIn(ctx, errOut, w, pinnedFile, url, svc, signInOptions{Profile: out.Profile, Open: browser.OpenOutputToStderr}) out.State = state + out.alreadySignedIn = alreadySignedIn // The URL is named when it was printed. A sign-in that failed before its page is printed names none. if urlShown { out.URL = url @@ -287,7 +308,7 @@ func signIn(ctx context.Context, errOut io.Writer, w *databricks.WorkspaceClient timeout, timedOut := errors.AsType[*mcp.SignInTimeoutError](err) switch { case timedOut: - return signInStateTimeout, false, urlShown, failureWithCause(categorySignInTimeout, err, + return signInStateTimeout, false, urlShown, failureWithCause(protos.AitoolsErrorCategorySignInTimeout, err, "sign-in to %s did not complete in %g minutes%s. Finish at %s then retry: %s", svc.FullName, signInWaitTimeout.Minutes(), lastCheckText(timeout), url, mcpLoginHint(svc.FullName, profileName, pinnedFile)) case errors.Is(err, mcp.ErrSignInFailed): diff --git a/cmd/aitools/mcp_login_test.go b/cmd/aitools/mcp_login_test.go index 8b11384fb37..4e3bc5d3868 100644 --- a/cmd/aitools/mcp_login_test.go +++ b/cmd/aitools/mcp_login_test.go @@ -20,7 +20,9 @@ import ( "github.com/databricks/cli/libs/aitools/agents" "github.com/databricks/cli/libs/aitools/installer" "github.com/databricks/cli/libs/aitools/mcp" + "github.com/databricks/cli/libs/cmdctx" "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/telemetry" "github.com/databricks/cli/libs/telemetry/protos" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/apierr" @@ -124,6 +126,32 @@ func TestMcpLoginRejectedTokenIsAuthFailed(t *testing.T) { assert.Contains(t, err.Error(), "Run: databricks auth login --profile logfood") } +// TestMcpLogoutForbiddenKeepsItsText checks that a 403 from logout is reported as the API said it, with no next steps +// added: root enriches a command's auth errors only when the command sets its profile's config for the context, and +// logout's upload config is not that config. The run still uploads with the profile it bound. +func TestMcpLogoutForbiddenKeepsItsText(t *testing.T) { + home := withHome(t) + server := loginTestServer(t, http.StatusForbidden, `{"error_code":"PERMISSION_DENIED","message":"no access"}`) + writeHomeConfig(t, home, "[logfood]\nhost = "+server.URL+"\nauth_type = pat\ntoken = dapi-logfood\n") + + parent := newMCPTestRoot() + parent.SetOut(&bytes.Buffer{}) + parent.SetErr(&bytes.Buffer{}) + parent.SetArgs([]string{"mcp", "logout", "system.ai.github", "--profile", "logfood"}) + ctx, _ := cmdio.NewTestContextWithStderr(telemetry.WithNewLogger(t.Context())) + err := parent.ExecuteContext(ctx) + logout, _, findErr := parent.Find([]string{"mcp", "logout"}) + require.NoError(t, findErr) + + require.Error(t, err) + assert.Contains(t, err.Error(), "no access") + assert.NotContains(t, err.Error(), "Next steps") + assert.False(t, cmdctx.HasConfigUsed(logout.Context()), "root does not enrich the error") + cfg, uploads := telemetry.UploadConfig(logout.Context()) + require.True(t, uploads) + assert.Equal(t, "logfood", cfg.Profile) +} + // TestMcpLogoutUnknownServiceIsAnError checks that logout of a service the profile cannot see fails, rather than // reporting that the caller is not signed in, so that a typo in the name is seen. func TestMcpLogoutUnknownServiceIsAnError(t *testing.T) { @@ -134,7 +162,7 @@ func TestMcpLogoutUnknownServiceIsAnError(t *testing.T) { out := mcpSignInOutput{Service: "system.ai.gihtub"} err := runMcpSignIn(t.Context(), io.Discard, "logfood", "system.ai.gihtub", mcpLogout, &out) require.Error(t, err) - assert.Equal(t, categoryServiceNotFound, errorCategory(err)) + assert.Equal(t, protos.AitoolsErrorCategoryMCPServiceNotFound, errorCategory(err)) assert.Contains(t, err.Error(), "system.ai.gihtub is not an MCP service visible to profile logfood") assert.Empty(t, out.State) } @@ -182,7 +210,7 @@ func TestMcpLogoutEndpointNotFoundIsNotAServiceError(t *testing.T) { out := mcpSignInOutput{Service: "system.ai.github"} err := runMcpSignIn(t.Context(), io.Discard, "logfood", "system.ai.github", mcpLogout, &out) require.Error(t, err) - assert.NotEqual(t, categoryServiceNotFound, errorCategory(err)) + assert.NotEqual(t, protos.AitoolsErrorCategoryMCPServiceNotFound, errorCategory(err)) assert.NotContains(t, err.Error(), "is not an MCP service visible") } diff --git a/cmd/aitools/mcp_remove.go b/cmd/aitools/mcp_remove.go index 3804cd59299..ca19e363e36 100644 --- a/cmd/aitools/mcp_remove.go +++ b/cmd/aitools/mcp_remove.go @@ -15,6 +15,8 @@ import ( "github.com/databricks/cli/libs/aitools/mcp/agentconfig" "github.com/databricks/cli/libs/aitools/mcpconfig" "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/telemetry" "github.com/databricks/cli/libs/telemetry/protos" "github.com/spf13/cobra" ) @@ -34,6 +36,14 @@ type removeOutput struct { Skipped []removeSkipped `json:"skipped"` Error string `json:"error,omitempty"` ErrorCategory protos.AitoolsErrorCategory `json:"error_category,omitempty"` + + // results is one telemetry result per (service, agent) attempted. It is not in the JSON. + results []protos.AitoolsMcpServerResult + // stateBefore is the state as it was read before the removal. It is not in the JSON. + stateBefore *mcp.State + // profile is the profile and config file that the selected servers are bound to, when they all are bound to one. It is + // the zero key otherwise. It is not in the JSON. + profile profileKey } type removeRemoved struct { @@ -74,7 +84,7 @@ type recordDrop struct { agent string } -// NewMcpRemoveCmd returns `aitools mcp remove`. It needs no network. +// NewMcpRemoveCmd returns `aitools mcp remove`. The edit needs no network; the telemetry upload after it is best-effort. func NewMcpRemoveCmd() *cobra.Command { var f mcpRemoveFlags cmd := &cobra.Command{ @@ -107,22 +117,12 @@ removes one.`, out := &removeOutput{Removed: []removeRemoved{}, Skipped: []removeSkipped{}} runErr := runMcpRemove(ctx, cmd, f, jsonMode, out) - if !jsonMode { - return runErr - } - - // The document carries a top-level failure; per-agent failures are already in skipped. - if top := topLevelFailure(runErr); top != nil { - out.Error = top.Error() - out.ErrorCategory = errorCategory(top) + err := runErr + if jsonMode { + err = writeRemoveJSON(cmd, out, runErr) } - if err := renderJSON(cmd.OutOrStdout(), out); err != nil { - return errors.Join(runErr, err) - } - if runErr != nil { - return root.ErrAlreadyPrinted - } - return nil + logRemoveEvent(ctx, out) + return err }, } @@ -132,6 +132,47 @@ removes one.`, return cmd } +// writeRemoveJSON writes the JSON document of a run that ended with runErr and returns the command's error. +func writeRemoveJSON(cmd *cobra.Command, out *removeOutput, runErr error) error { + // The document carries a top-level failure; per-agent failures are already in skipped. + if top := topLevelFailure(runErr); top != nil { + out.Error = top.Error() + out.ErrorCategory = errorCategory(top) + } + if err := renderJSON(cmd.OutOrStdout(), out); err != nil { + return errors.Join(runErr, err) + } + if runErr != nil { + return root.ErrAlreadyPrinted + } + return nil +} + +// logRemoveEvent buffers the REMOVE event and sets the config its exit-time upload authenticates with. It runs after the +// output is written. A run that logged no event touched no server, so it has nothing to upload and sets no config. +func logRemoveEvent(ctx context.Context, out *removeOutput) { + if !logMcpEvent(ctx, protos.AitoolsMcpEvent{ + CallingAgent: callingAgentType(), + Operation: protos.AitoolsMcpOperationRemove, + Servers: out.results, + }) { + return + } + if out.profile.profile == "" { + // The servers are bound to several profiles, so no one profile's token may stand in for the upload. + telemetry.SkipUpload(ctx) + return + } + // The upload sends the profile's token to the profile's host, so a profile whose recorded servers no longer match it + // uploads nothing. The records are the ones read before the removal. + if bindingDrifted(ctx, out.stateBefore, out.profile) { + log.Debugf(ctx, "telemetry: profile %s no longer matches an MCP server recorded for it; not uploading telemetry", out.profile.profile) + telemetry.SkipUpload(ctx) + return + } + setUploadConfig(ctx, out.profile.profile, out.profile.path) +} + // runMcpRemove removes the chosen servers from the agent configs and records the result. Results are printed // before a state failure is returned. An agent failure fails the command when --agents named the agent, or when a // selected server was removed from no agent; otherwise it is reported in skipped. @@ -165,6 +206,7 @@ func runMcpRemove(ctx context.Context, cmd *cobra.Command, f mcpRemoveFlags, jso if err != nil { return err } + out.stateBefore = state var selected []mcp.Entry if interactive && !f.all && len(names) == 0 && profileName == "" { // An agent with no installed server is refused as it is without a terminal, before the picker offers nothing for it. @@ -210,6 +252,7 @@ func runMcpRemove(ctx context.Context, cmd *cobra.Command, f mcpRemoveFlags, jso // --agents chooses the servers installed for those agents, so a server with no agent left is not chosen. selected = slices.DeleteFunc(selected, func(e mcp.Entry) bool { return len(e.Agents) == 0 }) } + out.profile = sharedProfile(selected, mcp.CheckEntries(ctx, &mcp.State{Servers: selected})) // Each agent config file is read once per command, and each server's config file is resolved once. files := newAgentFiles() @@ -221,12 +264,14 @@ func runMcpRemove(ctx context.Context, cmd *cobra.Command, f mcpRemoveFlags, jso if !jsonMode { printRemoveResults(ctx, out) } - return failure(categoryNameConflict, "nothing was removed: named MCP servers are left in place: %s", refusedNames(refused)) + return failure(protos.AitoolsErrorCategoryMCPNameConflict, "nothing was removed: named MCP servers are left in place: %s", refusedNames(refused)) } } var drops []recordDrop var agentErrs []error + // recordOnly are the indexes of out.results whose removal only drops a state record. + var recordOnly []int for _, e := range selected { removed := removeRemoved{Name: e.Name, Service: e.Service, Profile: e.Profile, Agents: []string{}} pin := resolveBinding(ctx, e) @@ -246,10 +291,17 @@ func runMcpRemove(ctx context.Context, cmd *cobra.Command, f mcpRemoveFlags, jso // There is no config this CLI can edit for the agent, so only the record can go. removed.Agents = append(removed.Agents, agentName) removed.RecordOnly = append(removed.RecordOnly, agentName) + recordOnly = append(recordOnly, len(out.results)) + out.results = append(out.results, removeServerResult(e.Service, agentName, removeDeleted, nil)) drops = append(drops, recordDrop{name: e.Name, agent: agentName}) continue } plan, err := removeFromAgent(ctx, files, e, a, pin, len(agentNames) > 0) + // A pruned entry changes the state only, so a save that fails undoes it, as for a record-only removal. + if plan.outcome == removePruned && err == nil { + recordOnly = append(recordOnly, len(out.results)) + } + out.results = append(out.results, removeServerResult(e.Service, agentName, plan.outcome, err)) if err != nil { errs = append(errs, err) out.Skipped = append(out.Skipped, removeSkipped{Name: e.Name, Agent: agentName, Reason: fmt.Sprintf("could not update %s: %s", agentDisplayName(agentName), err)}) @@ -265,7 +317,12 @@ func runMcpRemove(ctx context.Context, cmd *cobra.Command, f mcpRemoveFlags, jso out.Skipped = append(out.Skipped, removeSkipped{Name: e.Name, Agent: agentName, Reason: reason}) } } - // A server with no agent left in state is removed from state too, so it is reported as removed. + // A server with no agent left in state is removed from state too, so it is reported as removed. Only its + // record goes, which is a change to the state. + if len(e.Agents) == 0 { + recordOnly = append(recordOnly, len(out.results)) + out.results = append(out.results, protos.AitoolsMcpServerResult{Service: mcpServiceType(e.Service), Outcome: protos.AitoolsTargetOutcomeChanged}) + } if len(removed.Agents) > 0 || len(e.Agents) == 0 { out.Removed = append(out.Removed, removed) } @@ -281,6 +338,13 @@ func runMcpRemove(ctx context.Context, cmd *cobra.Command, f mcpRemoveFlags, jso if len(drops) > 0 || slices.ContainsFunc(selected, func(e mcp.Entry) bool { return len(e.Agents) == 0 }) { saveErr = dropRecords(ctx, drops, selected) } + if saveErr != nil { + // A record-only removal changes nothing but the state, so the save that failed undid it. + for _, i := range recordOnly { + out.results[i].Outcome = protos.AitoolsTargetOutcomeFailed + out.results[i].ErrorCategory = errorCategory(saveErr) + } + } if !jsonMode { printRemoveResults(ctx, out) } @@ -327,7 +391,7 @@ func selectInstalls(state *mcp.State, names []string, profileName string) ([]mcp return nil, err } if len(matched) == 0 { - return nil, &noRecordError{name: name, err: failure(categoryServiceNotFound, "no installed MCP server named %s%s", name, where)} + return nil, &noRecordError{name: name, err: failure(protos.AitoolsErrorCategoryMCPServiceNotFound, "no installed MCP server named %s%s", name, where)} } for _, i := range matched { chosen[i] = true @@ -375,7 +439,7 @@ func withAdoptHint(ctx context.Context, err error, state *mcp.State) error { if u.fix != "" { hint += ". To adopt it, so that remove works, run `" + u.fix + "`" } - return withCategory(fmt.Errorf("%w; %s", err, hint), categoryServiceNotFound) + return withCategory(fmt.Errorf("%w; %s", err, hint), protos.AitoolsErrorCategoryMCPServiceNotFound) } return err } @@ -454,7 +518,7 @@ func checkNamedInstalled(selected []mcp.Entry, targets []*agents.Agent) error { for _, a := range targets { agentNames = append(agentNames, a.Name) } - return failure(categoryServiceNotFound, "nothing was removed: named MCP servers are not installed for %s: %s", + return failure(protos.AitoolsErrorCategoryMCPServiceNotFound, "nothing was removed: named MCP servers are not installed for %s: %s", mcpAgentList(agentNames), strings.Join(servers, ", ")) } @@ -469,7 +533,7 @@ func checkTargetsInstalled(selected []mcp.Entry, targets []*agents.Agent) error _, ok := e.Agents[a.Name] return ok }) { - return failure(categoryServiceNotFound, "no selected MCP server is installed for %s", agentDisplayName(a.Name)) + return failure(protos.AitoolsErrorCategoryMCPServiceNotFound, "no selected MCP server is installed for %s", agentDisplayName(a.Name)) } } return nil diff --git a/cmd/aitools/mcp_remove_test.go b/cmd/aitools/mcp_remove_test.go index b0b0d0c0099..afa0f2a39bb 100644 --- a/cmd/aitools/mcp_remove_test.go +++ b/cmd/aitools/mcp_remove_test.go @@ -1,9 +1,11 @@ package aitools import ( + "bytes" "context" "encoding/json" "errors" + "log/slog" "maps" "os" "path/filepath" @@ -12,8 +14,15 @@ import ( "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/aitools/agents" + "github.com/databricks/cli/libs/aitools/installer" "github.com/databricks/cli/libs/aitools/mcp" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/env" + "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/log/handler" + "github.com/databricks/cli/libs/telemetry" "github.com/databricks/cli/libs/telemetry/protos" + "github.com/spf13/cobra" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -82,7 +91,7 @@ func TestSelectInstalls(t *testing.T) { _, err = selectInstalls(state, []string{"github-dogfood"}, "logfood") require.Error(t, err) assert.Equal(t, "no installed MCP server named github-dogfood on profile logfood", err.Error()) - assert.Equal(t, categoryServiceNotFound, errorCategory(err)) + assert.Equal(t, protos.AitoolsErrorCategoryMCPServiceNotFound, errorCategory(err)) // A profile with nothing installed has nothing to remove, the same as --all on an empty state. none, err := selectInstalls(state, nil, "typo") @@ -179,6 +188,23 @@ func TestMcpRemoveDeletesOwnedEntriesAndPrunesMissing(t *testing.T) { assert.Equal(t, "github-dogfood", state.Servers[0].Name) } +// TestMcpRemoveMakesNoRequestWhenTelemetryIsOff checks that remove edits its agent config without a request to the +// profile's host. Nothing is uploaded when telemetry is off, so no auth is resolved for an upload. +func TestMcpRemoveMakesNoRequestWhenTelemetryIsOff(t *testing.T) { + home := withHome(t) + t.Setenv("DATABRICKS_CLI_DISABLE_TELEMETRY", "1") + server := noRequestServer(t) + writeHomeConfig(t, home, "[logfood]\nhost = "+server.URL+"\nauth_type = oauth-m2m\nclient_id = id\nclient_secret = secret\n") + claude := agents.ByName("claude-code") + claudePath := filepath.Join(home, ".claude.json") + genie := seedInstall(t, claude, "genie_one_mcp-logfood", "system.ai.genie_one_mcp", "logfood", server.URL) + saveState(t, genie) + + _, _, err := runMcpCmd(t, "remove", "--names", "genie_one_mcp-logfood", "--output", "json") + require.NoError(t, err) + assert.NotContains(t, jsonServers(t, claudePath), "genie_one_mcp-logfood") +} + // wantOtherEntryReason is the skipped reason for an entry named name, in path, that this CLI did not write and that is not a // Databricks entry: it is left in place, and the record for the agent is dropped. func wantOtherEntryReason(name, path string) string { @@ -613,7 +639,7 @@ func TestMcpRemoveNamesUnmanagedEntrySaysHowToAdopt(t *testing.T) { require.ErrorIs(t, err, root.ErrAlreadyPrinted) out := readRemoveOutput(t, stdout.String()) assert.Equal(t, "no installed MCP server named genie_one_mcp-logfood; Claude Code has an entry named genie_one_mcp-logfood that runs aitools mcp serve with no install record. To adopt it, so that remove works, run `databricks aitools mcp add --names system.ai.genie_one_mcp --profile logfood --agents claude-code`", out.Error) - assert.Equal(t, categoryServiceNotFound, out.ErrorCategory) + assert.Equal(t, protos.AitoolsErrorCategoryMCPServiceNotFound, out.ErrorCategory) } func TestMcpRemoveRefusesAgentWithNothingInstalled(t *testing.T) { @@ -627,7 +653,7 @@ func TestMcpRemoveRefusesAgentWithNothingInstalled(t *testing.T) { require.ErrorIs(t, err, root.ErrAlreadyPrinted) out := readRemoveOutput(t, stdout.String()) assert.Equal(t, "no selected MCP server is installed for Cursor", out.Error) - assert.Equal(t, categoryServiceNotFound, out.ErrorCategory) + assert.Equal(t, protos.AitoolsErrorCategoryMCPServiceNotFound, out.ErrorCategory) // Nothing was removed by the refusal. state, err := mcp.LoadState(t.Context()) @@ -876,7 +902,7 @@ func TestMcpRemoveRefusesNamedServerThatAgentsNarrowsToNothing(t *testing.T) { out := readRemoveOutput(t, stdout.String()) assert.Empty(t, out.Removed) assert.Equal(t, "nothing was removed: named MCP servers are not installed for Cursor: github-logfood", out.Error) - assert.Equal(t, categoryServiceNotFound, out.ErrorCategory) + assert.Equal(t, protos.AitoolsErrorCategoryMCPServiceNotFound, out.ErrorCategory) claudeAfter, err := os.ReadFile(claudePath) require.NoError(t, err) @@ -1083,7 +1109,7 @@ func TestMcpRemovePickerRefusesAgentWithNoServer(t *testing.T) { err := runInteractive(t, "mcp", "remove", "--agents", "codex") require.Error(t, err) assert.Equal(t, "no selected MCP server is installed for "+agents.ByName(agents.NameCodex).DisplayName, err.Error()) - assert.Equal(t, categoryServiceNotFound, errorCategory(err)) + assert.Equal(t, protos.AitoolsErrorCategoryMCPServiceNotFound, errorCategory(err)) state, err := mcp.LoadState(t.Context()) require.NoError(t, err) assert.Len(t, state.Servers, 1) @@ -1162,3 +1188,264 @@ func claudeServerEntry(claudePath, name, service string) mcp.Entry { Agents: map[string]mcp.AgentRecord{agents.NameClaudeCode: {Path: claudePath, Scope: "user"}}, } } + +// runRemoveWithTelemetry runs `mcp remove` with args on a context that buffers telemetry, and returns the remove command, +// so a test can read the context it left, and its stdout. +// The logs are those of the command's logger at logLevel. +func runRemoveWithTelemetry(t *testing.T, stdout, logs *bytes.Buffer, logLevel slog.Level, args ...string) (*cobra.Command, error) { + t.Helper() + parent := newMCPTestRoot() + ctx := telemetry.WithNewLogger(t.Context()) + ctx = env.Set(ctx, "DATABRICKS_CLI_DISABLE_TELEMETRY", "") + ctx = log.NewContext(ctx, slog.New(handler.NewFriendlyHandler(logs, &handler.Options{Level: logLevel}))) + ctx, _ = cmdio.NewTestContextWithStderr(ctx) + parent.SetOut(stdout) + parent.SetErr(&bytes.Buffer{}) + parent.SetArgs(append([]string{"mcp", "remove"}, args...)) + err := parent.ExecuteContext(ctx) + remove, _, findErr := parent.Find([]string{"mcp", "remove"}) + require.NoError(t, findErr) + return remove, err +} + +// unsetConfigFile makes DATABRICKS_CONFIG_FILE unset for the test, as opposed to set to "". +func unsetConfigFile(t *testing.T) { + t.Setenv("DATABRICKS_CONFIG_FILE", "") + require.NoError(t, os.Unsetenv("DATABRICKS_CONFIG_FILE")) +} + +// TestMcpRemoveSetsNoUploadWithoutAnEvent checks that a run that removes nothing logs no event, so it sets no config for +// an upload. +func TestMcpRemoveSetsNoUploadWithoutAnEvent(t *testing.T) { + withHome(t) + var stdout, logs bytes.Buffer + remove, err := runRemoveWithTelemetry(t, &stdout, &logs, log.LevelDebug, "--all", "--output", "json") + require.NoError(t, err) + _, uploads := telemetry.UploadConfig(remove.Context()) + assert.False(t, uploads) + assert.Empty(t, telemetry.BufferedEvents(remove.Context())) +} + +// TestMcpRemoveUploadSkipsDriftedProfile checks that the upload is not authenticated with a profile that a server recorded +// for it no longer matches, as its host here is not the one the server was set up against. The removal is done either +// way. +func TestMcpRemoveUploadSkipsDriftedProfile(t *testing.T) { + home := withHome(t) + writeHomeConfig(t, home, "[logfood]\nhost = https://adb-2.test\nauth_type = pat\ntoken = t\n") + claude := agents.ByName("claude-code") + claudePath := filepath.Join(home, ".claude.json") + // Recorded against another host than the profile names now. + entry := seedInstall(t, claude, "genie_one_mcp-logfood", "system.ai.genie_one_mcp", "logfood", "https://adb-1.test") + saveState(t, entry) + + var stdout, logs bytes.Buffer + remove, err := runRemoveWithTelemetry(t, &stdout, &logs, log.LevelDebug, "--names", "genie_one_mcp-logfood", "--output", "json") + require.NoError(t, err) + require.Len(t, telemetry.BufferedEvents(remove.Context()), 1, "the removal is logged") + _, uploads := telemetry.UploadConfig(remove.Context()) + assert.False(t, uploads, "a drifted profile is not used to upload") + assert.NotContains(t, jsonServers(t, claudePath), "genie_one_mcp-logfood", "the removal is done") +} + +// TestMcpRemoveUploadUsesMatchingProfile checks that a profile whose recorded servers still match is used for the upload, +// read from the default config file when DATABRICKS_CONFIG_FILE is unset. +func TestMcpRemoveUploadUsesMatchingProfile(t *testing.T) { + home := withHome(t) + unsetConfigFile(t) + writeHomeConfig(t, home, "[logfood]\nhost = https://adb-1.test\nauth_type = pat\ntoken = t\n") + claude := agents.ByName("claude-code") + entry := seedInstall(t, claude, "genie_one_mcp-logfood", "system.ai.genie_one_mcp", "logfood", "https://adb-1.test") + saveState(t, entry) + + var stdout, logs bytes.Buffer + remove, err := runRemoveWithTelemetry(t, &stdout, &logs, log.LevelDebug, "--names", "genie_one_mcp-logfood", "--output", "json") + require.NoError(t, err) + cfg, uploads := telemetry.UploadConfig(remove.Context()) + require.True(t, uploads) + assert.Equal(t, "logfood", cfg.Profile) + assert.Equal(t, "https://adb-1.test", cfg.Host) + assert.Equal(t, "t", cfg.Token) +} + +// TestMcpRemoveUploadIgnoresEnvironmentToken checks that the upload authenticates with the profile's own token, never with +// a DATABRICKS_TOKEN of another workspace in the environment. +func TestMcpRemoveUploadIgnoresEnvironmentToken(t *testing.T) { + home := withHome(t) + unsetConfigFile(t) + t.Setenv("DATABRICKS_TOKEN", "token-of-another-workspace") + writeHomeConfig(t, home, "[logfood]\nhost = https://adb-1.test\nauth_type = pat\ntoken = t\n") + claude := agents.ByName("claude-code") + saveState(t, seedInstall(t, claude, "genie_one_mcp-logfood", "system.ai.genie_one_mcp", "logfood", "https://adb-1.test")) + + var stdout, logs bytes.Buffer + remove, err := runRemoveWithTelemetry(t, &stdout, &logs, log.LevelDebug, "--names", "genie_one_mcp-logfood", "--output", "json") + require.NoError(t, err) + cfg, uploads := telemetry.UploadConfig(remove.Context()) + require.True(t, uploads) + assert.Equal(t, "t", cfg.Token) +} + +// TestMcpRemoveUploadUsesConfigFileOfServer checks that a server pinned to a config file uploads with the profile of that +// file, not with a profile of the same name in the file DATABRICKS_CONFIG_FILE names. +func TestMcpRemoveUploadUsesConfigFileOfServer(t *testing.T) { + home := withHome(t) + pinned := filepath.Join(home, "pinned.databrickscfg") + require.NoError(t, os.WriteFile(pinned, []byte("[logfood]\nhost = https://adb-1.test\nauth_type = pat\ntoken = pinned-token\n"), 0o600)) + other := filepath.Join(home, "other.databrickscfg") + require.NoError(t, os.WriteFile(other, []byte("[dogfood]\nhost = https://adb-2.test\nauth_type = pat\ntoken = other-token\n"), 0o600)) + claude := agents.ByName("claude-code") + t.Setenv("DATABRICKS_CONFIG_FILE", pinned) + saveState(t, seedInstall(t, claude, "genie_one_mcp-logfood", "system.ai.genie_one_mcp", "logfood", "https://adb-1.test")) + t.Setenv("DATABRICKS_CONFIG_FILE", other) + + var stdout, logs bytes.Buffer + remove, err := runRemoveWithTelemetry(t, &stdout, &logs, log.LevelDebug, "--names", "genie_one_mcp-logfood", "--output", "json") + require.NoError(t, err) + cfg, uploads := telemetry.UploadConfig(remove.Context()) + require.True(t, uploads) + assert.Equal(t, "pinned-token", cfg.Token) +} + +// TestMcpRemoveUploadIgnoresSameNamedProfileInOtherFile checks that a profile of the same name in another config file does +// not stop the upload for the server that matches its own file. +func TestMcpRemoveUploadIgnoresSameNamedProfileInOtherFile(t *testing.T) { + home := withHome(t) + first := filepath.Join(home, "first.databrickscfg") + require.NoError(t, os.WriteFile(first, []byte("[logfood]\nhost = https://adb-1.test\nauth_type = pat\ntoken = first-token\n"), 0o600)) + second := filepath.Join(home, "second.databrickscfg") + require.NoError(t, os.WriteFile(second, []byte("[logfood]\nhost = https://adb-2.test\nauth_type = pat\ntoken = second-token\n"), 0o600)) + claude := agents.ByName("claude-code") + t.Setenv("DATABRICKS_CONFIG_FILE", first) + firstEntry := seedInstall(t, claude, "genie_one_mcp-logfood", "system.ai.genie_one_mcp", "logfood", "https://adb-1.test") + t.Setenv("DATABRICKS_CONFIG_FILE", second) + secondEntry := seedInstall(t, claude, "dbsql-logfood", "system.ai.dbsql", "logfood", "https://adb-2.test") + saveState(t, firstEntry, secondEntry) + t.Setenv("DATABRICKS_CONFIG_FILE", first) + + var stdout, logs bytes.Buffer + remove, err := runRemoveWithTelemetry(t, &stdout, &logs, log.LevelDebug, "--names", "genie_one_mcp-logfood", "--output", "json") + require.NoError(t, err) + cfg, uploads := telemetry.UploadConfig(remove.Context()) + require.True(t, uploads) + assert.Equal(t, "first-token", cfg.Token) +} + +// TestMcpRemoveUploadSkipsServersOfSeveralProfiles checks that a remove of servers bound to two profiles logs its event but +// uploads nothing, since no one profile's token may stand in for the upload. +func TestMcpRemoveUploadSkipsServersOfSeveralProfiles(t *testing.T) { + home := withHome(t) + writeHomeConfig(t, home, "[DEFAULT]\nhost = https://adb-3.test\nauth_type = pat\ntoken = t\n[logfood]\nhost = https://adb-1.test\nauth_type = pat\ntoken = t\n[other]\nhost = https://adb-2.test\nauth_type = pat\ntoken = t\n") + claude := agents.ByName("claude-code") + saveState(t, + seedInstall(t, claude, "genie_one_mcp-logfood", "system.ai.genie_one_mcp", "logfood", "https://adb-1.test"), + seedInstall(t, claude, "dbsql-other", "system.ai.dbsql", "other", "https://adb-2.test"), + ) + + var stdout, logs bytes.Buffer + remove, err := runRemoveWithTelemetry(t, &stdout, &logs, log.LevelDebug, "--all", "--output", "json") + require.NoError(t, err) + require.Len(t, telemetry.BufferedEvents(remove.Context()), 1, "the removal is logged") + _, uploads := telemetry.UploadConfig(remove.Context()) + assert.False(t, uploads, "servers of several profiles have no one profile to upload with") +} + +// TestMcpRemoveOfflineSetsUploadWithoutWarning checks that a remove that cannot reach the workspace warns nothing: the +// upload's config is resolved, and its host metadata lookup fails quietly, so there is no warning to report. +func TestMcpRemoveOfflineSetsUploadWithoutWarning(t *testing.T) { + home := withHome(t) + writeHomeConfig(t, home, "[logfood]\nhost = https://adb-1.test\nauth_type = pat\ntoken = t\n") + claude := agents.ByName("claude-code") + saveState(t, seedInstall(t, claude, "genie_one_mcp-logfood", "system.ai.genie_one_mcp", "logfood", "https://adb-1.test")) + + var stdout, logs bytes.Buffer + remove, err := runRemoveWithTelemetry(t, &stdout, &logs, log.LevelWarn, "--names", "genie_one_mcp-logfood", "--output", "json") + require.NoError(t, err) + _, uploads := telemetry.UploadConfig(remove.Context()) + assert.True(t, uploads) + assert.Empty(t, logs.String()) + assert.Contains(t, stdout.String(), `"removed"`) +} + +// seedPrunedInstall records an install of genie_one_mcp for logfood in claude-code whose entry the agent config no longer holds. +func seedPrunedInstall(t *testing.T, claudePath string) { + t.Helper() + claude := agents.ByName("claude-code") + saveState(t, seedInstall(t, claude, "genie_one_mcp-logfood", "system.ai.genie_one_mcp", "logfood", "https://adb-1.test")) + require.NoError(t, os.WriteFile(claudePath, []byte(`{"mcpServers":{}}`), 0o600)) +} + +// TestMcpRemovePrunedEntryReportsChanged checks that an entry the agent config no longer holds is reported CHANGED, since +// its record is dropped, and FAILED when the state cannot be saved, as a record-only removal is. +func TestMcpRemovePrunedEntryReportsChanged(t *testing.T) { + home := withHome(t) + claudePath := filepath.Join(home, ".claude.json") + + seedPrunedInstall(t, claudePath) + var stdout, logs bytes.Buffer + remove, err := runRemoveWithTelemetry(t, &stdout, &logs, log.LevelDebug, "--names", "genie_one_mcp-logfood", "--output", "json") + require.NoError(t, err) + events := telemetry.BufferedEvents(remove.Context()) + require.Len(t, events, 1) + require.NotNil(t, events[0].AitoolsMcpEvent) + require.Len(t, events[0].AitoolsMcpEvent.Servers, 1) + assert.Equal(t, protos.AitoolsTargetOutcomeChanged, events[0].AitoolsMcpEvent.Servers[0].Outcome) + + seedPrunedInstall(t, claudePath) + saveMCPState = func(context.Context, *mcp.State) error { return errors.New("state file is read-only") } + t.Cleanup(func() { saveMCPState = mcp.SaveState }) + stdout.Reset() + remove, err = runRemoveWithTelemetry(t, &stdout, &logs, log.LevelDebug, "--names", "genie_one_mcp-logfood", "--output", "json") + require.ErrorIs(t, err, root.ErrAlreadyPrinted) + events = telemetry.BufferedEvents(remove.Context()) + require.Len(t, events, 1) + require.NotNil(t, events[0].AitoolsMcpEvent) + require.Len(t, events[0].AitoolsMcpEvent.Servers, 1) + assert.Equal(t, protos.AitoolsTargetOutcomeFailed, events[0].AitoolsMcpEvent.Servers[0].Outcome) +} + +// TestMcpRemoveFailedStateReadAfterAnotherEventLogsNothing checks that a remove that cannot read the state logs nothing +// of its own, even when the run already buffered another event, and does not read the state it never read. +func TestMcpRemoveFailedStateReadAfterAnotherEventLogsNothing(t *testing.T) { + withHome(t) + dir, err := installer.GlobalSkillsDir(t.Context()) + require.NoError(t, err) + require.NoError(t, os.MkdirAll(filepath.Join(dir, ".state.json"), 0o700)) + + parent := newMCPTestRoot() + parent.SetOut(&bytes.Buffer{}) + parent.SetErr(&bytes.Buffer{}) + parent.SetArgs([]string{"mcp", "remove", "--all", "--output", "json"}) + ctx := telemetry.WithNewLogger(t.Context()) + telemetry.Log(ctx, protos.DatabricksCliLog{CliTestEvent: &protos.CliTestEvent{Name: protos.DummyCliEnumValue1}}) + ctx, _ = cmdio.NewTestContextWithStderr(ctx) + require.Error(t, parent.ExecuteContext(ctx)) + remove, _, findErr := parent.Find([]string{"mcp", "remove"}) + require.NoError(t, findErr) + events := telemetry.BufferedEvents(remove.Context()) + require.Len(t, events, 1) + assert.NotNil(t, events[0].CliTestEvent) +} + +// TestMcpRemoveRecordOnlyReportsFailedWhenStateIsNotSaved checks that a removal that only drops state records is reported +// FAILED when the state cannot be saved, since nothing was removed. +func TestMcpRemoveRecordOnlyReportsFailedWhenStateIsNotSaved(t *testing.T) { + withHome(t) + recordOnly := mcp.Entry{ + Name: "dbsql-logfood", Service: "system.ai.dbsql", Profile: "logfood", Host: "https://adb-1.test", Command: testDatabricksCommand, + Agents: map[string]mcp.AgentRecord{"pi": {Path: "/x/config.json", Scope: "user"}}, + } + saveState(t, recordOnly) + saveMCPState = func(context.Context, *mcp.State) error { return errors.New("state file is read-only") } + t.Cleanup(func() { saveMCPState = mcp.SaveState }) + + var stdout, logs bytes.Buffer + remove, err := runRemoveWithTelemetry(t, &stdout, &logs, log.LevelDebug, "--names", "dbsql-logfood", "--output", "json") + require.ErrorIs(t, err, root.ErrAlreadyPrinted) + + events := telemetry.BufferedEvents(remove.Context()) + require.Len(t, events, 1) + require.NotNil(t, events[0].AitoolsMcpEvent) + require.Len(t, events[0].AitoolsMcpEvent.Servers, 1) + assert.Equal(t, protos.AitoolsTargetOutcomeFailed, events[0].AitoolsMcpEvent.Servers[0].Outcome) + assert.Equal(t, protos.AitoolsErrorCategoryUncategorized, events[0].AitoolsMcpEvent.Servers[0].ErrorCategory) +} diff --git a/cmd/aitools/mcp_serve.go b/cmd/aitools/mcp_serve.go index f5ed6dbcf81..5cf13f93e4f 100644 --- a/cmd/aitools/mcp_serve.go +++ b/cmd/aitools/mcp_serve.go @@ -29,6 +29,8 @@ import ( "github.com/databricks/cli/libs/databrickscfg/profile" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/shellquote" + "github.com/databricks/cli/libs/telemetry" + "github.com/databricks/cli/libs/telemetry/protos" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/config" "github.com/databricks/databricks-sdk-go/config/credentials" @@ -195,13 +197,14 @@ func NewMcpServeCmd() *cobra.Command { cmd.Flags().StringVar(&f.service, strings.TrimPrefix(mcpconfig.ServiceFlag, "--"), "", "Full name of the MCP service, for example system.ai.github") cmd.Flags().StringVar(&f.agent, strings.TrimPrefix(mcpconfig.AgentFlag, "--"), "", - "Registry name of the calling agent, used only for the User-Agent") + "Registry name of the calling agent, for the User-Agent and telemetry") return cmd } // runMcpServe runs the binding checks, then bridges stdio to the service until stdin closes or a // signal arrives. func runMcpServe(cmd *cobra.Command, f mcpServeFlags) error { + start := time.Now() if cmd.Flag("output").Changed { return errors.New("--output does not apply to serve: its stdout carries the MCP protocol") } @@ -220,6 +223,7 @@ func runMcpServe(cmd *cobra.Command, f mcpServeFlags) error { if err != nil { return serveFailure(cmd, err, serveBindStatus) } + agent := proxyAgentType(f.agent) cfg, entry, err := mcp.Bind(cmd.Context(), mcp.BindOptions{ Profile: profileName, Service: f.service, @@ -236,7 +240,8 @@ func runMcpServe(cmd *cobra.Command, f mcpServeFlags) error { defer stop() // The bridge and the sign-in client share the guard, so a refresh failure in either one stops the proxy. // Each token fetch re-checks the binding first; see rebindingAuthenticate. - authenticate := rebindingAuthenticate(bindingCheck(cmd.Context(), state, entry, f.service, profileName, cfg), cfg.Authenticate) + check := bindingCheck(cmd.Context(), state, entry, f.service, profileName, cfg) + authenticate := rebindingAuthenticate(check, cfg.Authenticate) refresh := &refreshGuard{authenticate: authenticate, profile: profileName, pinnedFile: pinnedFile, cancel: stop} // The sign-in client is built only when serveChallenge reads the service or signs in. A server whose record says it @@ -248,7 +253,7 @@ func runMcpServe(cmd *cobra.Command, f mcpServeFlags) error { return serveFailure(cmd, err, 1) } } - challenge, err := serveChallenge(ctx, serveTarget{w: w, entry: entry, profile: profileName, service: f.service, pinnedFile: pinnedFile}, cmd.ErrOrStderr()) + challenge, err := serveChallenge(ctx, serveTarget{w: w, entry: entry, profile: profileName, service: f.service, pinnedFile: pinnedFile, agent: agent}, cmd.ErrOrStderr()) if err != nil { // A rejected token stops serve by cancelling ctx, so the refresh failure is checked before the signal, and its // message is the one that names the login. @@ -262,8 +267,15 @@ func runMcpServe(cmd *cobra.Command, f mcpServeFlags) error { // serveChallenge fails only on an auth failure (see lookupAtStart), which exits serveBindStatus. return serveFailure(cmd, err, serveBindStatus) } + // Only a serving start is logged, and it is uploaded now rather than at exit: the exit upload cannot refresh an expired + // token, so a session idle past the token's life would lose its start. The upload authenticates with the profile as it + // was bound here, with a config of its own that never refreshes a token, since the session's config refreshes its token. + setUploadConfig(cmd.Context(), profileName, cfg.ConfigFile) + if logMcpEvent(cmd.Context(), proxyStartedEvent(agent, f.service)) { + uploadAtStart(cmd, start) + } - return serveProxy(ctx, cmd, refresh, proxy.Options{ + err = serveProxy(ctx, cmd, refresh, proxy.Options{ In: cmd.InOrStdin(), Out: cmd.OutOrStdout(), Err: cmd.ErrOrStderr(), @@ -278,6 +290,30 @@ func runMcpServe(cmd *cobra.Command, f mcpServeFlags) error { }, Challenge: challenge, }) + dropUploadIfRebound(cmd.Context(), check, profileName) + return err +} + +// dropUploadIfRebound drops the exit upload when check, the binding check of each token fetch (see bindingCheck), finds +// that the profile no longer names the host that serve bound. The upload's token comes from the token cache, which is +// keyed by profile name alone, so after a re-point it would be the new host's token, sent to the bound host. remove drops +// its upload for a drifted binding the same way. +func dropUploadIfRebound(ctx context.Context, check func() error, profileName string) { + if err := checkBindingWithRetry(check); err != nil { + log.Debugf(ctx, "telemetry: profile %s no longer matches the binding serve started with; not uploading telemetry", profileName) + telemetry.SkipUpload(ctx) + } +} + +// uploadAtStart uploads the events logged so far, which at serve's start is its PROXY_STARTED. The upload waits for the +// service, bounded by its timeout, so the proxy starts after it. A failed upload keeps its events for the exit upload. +// The run is still going, so its exit code is 0. +func uploadAtStart(cmd *cobra.Command, start time.Time) { + ctx := cmd.Context() + err := telemetry.Upload(ctx, root.TelemetryExecutionContext(cmd, start, 0)) + if err != nil { + log.Debugf(ctx, "telemetry: upload at start failed: %s", err) + } } // bindingCheck returns the check that runs before each token is fetched. With an install record, entry, it checks the @@ -354,8 +390,8 @@ func (g guardCredentials) Configure(context.Context, *config.Config) (credential return credentials.CredentialsProviderFn(g.refresh.authenticateCall), nil } -// serveTarget is the server that a serve session bridges: its service, the bound profile, the install record, and the -// config file it is pinned to. w is the profile's client, which must be guardedClient's. +// serveTarget is the server that a serve session bridges: its service, the bound profile, the install record, the config +// file it is pinned to, and the agent that started it. w is the profile's client, which must be guardedClient's. type serveTarget struct { // w is nil when entry says the service needs no sign-in. serveChallenge returns before it would use it then. w *databricks.WorkspaceClient @@ -363,6 +399,7 @@ type serveTarget struct { profile string service string pinnedFile string + agent protos.AitoolsAgentType } // serveChallenge returns the sign-in handling of a service that needs provider sign-in, or nil for @@ -405,6 +442,7 @@ func serveChallenge(ctx context.Context, t serveTarget, errOut io.Writer) (*prox LogoutCommand: mcpLogoutHint(t.service, t.profile, t.pinnedFile), PinnedFile: t.pinnedFile, SignIn: serveSignIn(t), + Recorder: mcpProxyTelemetry{agent: t.agent, service: t.service}, }, nil } diff --git a/cmd/aitools/mcp_serve_test.go b/cmd/aitools/mcp_serve_test.go index d64a8d48062..676239b72f1 100644 --- a/cmd/aitools/mcp_serve_test.go +++ b/cmd/aitools/mcp_serve_test.go @@ -15,6 +15,7 @@ import ( "os/exec" "path/filepath" "runtime" + "slices" "strings" "sync" "sync/atomic" @@ -28,8 +29,13 @@ import ( "github.com/databricks/cli/libs/aitools/mcpconfig" "github.com/databricks/cli/libs/auth/storage" "github.com/databricks/cli/libs/auth/u2m" + "github.com/databricks/cli/libs/cmdctx" "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/dbr" + "github.com/databricks/cli/libs/env" "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/telemetry" + "github.com/databricks/cli/libs/telemetry/protos" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/config" "github.com/databricks/databricks-sdk-go/httpclient" @@ -618,6 +624,146 @@ func TestMcpServeRejectsServiceWithoutThreeParts(t *testing.T) { } } +// TestMcpServeFailedStartLogsNothing checks that a start that fails its binding checks logs no PROXY_STARTED, so nothing +// uploads. +func TestMcpServeFailedStartLogsNothing(t *testing.T) { + withHome(t) + parent := newMCPTestRoot() + parent.SetErr(&bytes.Buffer{}) + parent.SetArgs([]string{"mcp", "serve", "--profile", testProfile, "--service", testService}) + ctx := env.Set(telemetry.WithNewLogger(cmdio.MockDiscard(t.Context())), "DATABRICKS_CLI_DISABLE_TELEMETRY", "") + err := parent.ExecuteContext(ctx) + require.ErrorIs(t, err, root.ErrAlreadyPrinted) + assert.Equal(t, serveBindStatus, root.ExitStatus(err)) + serve, _, findErr := parent.Find([]string{"mcp", "serve"}) + require.NoError(t, findErr) + assert.Empty(t, telemetry.BufferedEvents(serve.Context())) +} + +// TestMcpServeRejectedStartLogsNothing checks that a start whose startup read is rejected by the profile's credential logs +// no PROXY_STARTED, so the run has nothing to upload, and sets no upload config. +func TestMcpServeRejectedStartLogsNothing(t *testing.T) { + home := withHome(t) + server := serviceServer(t, http.StatusUnauthorized, `{"error_code":"UNAUTHENTICATED","message":"Invalid access token."}`) + writeHomeConfig(t, home, "[logfood]\nhost = "+server.URL+"\nauth_type = pat\ntoken = dapi-expired\n") + + parent := newMCPTestRoot() + parent.SetErr(&bytes.Buffer{}) + parent.SetArgs([]string{"mcp", "serve", "--profile", testProfile, "--service", testService}) + ctx := telemetry.WithNewLogger(cmdio.MockDiscard(t.Context())) + err := parent.ExecuteContext(ctx) + require.ErrorIs(t, err, root.ErrAlreadyPrinted) + assert.Equal(t, serveBindStatus, root.ExitStatus(err)) + serve, _, findErr := parent.Find([]string{"mcp", "serve"}) + require.NoError(t, findErr) + assert.Empty(t, telemetry.BufferedEvents(serve.Context())) + _, uploads := telemetry.UploadConfig(serve.Context()) + assert.False(t, uploads) +} + +// TestMcpServeUploadUsesProfileBoundAtStart checks that the exit upload of a serving run authenticates with the profile as +// it was bound at start, so a token that the config file gets during the session does not change the upload. +func TestMcpServeUploadUsesProfileBoundAtStart(t *testing.T) { + home := withHome(t) + server := newEchoMCPServer(t) + writeHomeConfig(t, home, "[logfood]\nhost = "+server.URL+"\nworkspace_id = 222\nauth_type = pat\ntoken = dapi-bound\n") + + parent := newMCPTestRoot() + parent.SetErr(&bytes.Buffer{}) + var stdout lockedBuffer + parent.SetOut(&stdout) + in, stdin := io.Pipe() + parent.SetIn(in) + parent.SetArgs([]string{"mcp", "serve", "--profile", testProfile, "--service", testService}) + ctx := serveRunContext(t) + done := make(chan error, 1) + go func() { done <- parent.ExecuteContext(ctx) }() + + _, err := io.WriteString(stdin, `{"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}`+"\n") + require.NoError(t, err) + require.Eventually(t, func() bool { return strings.Contains(stdout.String(), `"id":1`) }, 10*time.Second, time.Millisecond) + writeHomeConfig(t, home, "[logfood]\nhost = "+server.URL+"\nworkspace_id = 222\nauth_type = pat\ntoken = dapi-rotated\n") + require.NoError(t, stdin.Close()) + require.NoError(t, <-done) + + serve, _, findErr := parent.Find([]string{"mcp", "serve"}) + require.NoError(t, findErr) + cfg, uploads := telemetry.UploadConfig(serve.Context()) + require.True(t, uploads) + assert.Equal(t, "dapi-bound", cfg.Token) +} + +// serveRunContext returns the context of a serve run whose telemetry uploads: enabled, with the exec ID and runtime that +// cmd/root sets on a run. +func serveRunContext(t *testing.T) context.Context { + t.Helper() + ctx := env.Set(telemetry.WithNewLogger(cmdio.MockDiscard(t.Context())), "DATABRICKS_CLI_DISABLE_TELEMETRY", "") + return cmdctx.GenerateExecId(dbr.DetectRuntime(ctx)) +} + +// TestMcpServeUploadsProxyStartedAtStart checks that serve uploads its PROXY_STARTED while it still serves, so a serve that +// is killed, or whose token expires while it idles, has sent it. The exit upload does not send it again. +func TestMcpServeUploadsProxyStartedAtStart(t *testing.T) { + home := withHome(t) + server, uploaded := newEchoMCPServerWithUploads(t) + writeHomeConfig(t, home, "[logfood]\nhost = "+server.URL+"\nworkspace_id = 222\nauth_type = pat\ntoken = dapi-bound\n") + + parent := newMCPTestRoot() + parent.SetErr(&bytes.Buffer{}) + var stdout lockedBuffer + parent.SetOut(&stdout) + in, stdin := io.Pipe() + parent.SetIn(in) + parent.SetArgs([]string{"mcp", "serve", "--profile", testProfile, "--service", testService}) + ctx := serveRunContext(t) + done := make(chan error, 1) + go func() { done <- parent.ExecuteContext(ctx) }() + + _, err := io.WriteString(stdin, `{"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}`+"\n") + require.NoError(t, err) + require.Eventually(t, func() bool { return strings.Contains(stdout.String(), `"id":1`) }, 10*time.Second, time.Millisecond) + assert.Equal(t, []protos.AitoolsMcpOperation{protos.AitoolsMcpOperationProxyStarted}, uploaded(), "serve is still running") + + require.NoError(t, stdin.Close()) + require.NoError(t, <-done) + serve, _, findErr := parent.Find([]string{"mcp", "serve"}) + require.NoError(t, findErr) + require.NoError(t, telemetry.Upload(serve.Context(), protos.ExecutionContext{})) + assert.Equal(t, []protos.AitoolsMcpOperation{protos.AitoolsMcpOperationProxyStarted}, uploaded(), "the exit upload does not resend it") +} + +// TestMcpServeDropsUploadAfterRepoint checks that a profile re-pointed at another host during the session leaves serve +// with no upload config at exit, even when no request ran after the re-point. The upload's token comes from the token +// cache by profile name, so it would be the new host's token, sent to the bound host. +func TestMcpServeDropsUploadAfterRepoint(t *testing.T) { + home := withHome(t) + server := newEchoMCPServer(t) + writeHomeConfig(t, home, "[logfood]\nhost = "+server.URL+"\nworkspace_id = 222\nauth_type = pat\ntoken = dapi-bound\n") + + parent := newMCPTestRoot() + parent.SetErr(&bytes.Buffer{}) + var stdout lockedBuffer + parent.SetOut(&stdout) + in, stdin := io.Pipe() + parent.SetIn(in) + parent.SetArgs([]string{"mcp", "serve", "--profile", testProfile, "--service", testService}) + ctx := serveRunContext(t) + done := make(chan error, 1) + go func() { done <- parent.ExecuteContext(ctx) }() + + _, err := io.WriteString(stdin, `{"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}`+"\n") + require.NoError(t, err) + require.Eventually(t, func() bool { return strings.Contains(stdout.String(), `"id":1`) }, 10*time.Second, time.Millisecond) + writeHomeConfig(t, home, "[logfood]\nhost = https://other.test\nworkspace_id = 222\nauth_type = pat\ntoken = dapi-other\n") + require.NoError(t, stdin.Close()) + require.NoError(t, <-done) + + serve, _, findErr := parent.Find([]string{"mcp", "serve"}) + require.NoError(t, findErr) + _, uploads := telemetry.UploadConfig(serve.Context()) + assert.False(t, uploads) +} + // TestMcpServeStateReadFailureExitsTwo checks that a state file that cannot be read stops the binding checks, so it // exits 2 like the other binding failures. func TestMcpServeStateReadFailureExitsTwo(t *testing.T) { @@ -670,11 +816,16 @@ func TestProxyResultRefreshFailureWinsOverStdinError(t *testing.T) { } require.Error(t, g.authenticateRequest(&http.Request{Header: http.Header{}})) cmd := NewMcpServeCmd() + cmd.SetContext(telemetry.WithNewLogger(t.Context())) + telemetry.SetUploadConfig(cmd.Context(), &config.Config{Host: "https://adb-1.test"}) var stderr bytes.Buffer cmd.SetErr(&stderr) err := proxyResult(cmd, g, errors.New("read stdin: input/output error")) + // The run uploads like any other: its upload config is the profile bound at start, which never refreshes a token. + _, uploads := telemetry.UploadConfig(cmd.Context()) + assert.True(t, uploads) assert.ErrorIs(t, err, root.ErrAlreadyPrinted) assert.Equal(t, serveBindStatus, root.ExitStatus(err)) assert.ErrorIs(t, err, cause) @@ -758,29 +909,70 @@ func failsToolsListOnce(err error) func(*http.Request) error { // so the session DELETE at shutdown is sent. func newEchoMCPServer(t *testing.T) *httptest.Server { t.Helper() + server := httptest.NewServer(http.HandlerFunc(echoMCP)) + t.Cleanup(server.Close) + return server +} + +// echoMCP answers an MCP request as newEchoMCPServer does. +func echoMCP(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodDelete { + w.WriteHeader(http.StatusOK) + return + } + var msg struct { + ID json.RawMessage `json:"id"` + Method string `json:"method"` + } + if err := json.NewDecoder(r.Body).Decode(&msg); err != nil { + w.WriteHeader(http.StatusBadRequest) + return + } + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Mcp-Session-Id", "session-1") + result := `{"tools":[]}` + if msg.Method == "initialize" { + result = `{"protocolVersion":"2025-06-18","capabilities":{},"serverInfo":{"name":"fake","version":"1"}}` + } + fmt.Fprintf(w, `{"jsonrpc":"2.0","id":%s,"result":%s}`, msg.ID, result) +} + +// newEchoMCPServerWithUploads is newEchoMCPServer that also takes telemetry uploads. The function it returns gives the +// operation of each AitoolsMcpEvent the uploads carried so far, in order. +func newEchoMCPServerWithUploads(t *testing.T) (*httptest.Server, func() []protos.AitoolsMcpOperation) { + t.Helper() + var mu sync.Mutex + var operations []protos.AitoolsMcpOperation server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Method == http.MethodDelete { - w.WriteHeader(http.StatusOK) + if r.URL.Path != "/telemetry-ext" { + echoMCP(w, r) return } - var msg struct { - ID json.RawMessage `json:"id"` - Method string `json:"method"` - } - if err := json.NewDecoder(r.Body).Decode(&msg); err != nil { + var body telemetry.RequestBody + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { w.WriteHeader(http.StatusBadRequest) return } - w.Header().Set("Content-Type", "application/json") - w.Header().Set("Mcp-Session-Id", "session-1") - result := `{"tools":[]}` - if msg.Method == "initialize" { - result = `{"protocolVersion":"2025-06-18","capabilities":{},"serverInfo":{"name":"fake","version":"1"}}` + mu.Lock() + defer mu.Unlock() + for _, p := range body.ProtoLogs { + var entry protos.FrontendLog + if err := json.Unmarshal([]byte(p), &entry); err != nil { + w.WriteHeader(http.StatusBadRequest) + return + } + if e := entry.Entry.DatabricksCliLog.AitoolsMcpEvent; e != nil { + operations = append(operations, e.Operation) + } } - fmt.Fprintf(w, `{"jsonrpc":"2.0","id":%s,"result":%s}`, msg.ID, result) + fmt.Fprintf(w, `{"numProtoSuccess":%d}`, len(body.ProtoLogs)) })) t.Cleanup(server.Close) - return server + return server, func() []protos.AitoolsMcpOperation { + mu.Lock() + defer mu.Unlock() + return slices.Clone(operations) + } } // TestServeProxyTransientRefreshFailureKeepsServing checks that a token failure which is not a rejection fails only the diff --git a/cmd/aitools/mcp_telemetry.go b/cmd/aitools/mcp_telemetry.go new file mode 100644 index 00000000000..057762fec92 --- /dev/null +++ b/cmd/aitools/mcp_telemetry.go @@ -0,0 +1,288 @@ +package aitools + +import ( + "context" + "strings" + + "github.com/databricks/cli/libs/aitools/agents" + "github.com/databricks/cli/libs/aitools/mcp" + "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/telemetry" + "github.com/databricks/cli/libs/telemetry/protos" +) + +// setUploadConfig sets the config of profileName in configFile for the run's telemetry upload: resolved within +// telemetryAuthBudget, with credentials that never refresh (mcp.BoundUploadConfig). A command calls it once its binding +// checks have passed, so the upload authenticates with the profile the command bound. Nothing is resolved when telemetry +// is disabled, and a profile that does not resolve uploads nothing. +func setUploadConfig(ctx context.Context, profileName, configFile string) { + if telemetry.Disabled(ctx) { + return + } + cfg, err := mcp.BoundUploadConfig(ctx, profileName, configFile, telemetryAuthBudget) + if err != nil { + log.Debugf(ctx, "telemetry: cannot resolve profile %s to upload with: %s", profileName, err) + telemetry.SkipUpload(ctx) + return + } + telemetry.SetUploadConfig(ctx, cfg) +} + +// mcpServiceTypes maps the system.ai leaves in the CLI's service table to their telemetry enum. +var mcpServiceTypes = map[string]protos.AitoolsMcpService{ + "genie_one_mcp": protos.AitoolsMcpServiceGenieOne, + "dbsql": protos.AitoolsMcpServiceDBSQL, + "web_search": protos.AitoolsMcpServiceWebSearch, + "sandbox": protos.AitoolsMcpServiceSandbox, + "slack": protos.AitoolsMcpServiceSlack, + "github": protos.AitoolsMcpServiceGitHub, + "atlassian": protos.AitoolsMcpServiceAtlassian, + "google_drive": protos.AitoolsMcpServiceGoogleDrive, + "google_calendar": protos.AitoolsMcpServiceGoogleCalendar, + "gmail": protos.AitoolsMcpServiceGmail, + "microsoft_365": protos.AitoolsMcpServiceMicrosoft365, +} + +// mcpServiceType maps a service full name to its telemetry enum. The name itself is never sent: +// a system.ai leaf in the table has its own value, other system.ai leaves are OTHER_SYSTEM, and +// services outside system.ai are CUSTOM. +func mcpServiceType(fullName string) protos.AitoolsMcpService { + parts := strings.Split(fullName, ".") + if len(parts) != 3 || !strings.EqualFold(parts[0]+"."+parts[1], mcp.SystemAILocation) { + return protos.AitoolsMcpServiceCustom + } + if t, ok := mcpServiceTypes[strings.ToLower(parts[2])]; ok { + return t + } + return protos.AitoolsMcpServiceOtherSystem +} + +// callingAgentType returns the agent that drives the CLI, Unspecified when none is detected. +func callingAgentType() protos.AitoolsAgentType { + a := agents.CallingAgent() + if a == nil { + return protos.AitoolsAgentTypeUnspecified + } + return agentType(a.Name) +} + +// proxyAgentType returns the agent that started the proxy: the --agent flag when given, else the +// detected agent. +func proxyAgentType(flag string) protos.AitoolsAgentType { + if flag == "" { + return callingAgentType() + } + return agentType(flag) +} + +// logMcpEvent buffers an MCP event, and reports whether it did; cmd/root uploads it at exit. An event without server +// results is not logged: a run that stopped before any server was reached has nothing to report. The bound profile count +// is read from the state file as it is after the operation. A state file that cannot be read drops the event; telemetry +// never fails a command. +func logMcpEvent(ctx context.Context, event protos.AitoolsMcpEvent) bool { + if len(event.Servers) == 0 { + return false + } + state, err := mcp.LoadState(ctx) + if err != nil { + log.Debugf(ctx, "telemetry: cannot read the MCP state; dropping the event: %s", err) + return false + } + event.BoundProfileCount = boundProfileCount(ctx, state) + telemetry.Log(ctx, protos.DatabricksCliLog{AitoolsMcpEvent: &event}) + return true +} + +// boundProfileCount returns the number of distinct profiles that have servers installed, grouped as list groups them +// (profileGroup), so DEFAULT in two config files counts twice. +func boundProfileCount(ctx context.Context, state *mcp.State) int64 { + profiles := map[profileKey]bool{} + checks := mcp.CheckEntries(ctx, state) + for i, e := range state.Servers { + profiles[profileGroup(e, checks[i])] = true + } + return int64(len(profiles)) +} + +// addSignInState is the sign-in state of a server after an add: the state that --login reports, else PENDING for a +// server that needs sign-in, else NOT_REQUIRED. A sign-in that failed before it read a state is PENDING. +func addSignInState(s addServerOutput) protos.AitoolsMcpSignInState { + if s.SignIn != nil && s.SignIn.State != "" { + return signInStateOf(s.SignIn.State) + } + if s.SignInRequired { + return protos.AitoolsMcpSignInStatePending + } + return protos.AitoolsMcpSignInStateNotRequired +} + +// addServerResults returns one result per (service, agent) of an add run. Installed entries are +// CHANGED, re-asserted ones UNCHANGED, and failed ones FAILED with their category. +func addServerResults(servers []addServerOutput) []protos.AitoolsMcpServerResult { + var results []protos.AitoolsMcpServerResult + for _, s := range servers { + for _, a := range s.Agents { + r := protos.AitoolsMcpServerResult{ + Service: mcpServiceType(s.Service), + Agent: agentType(a.Name), + SignInState: addSignInState(s), + } + switch a.Status { + case statusInstalled: + r.Outcome = protos.AitoolsTargetOutcomeChanged + case statusReasserted: + r.Outcome = protos.AitoolsTargetOutcomeUnchanged + default: + r.Outcome = protos.AitoolsTargetOutcomeFailed + r.ErrorCategory = a.ErrorCategory + } + results = append(results, r) + } + } + return results +} + +// removeServerResult returns the result of removing one server from one agent config. An entry that was deleted, or +// pruned (its record dropped), is CHANGED. An entry that was changed outside this CLI is SKIPPED: it is left in place. +func removeServerResult(service, agent string, outcome removeOutcome, err error) protos.AitoolsMcpServerResult { + r := protos.AitoolsMcpServerResult{Service: mcpServiceType(service), Agent: agentType(agent)} + switch { + case err != nil: + r.Outcome = protos.AitoolsTargetOutcomeFailed + r.ErrorCategory = errorCategory(err) + case outcome == removeKept: + r.Outcome = protos.AitoolsTargetOutcomeSkipped + default: + r.Outcome = protos.AitoolsTargetOutcomeChanged + } + return r +} + +// signInStateOf maps a sign-in state name of login to its telemetry state. A run that never learned the state leaves it +// unset. Anything but ACTIVE or NOT_REQUIRED is still PENDING. +func signInStateOf(state string) protos.AitoolsMcpSignInState { + switch state { + case "": + return "" + case signInStateActive: + return protos.AitoolsMcpSignInStateActive + case signInStateNotRequired: + return protos.AitoolsMcpSignInStateNotRequired + } + return protos.AitoolsMcpSignInStatePending +} + +// signInServerResult returns the result of one provider sign-in. The sign-in was already active +// (alreadySignedIn) or the service needed none, so nothing changed: UNCHANGED. +func signInServerResult(service, state string, alreadySignedIn bool, err error) protos.AitoolsMcpServerResult { + r := protos.AitoolsMcpServerResult{Service: mcpServiceType(service), SignInState: signInStateOf(state)} + switch { + case err != nil: + r.Outcome = protos.AitoolsTargetOutcomeFailed + r.ErrorCategory = errorCategory(err) + case alreadySignedIn || state == signInStateNotRequired: + r.Outcome = protos.AitoolsTargetOutcomeUnchanged + default: + r.Outcome = protos.AitoolsTargetOutcomeChanged + } + return r +} + +// signInEventOf is the SIGN_IN event of one provider sign-in, under add --login or login. A sign-in that found the caller +// already signed in is UNCHANGED. +func signInEventOf(service, state string, alreadySignedIn bool, err error) protos.AitoolsMcpEvent { + return protos.AitoolsMcpEvent{ + CallingAgent: callingAgentType(), + Operation: protos.AitoolsMcpOperationSignIn, + Servers: []protos.AitoolsMcpServerResult{signInServerResult(service, state, alreadySignedIn, err)}, + } +} + +// signOutServerResult returns the result of a logout. Logout does not read whether the service needs +// sign-in, so the sign-in state is not set. +func signOutServerResult(service, state string, err error) protos.AitoolsMcpServerResult { + r := protos.AitoolsMcpServerResult{Service: mcpServiceType(service)} + switch { + case err != nil: + r.Outcome = protos.AitoolsTargetOutcomeFailed + r.ErrorCategory = errorCategory(err) + case state == signOutStateDeleted: + r.Outcome = protos.AitoolsTargetOutcomeChanged + default: + r.Outcome = protos.AitoolsTargetOutcomeUnchanged + } + return r +} + +// proxyStartedEvent is the PROXY_STARTED event of serve, logged once the startup checks have passed and the proxy serves. +// A start that fails its checks, whether at the binding or on a rejected credential, logs nothing, so its run has nothing +// to upload. +func proxyStartedEvent(agent protos.AitoolsAgentType, service string) protos.AitoolsMcpEvent { + return protos.AitoolsMcpEvent{ + CallingAgent: agent, + Operation: protos.AitoolsMcpOperationProxyStarted, + Servers: []protos.AitoolsMcpServerResult{{Service: mcpServiceType(service)}}, + } +} + +// mcpProxyTelemetry is the proxy's telemetry for one serve session. It implements proxy.Recorder. +type mcpProxyTelemetry struct { + agent protos.AitoolsAgentType + service string +} + +// SignInPrompted logs SIGN_IN_PROMPTED once the sign-in page the authenticate tool opened has been +// waited on. signedIn reports whether the credential became ACTIVE. +func (t mcpProxyTelemetry) SignInPrompted(ctx context.Context, signedIn bool) { + logMcpEvent(ctx, signInPromptedEvent(t.agent, t.service, signedIn)) +} + +// signInPromptedEvent is the SIGN_IN_PROMPTED event of serve: the sign-in is required, and the +// sign-in state is ACTIVE when the wait ended with the credential, PENDING otherwise. +func signInPromptedEvent(agent protos.AitoolsAgentType, service string, signedIn bool) protos.AitoolsMcpEvent { + state := protos.AitoolsMcpSignInStatePending + if signedIn { + state = protos.AitoolsMcpSignInStateActive + } + return protos.AitoolsMcpEvent{ + CallingAgent: agent, + Operation: protos.AitoolsMcpOperationSignInPrompted, + Servers: []protos.AitoolsMcpServerResult{{ + Service: mcpServiceType(service), + ErrorCategory: protos.AitoolsErrorCategorySignInRequired, + SignInState: state, + }}, + } +} + +// bindingDrifted reports whether a server recorded for the profile of key no longer matches that profile in its config +// file: its host or workspace changed. The upload would send the profile's token to the host the profile names now, which +// the server was not set up against. Only the servers grouped under key are checked, so a profile of the same name in +// another config file does not count. It reads the config files only. +func bindingDrifted(ctx context.Context, state *mcp.State, key profileKey) bool { + checks := mcp.CheckEntries(ctx, state) + for i, e := range state.Servers { + if profileGroup(e, checks[i]) != key { + continue + } + if _, err := mcp.CheckBinding(ctx, state, e.Service, key.profile, key.path); err != nil { + return true + } + } + return false +} + +// sharedProfile returns the profile and config file that all entries are bound to, grouped as list groups them, or the +// zero key when they are bound to none or to several. checks are the checks of entries, in the same order. +func sharedProfile(entries []mcp.Entry, checks []mcp.EntryCheck) profileKey { + if len(entries) == 0 { + return profileKey{} + } + first := profileGroup(entries[0], checks[0]) + for i, e := range entries[1:] { + if profileGroup(e, checks[i+1]) != first { + return profileKey{} + } + } + return first +} diff --git a/cmd/aitools/mcp_telemetry_test.go b/cmd/aitools/mcp_telemetry_test.go new file mode 100644 index 00000000000..e6e7ec7f3fc --- /dev/null +++ b/cmd/aitools/mcp_telemetry_test.go @@ -0,0 +1,340 @@ +package aitools + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io" + "log/slog" + "maps" + "path/filepath" + "slices" + "testing" + + "github.com/databricks/cli/libs/aitools/agents" + "github.com/databricks/cli/libs/aitools/mcp" + "github.com/databricks/cli/libs/env" + "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/telemetry" + "github.com/databricks/cli/libs/telemetry/protos" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/config" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestMcpServiceType(t *testing.T) { + // Every leaf in the CLI's known table has its own enum, so a known service never reports OTHER_SYSTEM. + cases := map[string]protos.AitoolsMcpService{ + "system.ai.genie_one_mcp": protos.AitoolsMcpServiceGenieOne, + "system.ai.dbsql": protos.AitoolsMcpServiceDBSQL, + "system.ai.web_search": protos.AitoolsMcpServiceWebSearch, + "system.ai.sandbox": protos.AitoolsMcpServiceSandbox, + "system.ai.slack": protos.AitoolsMcpServiceSlack, + "system.ai.github": protos.AitoolsMcpServiceGitHub, + "system.ai.atlassian": protos.AitoolsMcpServiceAtlassian, + "system.ai.google_drive": protos.AitoolsMcpServiceGoogleDrive, + "system.ai.google_calendar": protos.AitoolsMcpServiceGoogleCalendar, + "system.ai.gmail": protos.AitoolsMcpServiceGmail, + "system.ai.microsoft_365": protos.AitoolsMcpServiceMicrosoft365, + // The schema name is case-insensitive in Unity Catalog. + "System.AI.GitHub": protos.AitoolsMcpServiceGitHub, + // A system.ai leaf the CLI does not know yet. + "system.ai.future_tool": protos.AitoolsMcpServiceOtherSystem, + // Anything outside system.ai, whatever its name. + "main.tools.github": protos.AitoolsMcpServiceCustom, + "system.ai": protos.AitoolsMcpServiceCustom, + "system.ai.a.github": protos.AitoolsMcpServiceCustom, + } + for name, want := range cases { + assert.Equal(t, want, mcpServiceType(name), name) + } +} + +// TestMcpServiceTypeCoversKnownServices checks that every service in the CLI's known table has its own enum, so a known +// service never reports OTHER_SYSTEM. A new table entry needs a new value in mcpServiceTypes. +func TestMcpServiceTypeCoversKnownServices(t *testing.T) { + for _, leaf := range mcp.KnownLeaves() { + assert.Contains(t, mcpServiceTypes, leaf, "system.ai.%s has no telemetry enum: add it to mcpServiceTypes", leaf) + assert.NotEqualf(t, protos.AitoolsMcpServiceOtherSystem, mcpServiceType(mcp.SystemAILocation+"."+leaf), + "system.ai.%s has no telemetry enum", leaf) + } +} + +func TestAddServerResults(t *testing.T) { + servers := []addServerOutput{ + { + Service: "system.ai.github", + SignInRequired: true, + SignIn: &addSignInOutput{State: signInStateActive}, + Agents: []addAgentOutput{ + {Name: "claude-code", Status: statusInstalled}, + {Name: "codex", Status: statusReasserted}, + {Name: "cursor", Status: statusFailed, ErrorCategory: protos.AitoolsErrorCategoryAgentConfigInvalid}, + }, + }, + { + Service: "main.tools.custom", + Agents: []addAgentOutput{{Name: "claude-code", Status: statusInstalled}}, + }, + } + + assert.Equal(t, []protos.AitoolsMcpServerResult{ + { + Service: protos.AitoolsMcpServiceGitHub, + Agent: protos.AitoolsAgentTypeClaudeCode, + Outcome: protos.AitoolsTargetOutcomeChanged, + SignInState: protos.AitoolsMcpSignInStateActive, + }, + { + Service: protos.AitoolsMcpServiceGitHub, + Agent: protos.AitoolsAgentTypeCodex, + Outcome: protos.AitoolsTargetOutcomeUnchanged, + SignInState: protos.AitoolsMcpSignInStateActive, + }, + { + Service: protos.AitoolsMcpServiceGitHub, + Agent: protos.AitoolsAgentTypeCursor, + Outcome: protos.AitoolsTargetOutcomeFailed, + ErrorCategory: protos.AitoolsErrorCategoryAgentConfigInvalid, + SignInState: protos.AitoolsMcpSignInStateActive, + }, + { + Service: protos.AitoolsMcpServiceCustom, + Agent: protos.AitoolsAgentTypeClaudeCode, + Outcome: protos.AitoolsTargetOutcomeChanged, + SignInState: protos.AitoolsMcpSignInStateNotRequired, + }, + }, addServerResults(servers)) +} + +func TestRemoveServerResult(t *testing.T) { + assert.Equal(t, protos.AitoolsTargetOutcomeChanged, + removeServerResult("system.ai.github", "claude-code", removeDeleted, nil).Outcome) + // A pruned entry changes the state, so it is CHANGED too. An entry changed outside this CLI is left in place: SKIPPED. + assert.Equal(t, protos.AitoolsTargetOutcomeChanged, + removeServerResult("system.ai.github", "claude-code", removePruned, nil).Outcome) + assert.Equal(t, protos.AitoolsTargetOutcomeSkipped, + removeServerResult("system.ai.github", "claude-code", removeKept, nil).Outcome) + + failed := removeServerResult("system.ai.github", "codex", removeDeleted, errors.New("config is not JSON")) + assert.Equal(t, protos.AitoolsMcpServerResult{ + Service: protos.AitoolsMcpServiceGitHub, + Agent: protos.AitoolsAgentTypeCodex, + Outcome: protos.AitoolsTargetOutcomeFailed, + ErrorCategory: protos.AitoolsErrorCategoryUncategorized, + }, failed) +} + +func TestSignInServerResult(t *testing.T) { + cases := []struct { + name string + state string + alreadySignedIn bool + err error + want protos.AitoolsMcpServerResult + }{ + { + name: "signed in now", + state: signInStateActive, + want: protos.AitoolsMcpServerResult{Service: protos.AitoolsMcpServiceGitHub, Outcome: protos.AitoolsTargetOutcomeChanged, SignInState: protos.AitoolsMcpSignInStateActive}, + }, + { + name: "already active", + state: signInStateActive, + alreadySignedIn: true, + want: protos.AitoolsMcpServerResult{Service: protos.AitoolsMcpServiceGitHub, Outcome: protos.AitoolsTargetOutcomeUnchanged, SignInState: protos.AitoolsMcpSignInStateActive}, + }, + { + name: "no sign-in needed", + state: signInStateNotRequired, + want: protos.AitoolsMcpServerResult{Service: protos.AitoolsMcpServiceGitHub, Outcome: protos.AitoolsTargetOutcomeUnchanged, SignInState: protos.AitoolsMcpSignInStateNotRequired}, + }, + { + // A run that never learned the state, such as a read that failed, reports none. + name: "state not learned", + err: failure(protos.AitoolsErrorCategoryMCPServiceNotFound, "not found"), + want: protos.AitoolsMcpServerResult{ + Service: protos.AitoolsMcpServiceGitHub, Outcome: protos.AitoolsTargetOutcomeFailed, + ErrorCategory: protos.AitoolsErrorCategoryMCPServiceNotFound, + }, + }, + { + name: "timed out", + state: signInStateTimeout, + err: failure(protos.AitoolsErrorCategorySignInTimeout, "timed out"), + want: protos.AitoolsMcpServerResult{ + Service: protos.AitoolsMcpServiceGitHub, Outcome: protos.AitoolsTargetOutcomeFailed, + ErrorCategory: protos.AitoolsErrorCategorySignInTimeout, SignInState: protos.AitoolsMcpSignInStatePending, + }, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, signInServerResult("system.ai.github", tc.state, tc.alreadySignedIn, tc.err)) + }) + } +} + +func TestSignOutServerResult(t *testing.T) { + // Logout does not read the sign-in state, so the result has none. + assert.Equal(t, protos.AitoolsMcpServerResult{Service: protos.AitoolsMcpServiceGitHub, Outcome: protos.AitoolsTargetOutcomeChanged}, + signOutServerResult("system.ai.github", signOutStateDeleted, nil)) + assert.Equal(t, protos.AitoolsMcpServerResult{Service: protos.AitoolsMcpServiceGitHub, Outcome: protos.AitoolsTargetOutcomeUnchanged}, + signOutServerResult("system.ai.github", signOutStateNotSignedIn, nil)) +} + +func TestProxyStartedEvent(t *testing.T) { + assert.Equal(t, protos.AitoolsMcpEvent{ + CallingAgent: protos.AitoolsAgentTypeClaudeCode, + Operation: protos.AitoolsMcpOperationProxyStarted, + Servers: []protos.AitoolsMcpServerResult{{Service: protos.AitoolsMcpServiceGenieOne}}, + }, proxyStartedEvent(protos.AitoolsAgentTypeClaudeCode, "system.ai.genie_one_mcp")) +} + +// TestLogMcpEventLogsRejectedCredential checks that an event that names a rejected credential is logged, and that the run +// uploads with it like any other run: the upload authenticates with the profile the run bound, which never refreshes. +func TestLogMcpEventLogsRejectedCredential(t *testing.T) { + withHome(t) + ctx := env.Set(telemetry.WithNewLogger(t.Context()), "DATABRICKS_CLI_DISABLE_TELEMETRY", "") + telemetry.SetUploadConfig(ctx, &config.Config{Profile: "logfood", Host: "https://adb-1.test"}) + assert.True(t, logMcpEvent(ctx, protos.AitoolsMcpEvent{ + Operation: protos.AitoolsMcpOperationSignIn, + Servers: []protos.AitoolsMcpServerResult{{ + Service: protos.AitoolsMcpServiceGitHub, + Outcome: protos.AitoolsTargetOutcomeFailed, + ErrorCategory: protos.AitoolsErrorCategoryAuthFailed, + }}, + })) + assert.Len(t, telemetry.BufferedEvents(ctx), 1) + _, uploads := telemetry.UploadConfig(ctx) + assert.True(t, uploads) +} + +// TestSetUploadConfigLogsNoWarningForFailedHostMetadata checks that an upload whose host metadata lookup fails warns +// nothing through the global logger, since the SDK would log that failure there. The command has succeeded, and the +// upload is best-effort. +func TestSetUploadConfigLogsNoWarningForFailedHostMetadata(t *testing.T) { + home := withHome(t) + writeHomeConfig(t, home, "[logfood]\nhost = https://adb-1.test\nauth_type = pat\ntoken = t\n") + factory := config.DefaultHostMetadataResolverFactory + // Like the registered factory (libs/hostmetadata), the resolver logs its failure as a warning and returns no error. + config.DefaultHostMetadataResolverFactory = func(*config.Config) config.HostMetadataResolver { + return func(ctx context.Context, host string) (*config.HostMetadata, error) { + log.Warnf(ctx, "[hostmetadata] failed to fetch host metadata for %s, will skip for 1m0s", host) + return nil, nil + } + } + t.Cleanup(func() { config.DefaultHostMetadataResolverFactory = factory }) + var warnings bytes.Buffer + defaultLogger := slog.Default() + slog.SetDefault(slog.New(slog.NewTextHandler(&warnings, &slog.HandlerOptions{Level: slog.LevelWarn}))) + t.Cleanup(func() { slog.SetDefault(defaultLogger) }) + + ctx := env.Set(telemetry.WithNewLogger(t.Context()), "DATABRICKS_CLI_DISABLE_TELEMETRY", "") + setUploadConfig(ctx, "logfood", filepath.Join(home, ".databrickscfg")) + _, uploads := telemetry.UploadConfig(ctx) + assert.True(t, uploads) + assert.Empty(t, warnings.String()) +} + +func TestSignInPromptedEvent(t *testing.T) { + for signedIn, state := range map[bool]protos.AitoolsMcpSignInState{ + true: protos.AitoolsMcpSignInStateActive, + false: protos.AitoolsMcpSignInStatePending, + } { + event := signInPromptedEvent(protos.AitoolsAgentTypeCursor, "system.ai.github", signedIn) + assert.Equal(t, protos.AitoolsMcpEvent{ + CallingAgent: protos.AitoolsAgentTypeCursor, + Operation: protos.AitoolsMcpOperationSignInPrompted, + Servers: []protos.AitoolsMcpServerResult{{ + Service: protos.AitoolsMcpServiceGitHub, + ErrorCategory: protos.AitoolsErrorCategorySignInRequired, + SignInState: state, + }}, + }, event) + } +} + +func TestBoundProfileCount(t *testing.T) { + dir := t.TempDir() + first := filepath.Join(dir, "first.cfg") + second := filepath.Join(dir, "second.cfg") + state := &mcp.State{Servers: []mcp.Entry{ + {Profile: "logfood", Service: "system.ai.github", ConfigFile: first}, + {Profile: "logfood", Service: "system.ai.genie_one_mcp", ConfigFile: first}, + // DEFAULT in two config files is two profiles, as list groups them. + {Profile: "DEFAULT", Service: "system.ai.github", ConfigFile: first}, + {Profile: "DEFAULT", Service: "system.ai.github", ConfigFile: second}, + }} + assert.Equal(t, int64(3), boundProfileCount(t.Context(), state)) + assert.Equal(t, int64(0), boundProfileCount(t.Context(), &mcp.State{})) +} + +func TestSignInEventOf(t *testing.T) { + // An ACTIVE credential that the add found signed in is UNCHANGED, as add --login reports it. + assert.Equal(t, protos.AitoolsMcpEvent{ + CallingAgent: callingAgentType(), + Operation: protos.AitoolsMcpOperationSignIn, + Servers: []protos.AitoolsMcpServerResult{{ + Service: protos.AitoolsMcpServiceGitHub, + Outcome: protos.AitoolsTargetOutcomeUnchanged, + SignInState: protos.AitoolsMcpSignInStateActive, + }}, + }, signInEventOf("system.ai.github", signInStateActive, true, nil)) +} + +// TestSignInInstalledURLFailureLogsFailedSignIn checks that an add --login whose sign-in URL cannot be built logs the +// sign-in as FAILED with the PENDING state of a failed sign-in, and records no state in the install record. +func TestSignInInstalledURLFailureLogsFailedSignIn(t *testing.T) { + withHome(t) + ctx := env.Set(telemetry.WithNewLogger(t.Context()), "DATABRICKS_CLI_DISABLE_TELEMETRY", "") + w := &databricks.WorkspaceClient{Config: &config.Config{Host: "https://adb-1.test"}} + servers := []addServerOutput{{ + Service: "system.ai.github", SignInRequired: true, + Agents: []addAgentOutput{{Name: agents.NameClaudeCode, Status: statusInstalled}}, + }} + plans := []addPlan{{service: mcp.Service{FullName: "system.ai.github", Leaf: "github"}}} + + require.Error(t, signInInstalled(ctx, io.Discard, w, addRequest{}, plans, servers)) + assert.Empty(t, servers[0].SignIn.State) + events := telemetry.BufferedEvents(ctx) + require.Len(t, events, 1) + require.NotNil(t, events[0].AitoolsMcpEvent) + require.Len(t, events[0].AitoolsMcpEvent.Servers, 1) + assert.Equal(t, protos.AitoolsTargetOutcomeFailed, events[0].AitoolsMcpEvent.Servers[0].Outcome) + assert.Equal(t, protos.AitoolsMcpSignInStatePending, events[0].AitoolsMcpEvent.Servers[0].SignInState) +} + +// TestMcpEventOmitsUserContent checks that the event carries enums and counts only: a custom service's name must not +// appear anywhere in the payload. +func TestMcpEventOmitsUserContent(t *testing.T) { + event := protos.AitoolsMcpEvent{ + CallingAgent: protos.AitoolsAgentTypeClaudeCode, + Operation: protos.AitoolsMcpOperationAdd, + BoundProfileCount: 1, + Servers: addServerResults([]addServerOutput{{ + Service: "acme_catalog.secret_schema.internal_tool", + Agents: []addAgentOutput{{Name: "claude-code", Status: statusInstalled}}, + }}), + } + b, err := json.Marshal(event) + require.NoError(t, err) + + var got map[string]any + require.NoError(t, json.Unmarshal(b, &got)) + assert.ElementsMatch(t, []string{"calling_agent", "operation", "servers", "bound_profile_count"}, slices.Collect(maps.Keys(got))) + assert.NotContains(t, string(b), "acme_catalog") + assert.NotContains(t, string(b), "secret_schema") + assert.NotContains(t, string(b), "internal_tool") + assert.Contains(t, string(b), `"service":"CUSTOM"`) +} + +// TestAddSignInStateWithoutAState checks that a sign-in that failed before it read a state is PENDING, as a server that +// needs sign-in is until its state is known. A server that needs no sign-in is NOT_REQUIRED. +func TestAddSignInStateWithoutAState(t *testing.T) { + failed := addServerOutput{Service: "system.ai.github", SignInRequired: true, SignIn: &addSignInOutput{ErrorCategory: protos.AitoolsErrorCategoryUncategorized}} + assert.Equal(t, protos.AitoolsMcpSignInStatePending, addSignInState(failed)) + assert.Equal(t, protos.AitoolsMcpSignInStateNotRequired, addSignInState(addServerOutput{Service: "system.ai.dbsql"})) +} diff --git a/cmd/aitools/mcp_test.go b/cmd/aitools/mcp_test.go index 3c7b3e28505..a55926202be 100644 --- a/cmd/aitools/mcp_test.go +++ b/cmd/aitools/mcp_test.go @@ -109,7 +109,7 @@ host = https://b.test _, err := resolveProfile(ctx, "") require.Error(t, err) assert.Equal(t, "several workspace profiles exist; pass --profile. Found: a, b", err.Error()) - assert.Equal(t, categoryProfileRequired, errorCategory(err)) + assert.Equal(t, protos.AitoolsErrorCategoryProfileRequired, errorCategory(err)) }) t.Run("no workspace profiles is an error naming the file", func(t *testing.T) { @@ -117,7 +117,7 @@ host = https://b.test _, err := resolveProfile(ctx, "") require.Error(t, err) assert.Contains(t, err.Error(), "no workspace profiles in ") - assert.Equal(t, categoryNoProfile, errorCategory(err)) + assert.Equal(t, protos.AitoolsErrorCategoryNoProfile, errorCategory(err)) assert.Contains(t, err.Error(), "databricks auth login --host --profile ") }) } @@ -127,11 +127,11 @@ func TestErrorCategoryUsesTypedErrorsOnly(t *testing.T) { assert.Equal(t, protos.AitoolsErrorCategoryUnknownAgent, errorCategory(typed)) assert.Equal(t, "unknown agent \"pi\"", typed.Error()) - wrapped := withCategory(fmt.Errorf("%s: %w", "boom", errors.New("cause")), categoryServiceNotFound) - assert.Equal(t, categoryServiceNotFound, errorCategory(wrapped)) + wrapped := withCategory(fmt.Errorf("%s: %w", "boom", errors.New("cause")), protos.AitoolsErrorCategoryMCPServiceNotFound) + assert.Equal(t, protos.AitoolsErrorCategoryMCPServiceNotFound, errorCategory(wrapped)) configInvalid := &os.PathError{Op: "parse", Path: "x", Err: agentconfig.ErrConfigInvalid} - assert.Equal(t, categoryAgentConfigInvalid, errorCategory(configInvalid)) + assert.Equal(t, protos.AitoolsErrorCategoryAgentConfigInvalid, errorCategory(configInvalid)) assert.Equal(t, protos.AitoolsErrorCategoryUncategorized, errorCategory(errors.New("some failure"))) } diff --git a/cmd/aitools/telemetry.go b/cmd/aitools/telemetry.go index 25d9be40308..9bf5bbdc1ed 100644 --- a/cmd/aitools/telemetry.go +++ b/cmd/aitools/telemetry.go @@ -12,6 +12,7 @@ import ( "github.com/databricks/cli/libs/aitools/installer" "github.com/databricks/cli/libs/cmdctx" "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/log/handler" "github.com/databricks/cli/libs/telemetry" "github.com/databricks/cli/libs/telemetry/protos" "github.com/databricks/databricks-sdk-go/config" @@ -83,7 +84,7 @@ func startTelemetryAuth(cmd *cobra.Command, args []string) *pendingAuth { if telemetry.Disabled(ctx) { return nil } - sctx := log.NewContext(ctx, slog.New(debugOnlyHandler{log.GetLogger(ctx).Handler()})) + sctx := log.NewContext(ctx, slog.New(handler.DebugOnlyHandler{Handler: log.GetLogger(ctx).Handler()})) shadow := &cobra.Command{Use: cmd.Use} shadow.Flags().AddFlagSet(cmd.Flags()) shadow.SetContext(root.SkipTokenRefresh(root.SkipLoadBundle(root.SkipPrompt(sctx)))) @@ -160,31 +161,6 @@ func (p *pendingAuth) finished() bool { } } -// debugOnlyHandler logs records above Debug (e.g. hostmetadata's "failed to -// fetch host metadata" Warn) at Debug, so they show only with --debug. -type debugOnlyHandler struct{ slog.Handler } - -func debugOnly(level slog.Level) slog.Level { - return min(level, log.LevelDebug) -} - -func (h debugOnlyHandler) Enabled(ctx context.Context, level slog.Level) bool { - return h.Handler.Enabled(ctx, debugOnly(level)) -} - -func (h debugOnlyHandler) Handle(ctx context.Context, r slog.Record) error { - r.Level = debugOnly(r.Level) - return h.Handler.Handle(ctx, r) -} - -func (h debugOnlyHandler) WithAttrs(attrs []slog.Attr) slog.Handler { - return debugOnlyHandler{h.Handler.WithAttrs(attrs)} -} - -func (h debugOnlyHandler) WithGroup(name string) slog.Handler { - return debugOnlyHandler{h.Handler.WithGroup(name)} -} - // installOpts is the subset of install options telemetry records, kept narrow // so every field that leaves the machine is visible here. type installOpts struct { diff --git a/cmd/root/root.go b/cmd/root/root.go index a473167237d..aae96c707e1 100644 --- a/cmd/root/root.go +++ b/cmd/root/root.go @@ -227,24 +227,31 @@ Stack Trace: } } - commandStr := commandString(cmd) ctx = cmd.Context() - telemetryErr := telemetry.Upload(cmd.Context(), protos.ExecutionContext{ + telemetryErr := telemetry.Upload(cmd.Context(), TelemetryExecutionContext(cmd, startTime, exitCode)) + if telemetryErr != nil { + log.Infof(ctx, "telemetry upload failed: %s", telemetryErr) + } + + return err +} + +// TelemetryExecutionContext returns the execution context that a telemetry upload of cmd's run carries. start is when the +// run started, and exitCode is its exit status, 0 for a run that is still going. Execute uploads with it at exit, and a +// command that uploads before it exits uses it too. +func TelemetryExecutionContext(cmd *cobra.Command, start time.Time, exitCode int) protos.ExecutionContext { + ctx := cmd.Context() + return protos.ExecutionContext{ CmdExecID: cmdctx.ExecId(ctx), StateLineageHash: telemetry.StateLineageHash(ctx), Version: build.GetInfo().Version, - Command: commandStr, + Command: commandString(cmd), OperatingSystem: runtime.GOOS, DbrVersion: dbr.RuntimeVersion(ctx).String(), - ExecutionTimeMs: time.Since(startTime).Milliseconds(), + ExecutionTimeMs: time.Since(start).Milliseconds(), ExitCode: int64(exitCode), - }) - if telemetryErr != nil { - log.Infof(ctx, "telemetry upload failed: %s", telemetryErr) } - - return err } // This function is used to report an unknown subcommand. diff --git a/libs/aitools/mcp/binding.go b/libs/aitools/mcp/binding.go index f1111a9604f..4f34fb3b3c8 100644 --- a/libs/aitools/mcp/binding.go +++ b/libs/aitools/mcp/binding.go @@ -5,15 +5,19 @@ import ( "errors" "fmt" "io" + "log/slog" "net/http" "path/filepath" "strings" "time" "github.com/databricks/cli/libs/aitools/mcpconfig" + "github.com/databricks/cli/libs/auth" "github.com/databricks/cli/libs/databrickscfg" "github.com/databricks/cli/libs/databrickscfg/profile" "github.com/databricks/cli/libs/env" + "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/log/handler" "github.com/databricks/cli/libs/shellquote" "github.com/databricks/databricks-sdk-go/config" ) @@ -109,14 +113,51 @@ func BoundConfig(profileName, configFile string) (*config.Config, error) { // budget of zero keeps the SDK's defaults. Resolving the profile runs on a context of its own, so a context deadline does // not reach it, and these budgets are what bound it (see profileMetadata.Load in cmd/auth). func BoundConfigWithin(profileName, configFile string, budget time.Duration) (*config.Config, error) { + return resolveBound(withinBudget(profileName, configFile, budget)) +} + +// BoundUploadConfig is BoundConfigWithin for a config that only a telemetry upload uses. Its credentials never refresh a +// token, since the command may exit before a refresh finishes, and a rotated single-use refresh token would be lost (see +// auth.CredentialsWithoutRefresh). It does not authenticate. +func BoundUploadConfig(ctx context.Context, profileName, configFile string, budget time.Duration) (*config.Config, error) { + cfg := withinBudget(profileName, configFile, budget) + cfg.Credentials = auth.CredentialsWithoutRefresh() + quietHostMetadata(ctx, cfg) + return resolveBound(cfg) +} + +// quietHostMetadata makes a failed host metadata lookup of cfg a debug message, so that a command which succeeded does not +// warn about its telemetry upload. The lookup runs with a debug-only logger of ctx, since the SDK's background context +// would send the registered resolver's warning to the global logger. A failed lookup falls back to the user config, as +// the SDK does. +func quietHostMetadata(ctx context.Context, cfg *config.Config) { + resolve := cfg.DefaultHostMetadataResolver() + if factory := config.DefaultHostMetadataResolverFactory; factory != nil { + if r := factory(cfg); r != nil { + resolve = r + } + } + logger := slog.New(handler.DebugOnlyHandler{Handler: log.GetLogger(ctx).Handler()}) + cfg.HostMetadataResolver = func(sdkCtx context.Context, host string) (*config.HostMetadata, error) { + meta, err := resolve(log.NewContext(sdkCtx, logger), host) + if err != nil { + log.Debugf(sdkCtx, "telemetry: cannot resolve host metadata for %s: %s", host, err) + return nil, nil + } + return meta, nil + } +} + +// withinBudget is the config of profileName in configFile, with the SDK's request and retry budgets set to budget. +func withinBudget(profileName, configFile string, budget time.Duration) *config.Config { seconds := int((budget + time.Second - 1) / time.Second) - return resolveBound(&config.Config{ + return &config.Config{ Profile: profileName, ConfigFile: configFile, Loaders: []config.Loader{databrickscfg.ResolveNonAuthFromEnv, config.ConfigFile}, HTTPTimeoutSeconds: seconds, RetryTimeoutSeconds: seconds, - }) + } } // resolveBound resolves cfg, as BoundConfig does. diff --git a/libs/aitools/mcp/binding_test.go b/libs/aitools/mcp/binding_test.go index a061f1e7513..b01ff28d8ad 100644 --- a/libs/aitools/mcp/binding_test.go +++ b/libs/aitools/mcp/binding_test.go @@ -10,8 +10,10 @@ import ( "os" "path/filepath" "testing" + "time" "github.com/databricks/cli/libs/aitools/mcp" + "github.com/databricks/cli/libs/auth" "github.com/databricks/cli/libs/env" "github.com/databricks/cli/libs/shellquote" "github.com/databricks/databricks-sdk-go/config" @@ -986,3 +988,17 @@ func TestConfigFileCommandUsesForwardSlashes(t *testing.T) { assert.Equal(t, "DATABRICKS_CONFIG_FILE="+filepath.ToSlash(path)+" databricks auth login --profile logfood", mcp.ConfigFileCommand(path, "databricks auth login --profile logfood")) } + +// TestBoundUploadConfigNeverRefreshes checks that the config of a telemetry upload never refreshes a token, and that the +// config a serve session binds keeps the default chain. +func TestBoundUploadConfigNeverRefreshes(t *testing.T) { + cfgPath := bindingTestConfig(t, bindingTestProfileContents(bindingTestToken)) + + upload, err := mcp.BoundUploadConfig(t.Context(), bindingTestProfile, cfgPath, time.Second) + require.NoError(t, err) + assert.Equal(t, auth.CredentialsWithoutRefresh(), upload.Credentials) + + session, err := mcp.BoundConfigWithin(bindingTestProfile, cfgPath, time.Second) + require.NoError(t, err) + assert.Nil(t, session.Credentials, "the session leaves Credentials unset, so the SDK keeps the default chain") +} diff --git a/libs/aitools/mcp/known.go b/libs/aitools/mcp/known.go index 867d2b0ff36..10bf6294947 100644 --- a/libs/aitools/mcp/known.go +++ b/libs/aitools/mcp/known.go @@ -1,6 +1,10 @@ package mcp -import "strings" +import ( + "maps" + "slices" + "strings" +) // SystemAILocation is the schema that ships the Databricks managed MCP services. const SystemAILocation = "system.ai" @@ -61,6 +65,11 @@ var knownSystemAI = map[string]KnownService{ }, } +// KnownLeaves returns the leaf names of the system.ai services in the CLI's table, sorted. +func KnownLeaves() []string { + return slices.Sorted(maps.Keys(knownSystemAI)) +} + // KnownOf returns the CLI's description of a service, or nil when it is not a system.ai service in the table. The leaf // is compared without case, as Unity Catalog names are. func KnownOf(fullName string) *KnownService { diff --git a/libs/aitools/mcp/known_test.go b/libs/aitools/mcp/known_test.go index 9f8face3bc0..33b13ad0183 100644 --- a/libs/aitools/mcp/known_test.go +++ b/libs/aitools/mcp/known_test.go @@ -1,6 +1,7 @@ package mcp import ( + "slices" "testing" "github.com/stretchr/testify/assert" @@ -40,6 +41,18 @@ func TestKnownSystemAITable(t *testing.T) { } } +// TestKnownLeavesAreTheSystemAITable checks that KnownLeaves lists the leaves of the system.ai table, sorted, with each +// leaf known as a system.ai service. +func TestKnownLeavesAreTheSystemAITable(t *testing.T) { + leaves := KnownLeaves() + assert.Contains(t, leaves, "genie_one_mcp") + assert.True(t, slices.IsSorted(leaves)) + assert.Len(t, leaves, len(knownSystemAI)) + for _, leaf := range leaves { + assert.NotNil(t, KnownOf(SystemAILocation+"."+leaf), leaf) + } +} + func TestProviderLabel(t *testing.T) { assert.Equal(t, "GitHub", ProviderLabel("system.ai.github")) assert.Equal(t, "Genie One", ProviderLabel("SYSTEM.AI.genie_one_mcp")) diff --git a/libs/aitools/mcp/proxy/bridge_test.go b/libs/aitools/mcp/proxy/bridge_test.go index 741f7d91e13..41b90af6116 100644 --- a/libs/aitools/mcp/proxy/bridge_test.go +++ b/libs/aitools/mcp/proxy/bridge_test.go @@ -2425,7 +2425,7 @@ func TestSignInWritesShareTheBridgeStderr(t *testing.T) { Config: &config.Config{Host: testHost}, Authenticate: func(*http.Request) error { return nil }, Profile: "logfood", - Challenge: &Challenge{Service: testService, Profile: "logfood", Host: testHost, Label: "GitHub", LoginCommand: loginCommand, SignIn: signIn}, + Challenge: &Challenge{Service: testService, Profile: "logfood", Host: testHost, Label: "GitHub", LoginCommand: loginCommand, SignIn: signIn, Recorder: &promptRecorder{}}, }) }() diff --git a/libs/aitools/mcp/proxy/challenge.go b/libs/aitools/mcp/proxy/challenge.go index c9e2313564b..5284bb60211 100644 --- a/libs/aitools/mcp/proxy/challenge.go +++ b/libs/aitools/mcp/proxy/challenge.go @@ -50,6 +50,15 @@ type SignInRequest struct { // cancelled. Its error is the answer's text. type SignInFunc func(ctx context.Context, req SignInRequest) (alreadySignedIn, urlShown bool, err error) +// Recorder receives the proxy's telemetry events. The command implements it, so the proxy does not +// build telemetry events itself. +type Recorder interface { + // SignInPrompted runs after the wait of an authenticate call that showed the sign-in page: one it opened, or one an + // earlier cancelled call opened, which the user may still have open. signedIn reports whether the credential became + // ACTIVE. A retry after a cancelled call reports its own outcome. + SignInPrompted(ctx context.Context, signedIn bool) +} + // Challenge turns on sign-in handling for a service that needs a provider sign-in. type Challenge struct { // Service is the full name of the MCP service, for example system.ai.github. @@ -73,6 +82,8 @@ type Challenge struct { // SignIn runs the sign-in. The command provides it, so the proxy never polls or opens a browser itself. Its error is // the answer's text, so it names the service, the profile and the fix. SignIn SignInFunc + // Recorder logs the sign-in prompt. The command provides it; it must not be nil. + Recorder Recorder } // signInHooks applies the sign-in rules to the bridge. Services without a challenge get no hooks. @@ -506,13 +517,19 @@ func (h *signInHooks) authenticate(ctx context.Context, id, progressToken json.R return nil } + open := !h.pageCancelled alreadySignedIn, urlShown, err := h.c.SignIn(ctx, SignInRequest{ - Open: !h.pageCancelled, + Open: open, Progress: func(pageURL string) { p.report(ctx, pageURL) }, Stderr: h.stderr, }) + // A prompt is a page the user was shown. A caller that was already signed in, or a sign-in that failed before its + // URL was printed, prompted nobody. + if !alreadySignedIn && urlShown { + h.c.Recorder.SignInPrompted(ctx, err == nil) + } h.pageCancelled = urlShown && err != nil && ctx.Err() != nil h.stateMu.Lock() h.alreadySignedIn = err == nil && alreadySignedIn diff --git a/libs/aitools/mcp/proxy/challenge_test.go b/libs/aitools/mcp/proxy/challenge_test.go index fa7784357b5..3c9fe2c019f 100644 --- a/libs/aitools/mcp/proxy/challenge_test.go +++ b/libs/aitools/mcp/proxy/challenge_test.go @@ -68,6 +68,24 @@ func (s *signInStub) requests() []SignInRequest { // loginCommand is the hint that the command builds for a pinned config file and a profile name with a space. const loginCommand = `DATABRICKS_CONFIG_FILE='/work/cfg/my file.cfg' databricks aitools mcp login system.ai.github --profile 'my profile'` +// promptRecorder records the SIGN_IN_PROMPTED events: one per sign-in page that authenticate opened. +type promptRecorder struct { + mu sync.Mutex + signedIn []bool +} + +func (r *promptRecorder) SignInPrompted(_ context.Context, signedIn bool) { + r.mu.Lock() + defer r.mu.Unlock() + r.signedIn = append(r.signedIn, signedIn) +} + +func (r *promptRecorder) prompts() []bool { + r.mu.Lock() + defer r.mu.Unlock() + return append([]bool(nil), r.signedIn...) +} + // challengedBridge returns a bridge to url with the sign-in hooks of a service that needs sign-in. func challengedBridge(url string, signIn SignInFunc) *Bridge { b := bridgeFor(url) @@ -255,6 +273,109 @@ func TestAuthenticateSaysAlreadySignedInAfterSignIn(t *testing.T) { require.NoError(t, p.finish(t)) } +// TestAuthenticateRecordsPromptWhenPageOpens checks that the prompt is recorded once per page that was shown, after its +// wait, with whether the credential became ACTIVE. A call that finds the caller already signed in records nothing. +func TestAuthenticateRecordsPromptWhenPageOpens(t *testing.T) { + srv, _ := challengedService(t) + signIn := &signInStub{reply: func(call int) (bool, error) { + if call == 1 { + return false, errors.New("sign-in did not complete") + } + return call == 3, nil + }} + recorder := &promptRecorder{} + c := testChallenge(signIn.SignIn) + c.Recorder = recorder + b := bridgeFor(srv.URL) + b.Hooks = newSignInHooks(c, io.Discard) + p := startPiped(t, t.Context(), b) + authenticate := `{"jsonrpc":"2.0","id":%d,"method":"tools/call","params":{"name":"authenticate","arguments":{}}}` + challenge(t, p) + + p.send(t, fmt.Sprintf(authenticate, 1)) + p.waitLines(t, 2) + p.send(t, fmt.Sprintf(authenticate, 2)) + p.waitLines(t, 4) + p.send(t, fmt.Sprintf(authenticate, 3)) + p.waitLines(t, 6) + + assert.Equal(t, []bool{true, true, true}, signIn.calls(), "every call opens the page") + assert.Equal(t, []bool{false, true}, recorder.prompts(), "call 3 was already signed in and records nothing") + require.NoError(t, p.finish(t)) +} + +// TestAuthenticateRecordsNoPromptWhenNoURLShown checks that a sign-in that fails before its URL is printed, such as a read +// of the workspace ID, shows no page, so nothing is recorded. +func TestAuthenticateRecordsNoPromptWhenNoURLShown(t *testing.T) { + srv, _ := challengedService(t) + var calls int + signIn := func(context.Context, SignInRequest) (bool, bool, error) { + calls++ + return false, false, errors.New("cannot read the workspace ID") + } + recorder := &promptRecorder{} + c := testChallenge(signIn) + c.Recorder = recorder + b := bridgeFor(srv.URL) + b.Hooks = newSignInHooks(c, io.Discard) + p := startPiped(t, t.Context(), b) + challenge(t, p) + + p.send(t, `{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"authenticate","arguments":{}}}`) + p.waitLines(t, 2) + + assert.Equal(t, 1, calls, "the sign-in ran") + assert.Empty(t, recorder.prompts(), "no page was shown, so no prompt is recorded") + require.NoError(t, p.finish(t)) +} + +// TestAuthenticateRecordsPromptOfCallThatWaitsOnCancelledPage checks that a call which waits on the page an earlier, +// cancelled call showed records its own outcome: the page was shown for it too, so a sign-in that completes there is +// recorded as ACTIVE, not left PENDING from the cancelled call. +func TestAuthenticateRecordsPromptOfCallThatWaitsOnCancelledPage(t *testing.T) { + srv, _ := newFakeEndpoint(t, answerWith(http.StatusOK, listChallenge)) + entered, done := make(chan struct{}), make(chan struct{}) + var mu sync.Mutex + var opens []bool + signIn := func(ctx context.Context, req SignInRequest) (bool, bool, error) { + mu.Lock() + opens = append(opens, req.Open) + first := len(opens) == 1 + mu.Unlock() + if !first { + return false, true, nil + } + close(entered) + <-ctx.Done() + close(done) + return false, true, ctx.Err() + } + recorder := &promptRecorder{} + c := testChallenge(signIn) + c.Recorder = recorder + b := bridgeFor(srv.URL) + b.Hooks = newSignInHooks(c, io.Discard) + p := startPiped(t, t.Context(), b) + authenticate := `{"jsonrpc":"2.0","id":%d,"method":"tools/call","params":{"name":"authenticate","arguments":{}}}` + + p.send(t, `{"jsonrpc":"2.0","id":2,"method":"tools/list"}`) + p.waitLines(t, 1) + p.send(t, fmt.Sprintf(authenticate, 5)) + waitClosed(t, entered, "the first sign-in to start") + p.send(t, `{"jsonrpc":"2.0","method":"notifications/cancelled","params":{"requestId":5}}`) + waitClosed(t, done, "the first sign-in to return") + p.send(t, fmt.Sprintf(authenticate, 6)) + require.Eventually(t, func() bool { + return len(recorder.prompts()) == 2 + }, 10*time.Second, time.Millisecond) + require.NoError(t, p.finish(t)) + + mu.Lock() + defer mu.Unlock() + assert.Equal(t, []bool{true, false}, opens, "the second call does not open the page again") + assert.Equal(t, []bool{false, true}, recorder.prompts(), "the cancelled call is PENDING, and the second call is ACTIVE") +} + func TestAuthenticateReopensPageAfterTimeout(t *testing.T) { srv, _ := challengedService(t) signIn := &signInStub{reply: func(call int) (bool, error) { @@ -343,7 +464,7 @@ func TestChallengeHintsPrintTheLoginCommand(t *testing.T) { return false, true, errors.New("sign-in to system.ai.github did not complete in 5 minutes. Finish at " + testURL + " then retry: " + loginCommand) } b := bridgeFor(srv.URL) - b.Hooks = newSignInHooks(&Challenge{Service: testService, Profile: "my profile", Host: testHost, Label: "GitHub", LoginCommand: loginCommand, SignIn: signIn}, io.Discard) + b.Hooks = newSignInHooks(&Challenge{Service: testService, Profile: "my profile", Host: testHost, Label: "GitHub", LoginCommand: loginCommand, SignIn: signIn, Recorder: &promptRecorder{}}, io.Discard) p := startPiped(t, t.Context(), b) p.send(t, `{"jsonrpc":"2.0","id":3,"method":"tools/call","params":{"name":"echo","arguments":{}}}`) @@ -487,7 +608,7 @@ func TestChallengeMatchesServiceNameWithoutCase(t *testing.T) { `{"mode":"url","url":"` + testURL + `"}]}}}` srv, _ := newFakeEndpoint(t, answerWith(http.StatusOK, body)) b := bridgeFor(srv.URL) - b.Hooks = newSignInHooks(&Challenge{Service: "SYSTEM.AI.GITHUB", Profile: "logfood", Host: testHost, Label: "GitHub", LoginCommand: loginCommand}, io.Discard) + b.Hooks = newSignInHooks(&Challenge{Service: "SYSTEM.AI.GITHUB", Profile: "logfood", Host: testHost, Label: "GitHub", LoginCommand: loginCommand, Recorder: &promptRecorder{}}, io.Discard) out, _ := runAll(t, b, `{"jsonrpc":"2.0","id":2,"method":"tools/list"}`+"\n") require.Len(t, out, 1) @@ -941,7 +1062,7 @@ func TestAuthFailureNamesPinnedConfigFile(t *testing.T) { http.Error(w, "unauthorized", http.StatusUnauthorized) }) b := bridgeFor(srv.URL) - b.Hooks = newSignInHooks(&Challenge{Service: testService, Profile: "logfood", Host: testHost, Label: "GitHub", LoginCommand: loginCommand, PinnedFile: "/work/cfg/my file.cfg"}, io.Discard) + b.Hooks = newSignInHooks(&Challenge{Service: testService, Profile: "logfood", Host: testHost, Label: "GitHub", LoginCommand: loginCommand, PinnedFile: "/work/cfg/my file.cfg", Recorder: &promptRecorder{}}, io.Discard) out, _ := runAll(t, b, `{"jsonrpc":"2.0","id":2,"method":"tools/list"}`+"\n") require.Len(t, out, 1) @@ -1011,6 +1132,7 @@ func testChallenge(signIn SignInFunc) *Challenge { LoginCommand: "databricks aitools mcp login system.ai.github --profile logfood", LogoutCommand: "databricks aitools mcp logout system.ai.github --profile logfood", SignIn: signIn, + Recorder: &promptRecorder{}, } } diff --git a/libs/aitools/mcp/proxy/run_test.go b/libs/aitools/mcp/proxy/run_test.go index 8a8f195027d..f4020a9d3a1 100644 --- a/libs/aitools/mcp/proxy/run_test.go +++ b/libs/aitools/mcp/proxy/run_test.go @@ -100,7 +100,7 @@ func TestRunAdvertisesListChangedOnlyForSignInServices(t *testing.T) { assert.Contains(t, out.String(), `"listChanged":false`) opts, out, _ = runOptions(t, srv.URL, cfg, initialize) - opts.Challenge = &Challenge{Service: testService, Profile: "logfood", Host: testHost, Label: "GitHub"} + opts.Challenge = &Challenge{Service: testService, Profile: "logfood", Host: testHost, Label: "GitHub", Recorder: &promptRecorder{}} require.NoError(t, Run(t.Context(), opts)) assert.Contains(t, out.String(), `"listChanged":true`) } diff --git a/libs/log/handler/debug_only.go b/libs/log/handler/debug_only.go new file mode 100644 index 00000000000..364ad5a34ec --- /dev/null +++ b/libs/log/handler/debug_only.go @@ -0,0 +1,33 @@ +package handler + +import ( + "context" + "log/slog" + + "github.com/databricks/cli/libs/log" +) + +// DebugOnlyHandler logs records above Debug (e.g. hostmetadata's "failed to +// fetch host metadata" Warn) at Debug, so they show only with --debug. +type DebugOnlyHandler struct{ slog.Handler } + +func debugOnly(level slog.Level) slog.Level { + return min(level, log.LevelDebug) +} + +func (h DebugOnlyHandler) Enabled(ctx context.Context, level slog.Level) bool { + return h.Handler.Enabled(ctx, debugOnly(level)) +} + +func (h DebugOnlyHandler) Handle(ctx context.Context, r slog.Record) error { + r.Level = debugOnly(r.Level) + return h.Handler.Handle(ctx, r) +} + +func (h DebugOnlyHandler) WithAttrs(attrs []slog.Attr) slog.Handler { + return DebugOnlyHandler{h.Handler.WithAttrs(attrs)} +} + +func (h DebugOnlyHandler) WithGroup(name string) slog.Handler { + return DebugOnlyHandler{h.Handler.WithGroup(name)} +} diff --git a/libs/log/handler/debug_only_test.go b/libs/log/handler/debug_only_test.go new file mode 100644 index 00000000000..3a313997d82 --- /dev/null +++ b/libs/log/handler/debug_only_test.go @@ -0,0 +1,29 @@ +package handler + +import ( + "bytes" + "log/slog" + "testing" + + "github.com/databricks/cli/libs/log" + "github.com/stretchr/testify/assert" +) + +func TestDebugOnlyHandlerLogsWarningsAtDebug(t *testing.T) { + var buf bytes.Buffer + base := slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: log.LevelDebug}) + logger := slog.New(DebugOnlyHandler{Handler: base}.WithAttrs([]slog.Attr{slog.String("k", "v")}).(DebugOnlyHandler).WithGroup("g")) + + logger.Warn("failed to fetch") + assert.Contains(t, buf.String(), "level=DEBUG") + assert.NotContains(t, buf.String(), "level=WARN") +} + +func TestDebugOnlyHandlerEnabledClampsLevel(t *testing.T) { + debugBase := slog.NewTextHandler(&bytes.Buffer{}, &slog.HandlerOptions{Level: log.LevelDebug}) + assert.True(t, DebugOnlyHandler{Handler: debugBase}.Enabled(t.Context(), slog.LevelWarn)) + + // A warning is still filtered when the base handler's level is above Debug. + errorBase := slog.NewTextHandler(&bytes.Buffer{}, &slog.HandlerOptions{Level: slog.LevelError}) + assert.False(t, DebugOnlyHandler{Handler: errorBase}.Enabled(t.Context(), slog.LevelWarn)) +} diff --git a/libs/telemetry/logger.go b/libs/telemetry/logger.go index 3a4f6d7edf6..54c72c7c624 100644 --- a/libs/telemetry/logger.go +++ b/libs/telemetry/logger.go @@ -7,7 +7,9 @@ import ( "fmt" "net/http" "net/http/httptrace" + "slices" "strconv" + "sync" "sync/atomic" "time" "uuid" @@ -38,6 +40,40 @@ func Disabled(ctx context.Context) bool { return env.Get(ctx, disableEnvVar) != "" } +// SetUploadConfig sets the config that Upload authenticates with, in place of the one cmdctx.ConfigUsed holds. A command +// sets it once its own binding checks have passed, so that the upload authenticates with the profile the command bound. +// Like Log, it is a no-op without a logger. +func SetUploadConfig(ctx context.Context, cfg *config.Config) { + if l, ok := loggerFromContext(ctx); ok { + l.upload = cfg + } +} + +// SkipUpload makes Upload send nothing for the run, whatever config the run has, including the one cmdctx.ConfigUsed +// holds. A command calls it when the profile it bound no longer matches, so that no config may stand in for it. Like Log, +// it is a no-op without a logger. +func SkipUpload(ctx context.Context) { + if l, ok := loggerFromContext(ctx); ok { + l.skipUpload = true + } +} + +// UploadConfig returns the config that Upload authenticates with: the one SetUploadConfig set, else the one +// cmdctx.ConfigUsed holds. It returns false when the run has no config, or when SkipUpload was called. +func UploadConfig(ctx context.Context) (*config.Config, bool) { + l, ok := loggerFromContext(ctx) + if !ok || l.skipUpload { + return nil, false + } + if l.upload != nil { + return l.upload, true + } + if cmdctx.HasConfigUsed(ctx) { + return cmdctx.ConfigUsed(ctx), true + } + return nil, false +} + func Log(ctx context.Context, event protos.DatabricksCliLog) { // A missing logger means telemetry was never initialized on this context // (e.g. a command invoked outside the normal cmd/root setup). Dropping the @@ -52,9 +88,17 @@ func Log(ctx context.Context, event protos.DatabricksCliLog) { } type logger struct { + // mu guards logs, which the goroutines of a command's MCP proxy write concurrently. + mu sync.Mutex logs []protos.FrontendLog stateLineageHash string + // upload is the config Upload authenticates with, as SetUploadConfig set it. Nil leaves cmdctx.ConfigUsed to it. + upload *config.Config + + // skipUpload is set by SkipUpload. It takes precedence over upload and cmdctx.ConfigUsed. + skipUpload bool + // running counts the goroutines that Upload started and stopped waiting // for, which may still be using the config (see within). running atomic.Int32 @@ -77,6 +121,8 @@ func StateLineageHash(ctx context.Context) string { } func (l *logger) log(event protos.DatabricksCliLog) { + l.mu.Lock() + defer l.mu.Unlock() l.logs = append(l.logs, protos.FrontendLog{ FrontendLogEventID: uuid.New().String(), Entry: protos.FrontendLogEntry{ @@ -85,6 +131,20 @@ func (l *logger) log(event protos.DatabricksCliLog) { }) } +// forget removes the sent events from the buffer, so that a later Upload does not send them again. It matches them by +// ID, since a proxy goroutine may have logged more events after the copy Upload sent was taken. +func (l *logger) forget(sent []protos.FrontendLog) { + ids := make(map[string]bool, len(sent)) + for _, s := range sent { + ids[s.FrontendLogEventID] = true + } + l.mu.Lock() + defer l.mu.Unlock() + l.logs = slices.DeleteFunc(l.logs, func(e protos.FrontendLog) bool { + return ids[e.FrontendLogEventID] + }) +} + const ( defaultUploadTimeout = 3 * time.Second @@ -97,9 +157,15 @@ const ( waitBetweenRetries = 200 * time.Millisecond ) +// Upload sends the events logged on ctx. Events the service accepts are removed from the buffer, so a later Upload sends +// only the events logged since. func Upload(ctx context.Context, ec protos.ExecutionContext) error { l := fromContext(ctx) - if len(l.logs) == 0 { + // Upload works on a copy, so that setting the execution context does not write the events a proxy goroutine may append to. + l.mu.Lock() + logs := slices.Clone(l.logs) + l.mu.Unlock() + if len(logs) == 0 { log.Debugf(ctx, "no telemetry logs to upload") return nil } @@ -111,16 +177,17 @@ func Upload(ctx context.Context, ec protos.ExecutionContext) error { } // Set the execution context for all logs. - for i := range l.logs { - l.logs[i].Entry.DatabricksCliLog.ExecutionContext = &ec + for i := range logs { + logs[i].Entry.DatabricksCliLog.ExecutionContext = &ec } - protoLogs, err := marshalLogs(l.logs) + protoLogs, err := marshalLogs(logs) if err != nil { return err } - if !cmdctx.HasConfigUsed(ctx) { + cfg, ok := UploadConfig(ctx) + if !ok { log.Debugf(ctx, "no auth config available; skipping telemetry upload") return nil } @@ -132,7 +199,7 @@ func Upload(ctx context.Context, ec protos.ExecutionContext) error { return errors.New("a previous telemetry upload is still running") } - timeout, err := uploadTimeout(ctx, l.logs) + timeout, err := uploadTimeout(ctx, logs) if err != nil { return err } @@ -147,7 +214,7 @@ func Upload(ctx context.Context, ec protos.ExecutionContext) error { } apiClient, err := within(ctx, &l.running, func() (*client.DatabricksClient, error) { - return newAPIClient(cmdctx.ConfigUsed(ctx), &l.running, true) + return newAPIClient(cfg, &l.running, true) }) if err != nil { return fmt.Errorf("failed to create API client: %w", err) @@ -167,6 +234,7 @@ func Upload(ctx context.Context, ec protos.ExecutionContext) error { // All logs were uploaded successfully. if err == nil && resp.NumProtoSuccess >= int64(len(protoLogs)) { log.Debugf(ctx, "All %d logs uploaded successfully", len(protoLogs)) + l.forget(logs) return nil } @@ -427,15 +495,17 @@ func attempt(ctx context.Context, apiClient *client.DatabricksClient, protoLogs return resp, nil } -// BufferedEvents returns the events logged on ctx, or nil when ctx has no -// logger. Upload does not clear them. Tests in other packages use it to assert -// what a command logs without uploading; cmd/aitools uses it to skip waiting -// for telemetry auth when there is nothing to upload. +// BufferedEvents returns the events logged on ctx and not yet uploaded, or nil +// when ctx has no logger. Tests in other packages use it to assert what a +// command logs without uploading; cmd/aitools uses it to skip waiting for +// telemetry auth when there is nothing to upload. func BufferedEvents(ctx context.Context) []protos.DatabricksCliLog { l, ok := loggerFromContext(ctx) if !ok { return nil } + l.mu.Lock() + defer l.mu.Unlock() out := make([]protos.DatabricksCliLog, len(l.logs)) for i, fl := range l.logs { out[i] = fl.Entry.DatabricksCliLog diff --git a/libs/telemetry/logger_test.go b/libs/telemetry/logger_test.go index aeae9584ff8..af85010c3bf 100644 --- a/libs/telemetry/logger_test.go +++ b/libs/telemetry/logger_test.go @@ -639,3 +639,110 @@ func TestSendTimeoutBoundsTokenAcquisition(t *testing.T) { t.Fatal("Send did not return after its context deadline") } } + +// TestUploadUsesConfigSetByCommand checks that the upload authenticates with the config the command set, in place of the +// one cmdctx.ConfigUsed holds. +func TestUploadUsesConfigSetByCommand(t *testing.T) { + bound := testserver.New(t) + t.Cleanup(bound.Close) + other := testserver.New(t) + t.Cleanup(other.Close) + var boundCalls, otherCalls int + bound.Handle("POST", "/telemetry-ext", func(req testserver.Request) any { + boundCalls++ + return ResponseBody{NumProtoSuccess: 1} + }) + other.Handle("POST", "/telemetry-ext", func(req testserver.Request) any { + otherCalls++ + return ResponseBody{NumProtoSuccess: 1} + }) + + ctx := WithNewLogger(t.Context()) + Log(ctx, protos.DatabricksCliLog{CliTestEvent: &protos.CliTestEvent{Name: protos.DummyCliEnumValue1}}) + ctx = cmdctx.SetConfigUsed(ctx, &config.Config{Host: other.URL, Token: "other"}) + SetUploadConfig(ctx, &config.Config{Host: bound.URL, Token: "bound"}) + + cfg, ok := UploadConfig(ctx) + require.True(t, ok) + assert.Equal(t, bound.URL, cfg.Host) + + require.NoError(t, Upload(ctx, protos.ExecutionContext{})) + assert.Equal(t, 1, boundCalls) + assert.Zero(t, otherCalls) +} + +// TestUploadConfigWithoutLogger checks that the upload slot is a no-op without a logger, as Log is. +func TestUploadConfigWithoutLogger(t *testing.T) { + ctx := t.Context() + SetUploadConfig(ctx, &config.Config{Host: "https://ws.test"}) + _, ok := UploadConfig(ctx) + assert.False(t, ok) +} + +// TestUploadSkippedDespiteConfigUsed checks that a skipped upload sends nothing, even when the run has a config from +// cmdctx.ConfigUsed and one set by the command before it skipped. +func TestUploadSkippedDespiteConfigUsed(t *testing.T) { + bound := testserver.New(t) + t.Cleanup(bound.Close) + used := testserver.New(t) + t.Cleanup(used.Close) + var calls int + for _, server := range []*testserver.Server{bound, used} { + server.Handle("POST", "/telemetry-ext", func(req testserver.Request) any { + calls++ + return ResponseBody{NumProtoSuccess: 1} + }) + } + + ctx := WithNewLogger(t.Context()) + Log(ctx, protos.DatabricksCliLog{CliTestEvent: &protos.CliTestEvent{Name: protos.DummyCliEnumValue1}}) + ctx = cmdctx.SetConfigUsed(ctx, &config.Config{Host: used.URL, Token: "used"}) + SetUploadConfig(ctx, &config.Config{Host: bound.URL, Token: "bound"}) + SkipUpload(ctx) + + _, ok := UploadConfig(ctx) + assert.False(t, ok) + require.NoError(t, Upload(ctx, protos.ExecutionContext{})) + assert.Zero(t, calls) +} + +// TestUploadSendsEachAcceptedEventOnce checks that an upload sends only the events logged since the earlier upload was +// accepted, so an event is not uploaded twice. +func TestUploadSendsEachAcceptedEventOnce(t *testing.T) { + server := testserver.New(t) + t.Cleanup(server.Close) + var uploaded []int + server.Handle("POST", "/telemetry-ext", func(req testserver.Request) any { + var body RequestBody + assert.NoError(t, json.Unmarshal(req.Body, &body)) + uploaded = append(uploaded, len(body.ProtoLogs)) + return ResponseBody{NumProtoSuccess: int64(len(body.ProtoLogs))} + }) + + ctx := WithNewLogger(t.Context()) + ctx = cmdctx.SetConfigUsed(ctx, &config.Config{Host: server.URL, Token: "token"}) + Log(ctx, protos.DatabricksCliLog{CliTestEvent: &protos.CliTestEvent{Name: protos.DummyCliEnumValue1}}) + require.NoError(t, Upload(ctx, protos.ExecutionContext{})) + Log(ctx, protos.DatabricksCliLog{CliTestEvent: &protos.CliTestEvent{Name: protos.DummyCliEnumValue2}}) + require.NoError(t, Upload(ctx, protos.ExecutionContext{})) + + assert.Equal(t, []int{1, 1}, uploaded) + assert.Empty(t, BufferedEvents(ctx)) +} + +// TestUploadKeepsEventsNotAccepted checks that an upload the service does not accept leaves its events buffered, for a +// later upload to send. +func TestUploadKeepsEventsNotAccepted(t *testing.T) { + server := testserver.New(t) + t.Cleanup(server.Close) + server.Handle("POST", "/telemetry-ext", func(req testserver.Request) any { + return ResponseBody{NumProtoSuccess: 0} + }) + + ctx := WithNewLogger(t.Context()) + ctx = cmdctx.SetConfigUsed(ctx, &config.Config{Host: server.URL, Token: "token"}) + Log(ctx, protos.DatabricksCliLog{CliTestEvent: &protos.CliTestEvent{Name: protos.DummyCliEnumValue1}}) + require.Error(t, Upload(ctx, protos.ExecutionContext{})) + + assert.Len(t, BufferedEvents(ctx), 1) +} diff --git a/libs/telemetry/protos/aitools_install.go b/libs/telemetry/protos/aitools_install.go index 051076f6241..34dd41c8cd8 100644 --- a/libs/telemetry/protos/aitools_install.go +++ b/libs/telemetry/protos/aitools_install.go @@ -74,6 +74,25 @@ const ( // The command's context was cancelled. SIGINT outside a prompt is not // counted here: it ends the process before telemetry is uploaded. AitoolsErrorCategoryCommandInterrupted AitoolsErrorCategory = "COMMAND_INTERRUPTED" + + // `aitools mcp`: profiles and sign-in. Bind failures send their category by value (mcp.BindCategory). PROFILE_NOT_FOUND + // and PROFILE_HOST_CHANGED mirror the schema, and no CLI command names them. + AitoolsErrorCategoryNoProfile AitoolsErrorCategory = "NO_PROFILE" + AitoolsErrorCategoryProfileRequired AitoolsErrorCategory = "PROFILE_REQUIRED" + AitoolsErrorCategoryProfileNotFound AitoolsErrorCategory = "PROFILE_NOT_FOUND" + AitoolsErrorCategoryProfileHostChanged AitoolsErrorCategory = "PROFILE_HOST_CHANGED" + AitoolsErrorCategoryAuthFailed AitoolsErrorCategory = "AUTH_FAILED" + AitoolsErrorCategorySignInRequired AitoolsErrorCategory = "SIGN_IN_REQUIRED" + AitoolsErrorCategorySignInTimeout AitoolsErrorCategory = "SIGN_IN_TIMEOUT" + + // `aitools mcp`: the service list, agent configs and the proxy. MCP_UPSTREAM_ERROR mirrors the schema, and no CLI + // command sends it yet: the proxy does not log upstream errors in v1. + AitoolsErrorCategoryMCPServiceNotFound AitoolsErrorCategory = "MCP_SERVICE_NOT_FOUND" + AitoolsErrorCategoryMCPPermissionDenied AitoolsErrorCategory = "MCP_PERMISSION_DENIED" + AitoolsErrorCategoryMCPLocationNotFound AitoolsErrorCategory = "MCP_LOCATION_NOT_FOUND" + AitoolsErrorCategoryMCPNameConflict AitoolsErrorCategory = "MCP_NAME_CONFLICT" + AitoolsErrorCategoryMCPUpstreamError AitoolsErrorCategory = "MCP_UPSTREAM_ERROR" + AitoolsErrorCategoryAgentConfigInvalid AitoolsErrorCategory = "AGENT_CONFIG_INVALID" ) // AitoolsAgentResult records one agent's failed or skipped outcome within an diff --git a/libs/telemetry/protos/aitools_mcp.go b/libs/telemetry/protos/aitools_mcp.go new file mode 100644 index 00000000000..b153f791dc5 --- /dev/null +++ b/libs/telemetry/protos/aitools_mcp.go @@ -0,0 +1,99 @@ +package protos + +// The types in this file mirror the aitools MCP messages of the databricks_cli +// lumberjack protos: AitoolsMcpEvent and its server result in +// proto/logs/frontend/databricks_cli/aitools_mcp.proto, and the enums in +// proto/logs/frontend/databricks_cli/enum.proto. + +// AitoolsMcpOperation mirrors AitoolsMcpOperation.Type in aitools_mcp.proto. +type AitoolsMcpOperation string + +const ( + AitoolsMcpOperationAdd AitoolsMcpOperation = "ADD" + AitoolsMcpOperationRemove AitoolsMcpOperation = "REMOVE" + AitoolsMcpOperationSignIn AitoolsMcpOperation = "SIGN_IN" + AitoolsMcpOperationSignOut AitoolsMcpOperation = "SIGN_OUT" + AitoolsMcpOperationProxyStarted AitoolsMcpOperation = "PROXY_STARTED" + // AitoolsMcpOperationSignInPrompted is logged for each authenticate call that showed the sign-in page, with the + // state of the sign-in when that call's wait ended. A retry after a cancelled call logs its own event. + AitoolsMcpOperationSignInPrompted AitoolsMcpOperation = "SIGN_IN_PROMPTED" +) + +// AitoolsMcpService mirrors AitoolsMcpService.Type in enum.proto. The CLI +// sends only these values, never a service name: the system.ai leaves in the +// table map to their own value, other system.ai leaves to OtherSystem, and any +// service outside system.ai to Custom. +type AitoolsMcpService string + +const ( + AitoolsMcpServiceOtherSystem AitoolsMcpService = "OTHER_SYSTEM" + AitoolsMcpServiceCustom AitoolsMcpService = "CUSTOM" + AitoolsMcpServiceGenieOne AitoolsMcpService = "GENIE_ONE" + AitoolsMcpServiceDBSQL AitoolsMcpService = "DBSQL" + AitoolsMcpServiceWebSearch AitoolsMcpService = "WEB_SEARCH" + AitoolsMcpServiceSandbox AitoolsMcpService = "SANDBOX" + AitoolsMcpServiceSlack AitoolsMcpService = "SLACK" + AitoolsMcpServiceGitHub AitoolsMcpService = "GITHUB" + AitoolsMcpServiceAtlassian AitoolsMcpService = "ATLASSIAN" + AitoolsMcpServiceGoogleDrive AitoolsMcpService = "GOOGLE_DRIVE" + AitoolsMcpServiceGoogleCalendar AitoolsMcpService = "GOOGLE_CALENDAR" + AitoolsMcpServiceGmail AitoolsMcpService = "GMAIL" + AitoolsMcpServiceMicrosoft365 AitoolsMcpService = "MICROSOFT_365" +) + +// AitoolsMcpSignInState mirrors AitoolsMcpSignInState.Type in enum.proto. It is +// the sign-in state of a service after the operation. +type AitoolsMcpSignInState string + +const ( + AitoolsMcpSignInStateNotRequired AitoolsMcpSignInState = "NOT_REQUIRED" + AitoolsMcpSignInStateActive AitoolsMcpSignInState = "ACTIVE" + AitoolsMcpSignInStatePending AitoolsMcpSignInState = "PENDING" +) + +// AitoolsTargetOutcome mirrors AitoolsTargetOutcome.Type in enum.proto. It is +// what an operation did to one target: an agent config entry, or a sign-in. +type AitoolsTargetOutcome string + +const ( + AitoolsTargetOutcomeChanged AitoolsTargetOutcome = "CHANGED" + AitoolsTargetOutcomeUnchanged AitoolsTargetOutcome = "UNCHANGED" + // AitoolsTargetOutcomeUpdateAvailable mirrors the schema. No MCP command sends it yet. + AitoolsTargetOutcomeUpdateAvailable AitoolsTargetOutcome = "UPDATE_AVAILABLE" + AitoolsTargetOutcomeSkipped AitoolsTargetOutcome = "SKIPPED" + AitoolsTargetOutcomeFailed AitoolsTargetOutcome = "FAILED" +) + +// AitoolsMcpServerResult is the result for one service, for one agent when the +// operation writes agent config. Unset fields are omitted: an agent is not set +// for sign-in, and an outcome is not set for proxy events. An error category is +// set on a FAILED target and on SIGN_IN_PROMPTED, where it is SIGN_IN_REQUIRED. A +// SKIPPED target carries none: remove skips an entry it left in place, without +// a failure. +type AitoolsMcpServerResult struct { + Service AitoolsMcpService `json:"service"` + Agent AitoolsAgentType `json:"agent,omitempty"` + Outcome AitoolsTargetOutcome `json:"outcome,omitempty"` + ErrorCategory AitoolsErrorCategory `json:"error_category,omitempty"` + SignInState AitoolsMcpSignInState `json:"sign_in_state,omitempty"` +} + +// AitoolsMcpEvent is emitted by `databricks aitools mcp` commands and by the +// proxy (`serve`). It carries enums and counts only: no host, profile, workspace +// ID, custom service or schema name, path, error text or tool name. +type AitoolsMcpEvent struct { + // CallingAgent is the coding agent that ran the command or started the proxy. + // AitoolsAgentTypeUnspecified when no agent is detected. + CallingAgent AitoolsAgentType `json:"calling_agent"` + + Operation AitoolsMcpOperation `json:"operation"` + + // Servers holds one entry per (service, agent) for ADD and REMOVE, one per + // service for SIGN_IN, SIGN_OUT, PROXY_STARTED and SIGN_IN_PROMPTED. + Servers []AitoolsMcpServerResult `json:"servers,omitempty"` + + // BoundProfileCount is the number of distinct profiles with MCP servers + // installed, read from the state file after the operation. Zero is a real + // value (every server removed), so it is not omitted. + BoundProfileCount int64 `json:"bound_profile_count"` +} diff --git a/libs/telemetry/protos/aitools_mcp_test.go b/libs/telemetry/protos/aitools_mcp_test.go new file mode 100644 index 00000000000..715a196103a --- /dev/null +++ b/libs/telemetry/protos/aitools_mcp_test.go @@ -0,0 +1,58 @@ +package protos + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestAitoolsMcpEventEncodesSnakeCaseKeys checks that the lumberjack proto keys these fields in snake_case. It pins the +// wire keys of an event and of one server result, and that unset enums are omitted rather than sent empty. +func TestAitoolsMcpEventEncodesSnakeCaseKeys(t *testing.T) { + b, err := json.Marshal(DatabricksCliLog{AitoolsMcpEvent: &AitoolsMcpEvent{ + CallingAgent: AitoolsAgentTypeClaudeCode, + Operation: AitoolsMcpOperationAdd, + BoundProfileCount: 0, + Servers: []AitoolsMcpServerResult{ + { + Service: AitoolsMcpServiceGitHub, + Agent: AitoolsAgentTypeCodex, + Outcome: AitoolsTargetOutcomeFailed, + ErrorCategory: AitoolsErrorCategoryAgentConfigInvalid, + SignInState: AitoolsMcpSignInStatePending, + }, + }, + }}) + require.NoError(t, err) + + var got map[string]any + require.NoError(t, json.Unmarshal(b, &got)) + event, ok := got["aitools_mcp_event"].(map[string]any) + require.True(t, ok, "aitools_mcp_event must be the event key") + + assert.Equal(t, "CLAUDE_CODE", event["calling_agent"]) + assert.Equal(t, "ADD", event["operation"]) + // Zero is a real count and is sent. + assert.EqualValues(t, 0, event["bound_profile_count"]) + + servers := event["servers"].([]any) + require.Len(t, servers, 1) + server := servers[0].(map[string]any) + assert.Equal(t, map[string]any{ + "service": "GITHUB", + "agent": "CODEX", + "outcome": "FAILED", + "error_category": "AGENT_CONFIG_INVALID", + "sign_in_state": "PENDING", + }, server) +} + +// TestAitoolsMcpServerResultOmitsUnsetFields checks that a proxy event has no agent on the result, no outcome and no error +// category when it succeeded. +func TestAitoolsMcpServerResultOmitsUnsetFields(t *testing.T) { + b, err := json.Marshal(AitoolsMcpServerResult{Service: AitoolsMcpServiceOtherSystem}) + require.NoError(t, err) + assert.JSONEq(t, `{"service":"OTHER_SYSTEM"}`, string(b)) +} diff --git a/libs/telemetry/protos/frontend_log.go b/libs/telemetry/protos/frontend_log.go index d7968016109..2341357741c 100644 --- a/libs/telemetry/protos/frontend_log.go +++ b/libs/telemetry/protos/frontend_log.go @@ -27,6 +27,7 @@ type DatabricksCliLog struct { SshAgentShimEvent *SshAgentShimEvent `json:"ssh_agent_shim_event,omitempty"` BundleConfigRemoteSyncEvent *BundleConfigRemoteSyncEvent `json:"bundle_config_remote_sync_event,omitempty"` AitoolsInstallEvent *AitoolsInstallEvent `json:"aitools_install_event,omitempty"` + AitoolsMcpEvent *AitoolsMcpEvent `json:"aitools_mcp_event,omitempty"` SetupLocalEvent *SetupLocalEvent `json:"setup_local_event,omitempty"` AirRunEvent *AirRunEvent `json:"air_run_event,omitempty"` AitoolsHintEvent *AitoolsHintEvent `json:"aitools_hint_event,omitempty"`