diff --git a/.nextchanges/cli/auth-keyring-large-tokens.md b/.nextchanges/cli/auth-keyring-large-tokens.md new file mode 100644 index 00000000000..6ad59edefeb --- /dev/null +++ b/.nextchanges/cli/auth-keyring-large-tokens.md @@ -0,0 +1 @@ +* Fix `databricks auth login` failing with `store token: data passed to Set was too big` on macOS and Windows when the OAuth access token is too large for one OS keyring item. Such a token is now stored across several keyring items. ([#7845](https://github.com/databricks/cli/pull/7845)) diff --git a/acceptance/cmd/auth/login/device-code/non-interactive/errors/output.txt b/acceptance/cmd/auth/login/device-code/non-interactive/errors/output.txt index fedaf8d96a1..3e25d9a5383 100644 --- a/acceptance/cmd/auth/login/device-code/non-interactive/errors/output.txt +++ b/acceptance/cmd/auth/login/device-code/non-interactive/errors/output.txt @@ -36,6 +36,11 @@ Error: --confirmation-code requires --use-device-code >>> musterr [CLI] auth login --host [DATABRICKS_URL] --profile test#device-code --use-device-code Error: profile name "test#device-code" must not end with #device-code, which marks pending device code logins +=== Profile name that marks an access token part + +>>> musterr [CLI] auth login --host [DATABRICKS_URL] --profile test#access-token-0 --use-device-code +Error: profile name "test#access-token-0" must not contain #access-token-, which marks the parts of large access tokens + === Account host without an account ID >>> musterr [CLI] auth login --host https://accounts.cloud.databricks.test --use-device-code --user-code WDJB-0001 --confirmation-code FTFA-QQKY diff --git a/acceptance/cmd/auth/login/device-code/non-interactive/errors/script b/acceptance/cmd/auth/login/device-code/non-interactive/errors/script index bfbc334230a..fbb274624f1 100644 --- a/acceptance/cmd/auth/login/device-code/non-interactive/errors/script +++ b/acceptance/cmd/auth/login/device-code/non-interactive/errors/script @@ -23,6 +23,9 @@ trace musterr $CLI auth login --host $DATABRICKS_HOST --profile test --user-code title "Profile name that marks a pending login" trace musterr $CLI auth login --host $DATABRICKS_HOST --profile 'test#device-code' --use-device-code +title "Profile name that marks an access token part" +trace musterr $CLI auth login --host $DATABRICKS_HOST --profile 'test#access-token-0' --use-device-code + # The non-interactive login cannot prompt for the account ID. title "Account host without an account ID" trace musterr $CLI auth login --host https://accounts.cloud.databricks.test --use-device-code --user-code WDJB-0001 --confirmation-code FTFA-QQKY diff --git a/cmd/auth/login.go b/cmd/auth/login.go index 1583730ce4f..c9f185da6c4 100644 --- a/cmd/auth/login.go +++ b/cmd/auth/login.go @@ -127,6 +127,9 @@ func storeLoginToken(ctx context.Context, tokenStore storage.Store, mode storage tokenStore = storage.WrapForOAuthArgument(ctx, tokenStore, mode, arg) key := arg.GetCacheKey() if err := tokenStore.Put(key, storage.Entry{Token: token}); err != nil { + // This includes storage.ErrAccessTokenNotStored, which a refresh treats + // as stored: a failed login can be run again, while a refresh has to + // keep the refresh token that the server just replaced. storeErr := fmt.Errorf("store token: %w", err) // The profile is already saved, but the old token may still be present. // Delete it so later commands cannot use it with the updated profile. diff --git a/cmd/auth/login_device_code.go b/cmd/auth/login_device_code.go index 59f5599fc43..34ec06b4a96 100644 --- a/cmd/auth/login_device_code.go +++ b/cmd/auth/login_device_code.go @@ -167,10 +167,18 @@ func pendingLoginKey(cacheKey string) string { // validateLoginProfileName returns an error if the token of profile name would // share its token store key with the pending login of another profile. +// It also rejects a name that would share a key with a part of another +// profile's access token. func validateLoginProfileName(name string) error { if strings.HasSuffix(name, pendingLoginKeySuffix) { return fmt.Errorf("profile name %q must not end with %s, which marks pending device code logins", name, pendingLoginKeySuffix) } + // Windows Credential Manager ignores case in credential names, so a + // profile named "dev#Access-Token-0" would share the item of a part of dev. + // https://learn.microsoft.com/en-us/windows/win32/api/wincred/ns-wincred-credentialw + if strings.Contains(strings.ToLower(name), storage.AccessTokenPartMarker) { + return fmt.Errorf("profile name %q must not contain %s, which marks the parts of large access tokens", name, storage.AccessTokenPartMarker) + } return nil } diff --git a/cmd/auth/login_device_code_test.go b/cmd/auth/login_device_code_test.go index 0e6fa1eb40c..9c0943e195c 100644 --- a/cmd/auth/login_device_code_test.go +++ b/cmd/auth/login_device_code_test.go @@ -843,3 +843,39 @@ func TestLoginDeviceCodeWrongConfirmationCode(t *testing.T) { }) } } + +func TestValidateLoginProfileName(t *testing.T) { + cases := []struct { + name string + profile string + wantErr string + }{ + {name: "plain name", profile: "dev"}, + { + name: "key of a pending device code login", + profile: "dev#device-code", + wantErr: `profile name "dev#device-code" must not end with #device-code, which marks pending device code logins`, + }, + { + name: "key of an access token part", + profile: "dev#access-token-0", + wantErr: `profile name "dev#access-token-0" must not contain #access-token-, which marks the parts of large access tokens`, + }, + { + name: "key of an access token part in another letter case", + profile: "dev#Access-Token-0", + wantErr: `profile name "dev#Access-Token-0" must not contain #access-token-, which marks the parts of large access tokens`, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + err := validateLoginProfileName(tc.profile) + if tc.wantErr == "" { + require.NoError(t, err) + return + } + assert.EqualError(t, err, tc.wantErr) + }) + } +} diff --git a/cmd/auth/login_test.go b/cmd/auth/login_test.go index 97b8d82cc9f..82617939110 100644 --- a/cmd/auth/login_test.go +++ b/cmd/auth/login_test.go @@ -1604,7 +1604,9 @@ func TestPromptForProfileRejectsTakenName(t *testing.T) { tm.WaitFor(`>> profile name "__settings__" is reserved for internal use`) tm.Type(strings.Repeat(termtest.KeyBackspace, len("__settings__")) + "take#device-code" + termtest.KeyEnter) tm.WaitFor(`>> profile name "take#device-code" must not end with #device-code, which marks pending device code logins`) - tm.Type(strings.Repeat(termtest.KeyBackspace, len("#device-code")) + termtest.KeyEnter) + tm.Type(strings.Repeat(termtest.KeyBackspace, len("#device-code")) + "#access-token-0" + termtest.KeyEnter) + tm.WaitFor(`>> profile name "take#access-token-0" must not contain #access-token-, which marks the parts of large access tokens`) + tm.Type(strings.Repeat(termtest.KeyBackspace, len("#access-token-0")) + termtest.KeyEnter) name, err := tm.Result() require.NoError(t, err) assert.Equal(t, "take", name) diff --git a/cmd/auth/logout_keyring_os_test.go b/cmd/auth/logout_keyring_os_test.go new file mode 100644 index 00000000000..fe56b6a4749 --- /dev/null +++ b/cmd/auth/logout_keyring_os_test.go @@ -0,0 +1,71 @@ +//go:build darwin || windows + +package auth + +import ( + "os" + "strings" + "testing" + "uuid" + + "github.com/databricks/cli/libs/auth/storage" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/databrickscfg/profile" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/zalando/go-keyring" + "golang.org/x/oauth2" +) + +// TestLogoutRemovesSplitTokenFromOSKeyring logs out of a profile whose token +// the real OS keyring holds in several items, because it is too large for +// one, and checks that none of them is left. It uses the store that logout +// uses, with its 3-second timeout per keyring operation. +// +// It writes to the real OS keyring and therefore runs on CI only (GitHub +// Actions sets CI=true). +func TestLogoutRemovesSplitTokenFromOSKeyring(t *testing.T) { + if os.Getenv("CI") == "" { + t.Skip("touches the real OS keyring; runs on CI only") + } + + // The keyring store keeps every item in this service. + const service = "databricks-cli" + profileName := "__probe_" + uuid.New().String() + items := []string{ + profileName, + profileName + storage.AccessTokenPartMarker + "0", + profileName + storage.AccessTokenPartMarker + "1", + profileName + storage.AccessTokenPartMarker + "2", + } + t.Cleanup(func() { + for _, account := range items { + _ = keyring.Delete(service, account) + } + }) + + ctx := cmdio.MockDiscard(t.Context()) + configPath := writeTempConfig(t, "["+profileName+"]\nhost = https://keyring-logout.cloud.databricks.test\nauth_type = databricks-cli\n") + t.Setenv("DATABRICKS_CONFIG_FILE", configPath) + + tokenStore := storage.NewKeyringStore() + // The size of the access tokens in https://github.com/databricks/cli/issues/7707. + tok := &oauth2.Token{AccessToken: strings.Repeat("a", 3250), RefreshToken: "refresh-token"} + require.NoError(t, tokenStore.Put(profileName, storage.Entry{Token: tok})) + _, err := keyring.Get(service, items[1]) + require.NoError(t, err, "the token was not split") + + err = runLogout(ctx, logoutArgs{ + profileName: profileName, + autoApprove: true, + profiler: profile.DefaultProfiler, + tokenStore: tokenStore, + configFilePath: configPath, + }) + require.NoError(t, err) + + for _, account := range items { + _, err := keyring.Get(service, account) + assert.ErrorIs(t, err, keyring.ErrNotFound, "item %q is left in the OS keyring", account) + } +} diff --git a/libs/auth/storage/keyring.go b/libs/auth/storage/keyring.go index 1beccdca3e5..7a70c28c8ec 100644 --- a/libs/auth/storage/keyring.go +++ b/libs/auth/storage/keyring.go @@ -2,9 +2,15 @@ package storage import ( "cmp" + "crypto/sha256" + "encoding/base64" + "encoding/hex" "encoding/json" "errors" "fmt" + "runtime" + "strconv" + "strings" "time" "uuid" @@ -17,6 +23,27 @@ import ( // cache key the U2M manager passes through Store.Put / Lookup. const keyringServiceName = "databricks-cli" +// AccessTokenPartMarker separates a cache key from the index of one of its +// access token parts, the items that hold the access token of an entry too +// large for one item. A cache key must not contain it, or the entry would +// share its account with a part of another entry. +const AccessTokenPartMarker = "#access-token-" + +// accessTokenPartSize is how many bytes of an access token go into one part. +// It stays under the smallest backend limit. Windows Credential Manager caps +// a credential at 2560 bytes. go-keyring's macOS backend base64-encodes the +// secret into one command line for `security -i`, which reads at most 4096 +// bytes per line, so about 3000 bytes fit. +// https://learn.microsoft.com/en-us/windows/win32/api/wincred/ns-wincred-credentialw +// https://github.com/zalando/go-keyring/blob/v0.2.8/keyring_darwin.go#L86-L89 +const accessTokenPartSize = 2048 + +// ErrAccessTokenNotStored is wrapped in the error of a Put that stored the +// entry of a split token, with its refresh token, but not all of its access +// token parts. Lookup then returns the token without an access token, so the +// next command refreshes it. +var ErrAccessTokenNotStored = errors.New("access token not stored") + // keyringProbeAccountPrefix is prefixed onto a per-call random suffix to form // the account name ProbeKeyring writes and deletes. A fixed name like // "__probe__" could collide with a user profile of the same name (which is @@ -34,6 +61,14 @@ const keyringProbeAccountPrefix = "__probe_" // a GUI unlock prompt that no one answers in a headless session. const defaultKeyringTimeout = 3 * time.Second +// splitLookupAttempts and splitLookupDelay make Lookup read an incomplete +// split token again after 25, 50, 100 and 200 milliseconds, like the store +// update recovery in u2m. +const ( + splitLookupAttempts = 5 + splitLookupDelay = 25 * time.Millisecond +) + // keyringBackend is the subset of zalando/go-keyring the cache depends on. // Extracted as an interface so tests can inject a fake. type keyringBackend interface { @@ -47,17 +82,59 @@ type keyringBackend interface { // checksum, store time, ...) without breaking older CLI versions that read // the same entry. type keyringEntry struct { + entryFields + + // Split is set for a token that is too large for one keyring item. CLI + // versions that predate it still read the entry, so Token is then empty + // rather than nil, because releases before v1.24 dereference it. It must + // not carry the refresh token either: they would rotate it and then fail + // to store the oversized result. + Split *splitToken `json:"split,omitempty"` +} + +// entryFields holds the fields of Entry. Entry converts to and from it, so a +// field added to Entry but not here fails to compile instead of being dropped +// by the keyring store. +type entryFields struct { Token *oauth2.Token `json:"token"` PendingLogin *PendingLogin `json:"pending_login,omitempty"` } +// splitToken is the part of an oversized token that stays in its entry. +type splitToken struct { + // Token is the token without its access token. + Token oauth2.Token `json:"token"` + + // AccessTokenParts is the number of parts that hold the access token and + // AccessTokenSHA256 is its hex digest. The parts are written one at a + // time, so only the digest tells a complete access token from one that + // another process is still writing or has since replaced. + AccessTokenParts int `json:"access_token_parts"` + AccessTokenSHA256 string `json:"access_token_sha256"` +} + // zalandoBackend delegates to the process-wide zalando/go-keyring provider. type zalandoBackend struct{} func (zalandoBackend) Set(service, account, secret string) error { + if runtime.GOOS == "darwin" && darwinCommandTooBig(service, account, secret) { + return keyring.ErrSetDataTooBig + } return keyring.Set(service, account, secret) } +// darwinCommandTooBig reports whether go-keyring's macOS backend will reject +// secret as too big. It does so only after starting `security -i`, which it +// then never reaps, so Set checks first. The count leaves out shell quotes, +// which only make the command longer, so no secret that go-keyring would store +// is rejected. +// https://github.com/zalando/go-keyring/blob/v0.2.8/keyring_darwin.go#L82-L88 +func darwinCommandTooBig(service, account, secret string) bool { + password := "go-keyring-base64:" + base64.StdEncoding.EncodeToString([]byte(secret)) + command := fmt.Sprintf("add-generic-password -U -s %s -a %s -w %s\n", service, account, password) + return len(command) > 4096 +} + func (zalandoBackend) Get(service, account string) (string, error) { return keyring.Get(service, account) } @@ -76,6 +153,10 @@ type keyringStore struct { backend keyringBackend timeout time.Duration keyringSvcName string + + // retryDelay is Lookup's first wait before it reads an incomplete split + // token again, splitLookupDelay if zero. It doubles on each attempt. + retryDelay time.Duration } // NewKeyringStore returns a Store backed by the OS-native secure store (via @@ -147,49 +228,187 @@ func probeReadWithBackend(backend keyringBackend, timeout time.Duration) error { // Put implements the Store interface. func (k *keyringStore) Put(key string, e Entry) error { - raw, err := json.Marshal(keyringEntry(e)) + raw, err := json.Marshal(keyringEntry{entryFields: entryFields(e)}) if err != nil { return fmt.Errorf("marshal token: %w", err) } - return k.withTimeout("set", func() error { - return k.backend.Set(k.keyringSvcName, key, string(raw)) + err = k.set(key, string(raw)) + if e.Token != nil && errors.Is(err, keyring.ErrSetDataTooBig) { + return k.putSplit(key, entryFields(e)) + } + return err +} + +// putSplit stores a token that is too large for one keyring item. The entry +// under key keeps everything but the access token, which goes into parts next +// to it. +// +// The entry is written first because it holds the refresh token. The server +// rotates the refresh token on every refresh, so losing the new one logs the +// user out, while losing the access token only costs another refresh. +func (k *keyringStore) putSplit(key string, fields entryFields) error { + t := fields.Token + sum := sha256.Sum256([]byte(t.AccessToken)) + n := (len(t.AccessToken) + accessTokenPartSize - 1) / accessTokenPartSize + rest := *t + rest.AccessToken = "" + fields.Token = &oauth2.Token{} + raw, err := json.Marshal(keyringEntry{ + entryFields: fields, + Split: &splitToken{ + Token: rest, + AccessTokenParts: n, + AccessTokenSHA256: hex.EncodeToString(sum[:]), + }, }) + if err != nil { + return fmt.Errorf("marshal token: %w", err) + } + if err := k.set(key, string(raw)); err != nil { + return err + } + for i := range n { + start := i * accessTokenPartSize + part := t.AccessToken[start:min(start+accessTokenPartSize, len(t.AccessToken))] + if err := k.set(accessTokenPartKey(key, i), part); err != nil { + return fmt.Errorf("%w: %w", ErrAccessTokenNotStored, err) + } + } + return nil } // Delete implements the Store interface. Removing a missing entry is a no-op. func (k *keyringStore) Delete(key string) error { - return k.withTimeout("delete", func() error { - err := k.backend.Delete(k.keyringSvcName, key) + err := k.remove(key) + if err != nil && !errors.Is(err, keyring.ErrNotFound) { + return err + } + // A token that fits in one item can replace a split one without touching + // its access token parts, so the entry does not say how many there are. + // They are counted and then deleted from the last, so that a Delete that + // fails part way leaves parts that the next one still finds. + n := 0 + for ; ; n++ { + _, err := k.get(accessTokenPartKey(key, n)) if errors.Is(err, keyring.ErrNotFound) { - return nil + break } - return err - }) + if err != nil { + return err + } + } + for i := n - 1; i >= 0; i-- { + err := k.remove(accessTokenPartKey(key, i)) + if err != nil && !errors.Is(err, keyring.ErrNotFound) { + return err + } + } + return nil } // Lookup implements the Store interface, returning ErrNotFound on a miss. +// +// A split token with missing or mismatched parts is read again a few times +// before it is returned without its access token, because another process +// may be writing or deleting it: refreshing at once could spend a refresh +// token that process replaced, or restore a login it deleted. func (k *keyringStore) Lookup(key string) (Entry, error) { - var raw string + delay := cmp.Or(k.retryDelay, splitLookupDelay) + for attempt := 1; ; attempt++ { + e, complete, err := k.lookup(key) + if err != nil || complete || attempt == splitLookupAttempts { + return e, err + } + time.Sleep(delay) + delay *= 2 + } +} + +// lookup reads the entry under key once. It reports whether the entry is +// complete, which only a split token with missing or mismatched parts is not. +func (k *keyringStore) lookup(key string) (Entry, bool, error) { + raw, err := k.get(key) + if errors.Is(err, keyring.ErrNotFound) { + return Entry{}, false, ErrNotFound + } + if err != nil { + return Entry{}, false, wrapKeyringUnreachable(err) + } + + var entry keyringEntry + if err := json.Unmarshal([]byte(raw), &entry); err != nil { + return Entry{}, false, fmt.Errorf("unmarshal token: %w", err) + } + if entry.Split != nil { + return k.lookupSplit(key, entry) + } + return Entry(entry.entryFields), true, nil +} + +// lookupSplit assembles a token stored by putSplit. When the access token +// parts are missing or do not match the entry, because a write was interrupted +// or another process is writing, it returns the token without an access token +// and reports it as incomplete. +func (k *keyringStore) lookupSplit(key string, entry keyringEntry) (Entry, bool, error) { + s := entry.Split + t := s.Token + e := Entry(entry.entryFields) + e.Token = &t + var accessToken strings.Builder + for i := range s.AccessTokenParts { + part, err := k.get(accessTokenPartKey(key, i)) + if errors.Is(err, keyring.ErrNotFound) { + return e, false, nil + } + if err != nil { + return Entry{}, false, wrapKeyringUnreachable(err) + } + accessToken.WriteString(part) + } + sum := sha256.Sum256([]byte(accessToken.String())) + if hex.EncodeToString(sum[:]) != s.AccessTokenSHA256 { + return e, false, nil + } + t.AccessToken = accessToken.String() + return e, true, nil +} + +// accessTokenPartKey returns the key of the i-th access token part of the +// entry stored under key. +func accessTokenPartKey(key string, i int) string { + return key + AccessTokenPartMarker + strconv.Itoa(i) +} + +// set writes secret under key within the store's timeout. +func (k *keyringStore) set(key, secret string) error { + return k.withTimeout("set", func() error { + return k.backend.Set(k.keyringSvcName, key, secret) + }) +} + +// get reads the secret under key within the store's timeout. +func (k *keyringStore) get(key string) (string, error) { + var secret string err := k.withTimeout("get", func() error { got, gerr := k.backend.Get(k.keyringSvcName, key) if gerr != nil { return gerr } - raw = got + secret = got return nil }) - if errors.Is(err, keyring.ErrNotFound) { - return Entry{}, ErrNotFound - } + // After a timeout the goroutine may still be writing secret. if err != nil { - return Entry{}, wrapKeyringUnreachable(err) + return "", err } + return secret, nil +} - var entry keyringEntry - if err := json.Unmarshal([]byte(raw), &entry); err != nil { - return Entry{}, fmt.Errorf("unmarshal token: %w", err) - } - return Entry(entry), nil +// remove deletes the item under key within the store's timeout. +func (k *keyringStore) remove(key string) error { + return k.withTimeout("delete", func() error { + return k.backend.Delete(k.keyringSvcName, key) + }) } // wrapKeyringUnreachable wraps a non-ErrNotFound keyring error with diff --git a/libs/auth/storage/keyring_darwin_test.go b/libs/auth/storage/keyring_darwin_test.go new file mode 100644 index 00000000000..64651ba723e --- /dev/null +++ b/libs/auth/storage/keyring_darwin_test.go @@ -0,0 +1,35 @@ +package storage + +import ( + "os" + "strings" + "testing" + "uuid" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/zalando/go-keyring" +) + +// TestZalandoBackend_Set_RejectsLargeSecretWithoutSecurityProcess pins that +// zalandoBackend rejects a secret that is too big on macOS without calling +// go-keyring, which would start `security -i` and leave its stdin pipe open in +// this process. Nothing reaches the keychain either way, so it runs outside CI +// too. This file sorts before keyring_size_limit_os_test.go, so the test runs +// before TestKeyringSet_RejectsLargeSecret, which calls go-keyring directly and +// leaves such a pipe, which a finalizer could close between the two counts. +func TestZalandoBackend_Set_RejectsLargeSecretWithoutSecurityProcess(t *testing.T) { + before := openFileCount(t) + + err := zalandoBackend{}.Set(keyringServiceName, keyringProbeAccountPrefix+uuid.New().String(), strings.Repeat("a", 5000)) + require.ErrorIs(t, err, keyring.ErrSetDataTooBig) + assert.Equal(t, before, openFileCount(t), "go-keyring started `security -i`, which keeps a pipe open") +} + +// openFileCount returns the number of files this process has open. +func openFileCount(t *testing.T) int { + t.Helper() + entries, err := os.ReadDir("/dev/fd") + require.NoError(t, err) + return len(entries) +} diff --git a/libs/auth/storage/keyring_size_limit_os_test.go b/libs/auth/storage/keyring_size_limit_os_test.go index ea2451c4b2d..aa1d2d79099 100644 --- a/libs/auth/storage/keyring_size_limit_os_test.go +++ b/libs/auth/storage/keyring_size_limit_os_test.go @@ -4,44 +4,148 @@ package storage import ( "os" + "runtime" + "strconv" "strings" "testing" "uuid" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/zalando/go-keyring" - "golang.org/x/oauth2" ) -// TestKeyringStore_Put_RealBackendRejectsLargeToken pins that the go-keyring -// release in go.mod rejects a 3250-byte access token on macOS and Windows, +// TestKeyringSet_RejectsLargeSecret pins that the go-keyring +// release in go.mod rejects a 3250-byte secret on macOS and Windows, // which is the behavior fakeBackend's caps in keyring_size_limit_test.go -// transcribe. It is the only test here that observes the real library, so it -// is what fails when a bumped or replaced keyring library accepts larger -// secrets and the modeled caps go stale. +// transcribe. It is the only test here that observes the real library reject +// a secret, so it is what fails when a bumped or replaced keyring library +// accepts larger secrets and the modeled caps go stale. +// +// It calls go-keyring directly: the store splits a token of this size, and +// zalandoBackend checks the size itself on macOS. // // It calls the real backend and therefore runs on CI only (GitHub Actions // sets CI=true). The library's size check fires before the OS keyring is // contacted, so on the expected path nothing is written. Only once the cap is -// gone does the token land in the runner's keyring, and then the cleanup +// gone does the secret land in the runner's keyring, and then the cleanup // removes it. -func TestKeyringStore_Put_RealBackendRejectsLargeToken(t *testing.T) { +func TestKeyringSet_RejectsLargeSecret(t *testing.T) { if os.Getenv("CI") == "" { t.Skip("touches the real OS keyring; runs on CI only") } - store := NewKeyringStore() account := keyringProbeAccountPrefix + uuid.New().String() - // The access token alone exceeds both caps (macOS after base64, Windows - // raw), so no other token field is needed. - tok := &oauth2.Token{AccessToken: strings.Repeat("a", issue7707LargeAccessTokenBytes)} + // The secret alone exceeds both caps (macOS after base64, Windows raw). + secret := strings.Repeat("a", issue7707LargeAccessTokenBytes) - err := store.Put(account, Entry{Token: tok}) + err := keyring.Set(keyringServiceName, account, secret) if err == nil { - t.Cleanup(func() { _ = store.Delete(account) }) + t.Cleanup(func() { _ = keyring.Delete(keyringServiceName, account) }) } require.ErrorIs(t, err, keyring.ErrSetDataTooBig, - "go-keyring accepted a %d-byte access token; the OS caps modeled in fakeBackend are stale, re-verify them against the new backend", + "go-keyring accepted a %d-byte secret; the OS caps modeled in fakeBackend are stale, re-verify them against the new backend", issue7707LargeAccessTokenBytes) } + +// newRealKeyringStore returns a store on the real OS keyring and a key that no +// other test or profile uses. The store has the test timeout rather than the +// 3-second production one, so that a slow runner cannot fail these tests. The +// cleanup removes the entry and its parts through the backend, so it does not +// depend on the store's Delete and also runs when the test fails. +// +// The tests that use it write to the real OS keyring and therefore run on CI +// only (GitHub Actions sets CI=true). +func newRealKeyringStore(t *testing.T) (*keyringStore, keyringBackend, string) { + t.Helper() + if os.Getenv("CI") == "" { + t.Skip("touches the real OS keyring; runs on CI only") + } + + backend := zalandoBackend{} + key := keyringProbeAccountPrefix + uuid.New().String() + t.Cleanup(func() { + for _, account := range itemsOf(key, maxTestParts) { + _ = backend.Delete(keyringServiceName, account) + } + }) + return newTestStore(backend), backend, key +} + +func TestKeyringStore_RealBackend_SplitTokenRoundTrips(t *testing.T) { + cases := []struct { + accessTokenSize int + wantParts int + }{ + // https://github.com/databricks/cli/issues/7707 + {accessTokenSize: issue7707LargeAccessTokenBytes, wantParts: 2}, + {accessTokenSize: 5000, wantParts: 3}, + } + + for _, tc := range cases { + t.Run(strconv.Itoa(tc.accessTokenSize)+" bytes", func(t *testing.T) { + c, backend, key := newRealKeyringStore(t) + want := testToken(tc.accessTokenSize) + + require.NoError(t, c.Put(key, Entry{Token: want})) + assert.Equal(t, itemsOf(key, tc.wantParts), presentItems(t, backend, key)) + + got, err := c.Lookup(key) + require.NoError(t, err) + assert.Equal(t, want.AccessToken, got.Token.AccessToken) + assert.Equal(t, want.TokenType, got.Token.TokenType) + assert.Equal(t, want.RefreshToken, got.Token.RefreshToken) + assert.True(t, want.Expiry.Equal(got.Token.Expiry)) + + require.NoError(t, c.Delete(key)) + assert.Empty(t, presentItems(t, backend, key)) + }) + } +} + +// Released CLI versions on the same machine read a token that fits, so it +// must stay one item in their format. +func TestKeyringStore_RealBackend_TokenThatFitsIsOneItem(t *testing.T) { + c, backend, key := newRealKeyringStore(t) + tok := testToken(issue7707SmallAccessTokenBytes) + + require.NoError(t, c.Put(key, Entry{Token: tok})) + + got, err := backend.Get(keyringServiceName, key) + require.NoError(t, err) + assert.Equal(t, releasedEntry(t, tok), got) + assert.Equal(t, itemsOf(key, 0), presentItems(t, backend, key)) +} + +func TestKeyringStore_RealBackend_TokenSizeTransitions(t *testing.T) { + c, backend, key := newRealKeyringStore(t) + assertTokenSizeTransitions(t, c, backend, key) +} + +// The cap that fakeBackend models for this OS must match the real backend at +// its edge: the largest token that fits is stored and read back whole in one +// item, and one byte more is split. On macOS this is where go-keyring's check +// and the 4096-byte line of `security -i` meet. +func TestKeyringStore_RealBackend_SplitsOnlyTokenThatDoesNotFit(t *testing.T) { + c, backend, key := newRealKeyringStore(t) + secretTooBig := darwinSecurityCommandTooBig + if runtime.GOOS == "windows" { + secretTooBig = windowsCredentialBlobTooBig + } + n := largestAccessTokenInOneItem(t, secretTooBig, key) + + fits := testToken(n) + require.NoError(t, c.Put(key, Entry{Token: fits})) + raw, err := backend.Get(keyringServiceName, key) + require.NoError(t, err) + assert.Equal(t, releasedEntry(t, fits), raw) + assert.Equal(t, itemsOf(key, 0), presentItems(t, backend, key)) + + tooBig := testToken(n + 1) + require.NoError(t, c.Put(key, Entry{Token: tooBig})) + assert.Equal(t, itemsOf(key, 2), presentItems(t, backend, key)) + got, err := c.Lookup(key) + require.NoError(t, err) + assert.Equal(t, tooBig.AccessToken, got.Token.AccessToken) +} diff --git a/libs/auth/storage/keyring_size_limit_test.go b/libs/auth/storage/keyring_size_limit_test.go index aefe3cd100d..d07c7f9bb90 100644 --- a/libs/auth/storage/keyring_size_limit_test.go +++ b/libs/auth/storage/keyring_size_limit_test.go @@ -8,7 +8,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/zalando/go-keyring" "golang.org/x/oauth2" ) @@ -58,25 +57,25 @@ func shellSafe(r rune) bool { return strings.ContainsRune("_@%+=:,./-", r) } +// osSecretCaps are go-keyring's per-OS secret caps as fakeBackend models them. +var osSecretCaps = []struct { + name string + secretTooBig func(service, account, secret string) bool +}{ + {name: "darwin security -i line", secretTooBig: darwinSecurityCommandTooBig}, + {name: "windows credential blob", secretTooBig: windowsCredentialBlobTooBig}, + {name: "linux secret service has no cap", secretTooBig: nil}, +} + // TestKeyringStore_Put_OSSecretSizeCaps pins how the CLI's keyring envelope // fares against go-keyring's per-OS secret caps. The caps are modeled in // fakeBackend so the test is keychain-agnostic and runs on every OS; -// TestKeyringStore_Put_RealBackendRejectsLargeToken in +// TestKeyringSet_RejectsLargeSecret in // keyring_size_limit_os_test.go checks the real library on CI and fails when // the modeled caps go stale. A fix for #7707 that changes what Put stores // must update this test on purpose. func TestKeyringStore_Put_OSSecretSizeCaps(t *testing.T) { - cases := []struct { - name string - secretTooBig func(service, account, secret string) bool - largeRejected bool - }{ - {name: "darwin security -i line", secretTooBig: darwinSecurityCommandTooBig, largeRejected: true}, - {name: "windows credential blob", secretTooBig: windowsCredentialBlobTooBig, largeRejected: true}, - {name: "linux secret service has no cap", secretTooBig: nil, largeRejected: false}, - } - - for _, tc := range cases { + for _, tc := range osSecretCaps { t.Run(tc.name, func(t *testing.T) { backend := newFakeBackend() backend.secretTooBig = tc.secretTooBig @@ -91,12 +90,36 @@ func TestKeyringStore_Put_OSSecretSizeCaps(t *testing.T) { // The access token alone exceeds both caps (macOS after base64, // Windows raw), so no other token field is needed. large := &oauth2.Token{AccessToken: strings.Repeat("a", issue7707LargeAccessTokenBytes)} - err = store.Put("my-profile", Entry{Token: large}) - if tc.largeRejected { - require.ErrorIs(t, err, keyring.ErrSetDataTooBig) - } else { - require.NoError(t, err) - } + require.NoError(t, store.Put("my-profile", Entry{Token: large})) + got, err = store.Lookup("my-profile") + require.NoError(t, err) + assert.Equal(t, large.AccessToken, got.Token.AccessToken) }) } } + +// TestDarwinCommandTooBig pins that the macOS size check, which lets the store +// skip go-keyring, only rejects a secret that go-keyring rejects too. The check +// leaves out the shell quotes that go-keyring adds, so it is exact for an +// account that needs none. +func TestDarwinCommandTooBig(t *testing.T) { + accounts := []string{ + "my-profile", + "https://accounts.cloud.databricks.test/oidc/accounts/968367da-7edd-44f7-9dea-3e0b20b0ec97", + // go-keyring quotes these. + "my profile", + "my-profile#access-token-0", + } + for _, account := range accounts { + quoted := shellQuote(account) != account + for n := 2800; n <= 3100; n++ { + secret := strings.Repeat("a", n) + got := darwinCommandTooBig(keyringServiceName, account, secret) + want := darwinSecurityCommandTooBig(keyringServiceName, account, secret) + require.False(t, got && !want, "account %q, %d-byte secret: rejected, but go-keyring stores it", account, n) + if !quoted { + require.Equal(t, want, got, "account %q, %d-byte secret", account, n) + } + } + } +} diff --git a/libs/auth/storage/keyring_test.go b/libs/auth/storage/keyring_test.go index 960000cc2f6..421edd57fa6 100644 --- a/libs/auth/storage/keyring_test.go +++ b/libs/auth/storage/keyring_test.go @@ -3,6 +3,10 @@ package storage import ( "encoding/json" "errors" + "fmt" + "slices" + "strconv" + "strings" "testing" "time" @@ -25,9 +29,17 @@ type fakeBackend struct { getBlock bool deleteBlock bool + // errAccount limits the errors and blocks above to accounts that contain + // it, such as AccessTokenPartMarker for the access token parts. + errAccount string + // secretTooBig, when set, mirrors an OS backend's size check: Set rejects // the secret with keyring.ErrSetDataTooBig before storing anything. secretTooBig func(service, account, secret string) bool + + // afterGet, when set, runs after each Get has read the item, so that a + // test can change the items between two reads as another process would. + afterGet func(account string) } func newFakeBackend() *fakeBackend { @@ -36,14 +48,28 @@ func newFakeBackend() *fakeBackend { func itemKey(service, account string) string { return service + ":" + account } +// accounts returns the sorted accounts of the items in the databricks-cli +// service. +func (f *fakeBackend) accounts() []string { + var accounts []string + for k := range f.items { + if account, ok := strings.CutPrefix(k, itemKey(keyringServiceName, "")); ok { + accounts = append(accounts, account) + } + } + slices.Sort(accounts) + return accounts +} + func (f *fakeBackend) Set(service, account, secret string) error { - if f.setBlock { + matches := strings.Contains(account, f.errAccount) + if f.setBlock && matches { select {} } if f.secretTooBig != nil && f.secretTooBig(service, account, secret) { return keyring.ErrSetDataTooBig } - if f.setErr != nil { + if f.setErr != nil && matches { return f.setErr } f.items[itemKey(service, account)] = secret @@ -51,13 +77,17 @@ func (f *fakeBackend) Set(service, account, secret string) error { } func (f *fakeBackend) Get(service, account string) (string, error) { - if f.getBlock { + matches := strings.Contains(account, f.errAccount) + if f.getBlock && matches { select {} } - if f.getErr != nil { + if f.getErr != nil && matches { return "", f.getErr } v, ok := f.items[itemKey(service, account)] + if f.afterGet != nil { + f.afterGet(account) + } if !ok { return "", keyring.ErrNotFound } @@ -65,13 +95,18 @@ func (f *fakeBackend) Get(service, account string) (string, error) { } func (f *fakeBackend) Delete(service, account string) error { - if f.deleteBlock { + matches := strings.Contains(account, f.errAccount) + if f.deleteBlock && matches { select {} } - if f.deleteErr != nil { + if f.deleteErr != nil && matches { return f.deleteErr } - delete(f.items, itemKey(service, account)) + k := itemKey(service, account) + if _, ok := f.items[k]; !ok { + return keyring.ErrNotFound + } + delete(f.items, k) return nil } @@ -94,6 +129,8 @@ func newTestStoreWithTimeout(backend keyringBackend, timeout time.Duration) *key backend: backend, timeout: timeout, keyringSvcName: "databricks-cli", + // Tests make their own concurrent writes, so Lookup need not wait. + retryDelay: time.Nanosecond, } } @@ -270,6 +307,604 @@ func TestKeyringStore_StoreNil_TimesOut(t *testing.T) { assert.ErrorAs(t, err, &timeoutErr, "expected TimeoutError, got %T: %v", err, err) } +// testAccessToken returns an access token of n bytes in which no two parts +// are alike, so that a test notices parts in the wrong order. +func testAccessToken(n int) string { + var b strings.Builder + for i := 0; b.Len() < n; i++ { + b.WriteString(strconv.Itoa(i)) + b.WriteByte('.') + } + return b.String()[:n] +} + +// testToken returns a token with every field set and an access token of +// accessTokenSize bytes. +func testToken(accessTokenSize int) *oauth2.Token { + return &oauth2.Token{ + AccessToken: testAccessToken(accessTokenSize), + TokenType: "Bearer", + RefreshToken: "refresh-token", + Expiry: time.Date(2026, 11, 17, 12, 0, 0, 0, time.UTC), + } +} + +// tooBigForAnyOS applies the macOS and the Windows cap together, because a +// token has to fit both for the store to behave the same everywhere. +func tooBigForAnyOS(service, account, secret string) bool { + return darwinSecurityCommandTooBig(service, account, secret) || windowsCredentialBlobTooBig(service, account, secret) +} + +// newLimitedTestStore returns a store on a fake backend that applies the +// macOS and the Windows cap. +func newLimitedTestStore() (*keyringStore, *fakeBackend) { + backend := newFakeBackend() + backend.secretTooBig = tooBigForAnyOS + return newTestStore(backend), backend +} + +// releasedEntry returns the item that released CLI versions write for tok and +// read back. +func releasedEntry(t *testing.T, tok *oauth2.Token) string { + t.Helper() + raw, err := json.Marshal(map[string]*oauth2.Token{"token": tok}) + require.NoError(t, err) + return string(raw) +} + +// itemsOf returns key followed by the keys of its first n access token parts. +func itemsOf(key string, n int) []string { + items := []string{key} + for i := range n { + items = append(items, accessTokenPartKey(key, i)) + } + return items +} + +// maxTestParts is how many access token parts presentItems looks for, one +// more than the largest test token has. +const maxTestParts = 4 + +// presentItems returns which of key and its first maxTestParts access token +// parts the backend holds. It only calls Get, so it also inspects the real OS +// keyring, which cannot list its items. +func presentItems(t *testing.T, backend keyringBackend, key string) []string { + t.Helper() + var present []string + for _, account := range itemsOf(key, maxTestParts) { + _, err := backend.Get(keyringServiceName, account) + if errors.Is(err, keyring.ErrNotFound) { + continue + } + require.NoError(t, err) + present = append(present, account) + } + return present +} + +// largestAccessTokenInOneItem returns the size of the largest access token for +// which the released entry of testToken under key passes secretTooBig. +func largestAccessTokenInOneItem(t *testing.T, secretTooBig func(service, account, secret string) bool, key string) int { + t.Helper() + n := 0 + for !secretTooBig(keyringServiceName, key, releasedEntry(t, testToken(n+1))) { + n++ + } + return n +} + +// Other CLI versions read the same entry, so a token that fits must keep the +// format they wrote and understand, under every OS cap. +func TestKeyringStore_Put_TokenThatFitsKeepsSingleItemFormat(t *testing.T) { + for _, tc := range osSecretCaps { + t.Run(tc.name, func(t *testing.T) { + backend := newFakeBackend() + backend.secretTooBig = tc.secretTooBig + c := newTestStore(backend) + tok := testToken(issue7707SmallAccessTokenBytes) + + require.NoError(t, c.Put("my-profile", Entry{Token: tok})) + assert.Equal(t, map[string]string{itemKey(keyringServiceName, "my-profile"): releasedEntry(t, tok)}, backend.items) + }) + } +} + +// A token is split only when the backend rejects it as one item: the largest +// token that fits stays in the released format, and one byte more is split. +// The macOS cap counts the account too, so less fits under a long cache key +// such as a host URL. +func TestKeyringStore_Put_SplitsOnlyTokenThatDoesNotFit(t *testing.T) { + keys := []struct { + name string + key string + }{ + {name: "profile key", key: "my-profile"}, + {name: "host key", key: "https://accounts.cloud.databricks.test/oidc/accounts/968367da-7edd-44f7-9dea-3e0b20b0ec97"}, + } + + for _, tc := range osSecretCaps { + if tc.secretTooBig == nil { + continue + } + for _, k := range keys { + t.Run(tc.name+", "+k.name, func(t *testing.T) { + n := largestAccessTokenInOneItem(t, tc.secretTooBig, k.key) + backend := newFakeBackend() + backend.secretTooBig = tc.secretTooBig + c := newTestStore(backend) + + fits := testToken(n) + require.NoError(t, c.Put(k.key, Entry{Token: fits})) + assert.Equal(t, map[string]string{itemKey(keyringServiceName, k.key): releasedEntry(t, fits)}, backend.items) + + tooBig := testToken(n + 1) + require.NoError(t, c.Put(k.key, Entry{Token: tooBig})) + assert.Equal(t, itemsOf(k.key, 2), backend.accounts()) + got, err := c.Lookup(k.key) + require.NoError(t, err) + assert.Equal(t, tooBig.AccessToken, got.Token.AccessToken) + }) + } + } +} + +func TestKeyringStore_Put_SplitsTokenTooBigForOneItem(t *testing.T) { + cases := []struct { + name string + accessTokenSize int + wantParts int + }{ + {name: "over the Windows limit", accessTokenSize: 2600, wantParts: 2}, + // https://github.com/databricks/cli/issues/7707 + {name: "over the macOS limit", accessTokenSize: issue7707LargeAccessTokenBytes, wantParts: 2}, + {name: "three access token parts", accessTokenSize: 5000, wantParts: 3}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + c, backend := newLimitedTestStore() + want := testToken(tc.accessTokenSize) + + require.NoError(t, c.Put("my-profile", Entry{Token: want})) + assert.Equal(t, itemsOf("my-profile", tc.wantParts), backend.accounts()) + + got, err := c.Lookup("my-profile") + require.NoError(t, err) + assert.Equal(t, want.AccessToken, got.Token.AccessToken) + assert.Equal(t, want.TokenType, got.Token.TokenType) + assert.Equal(t, want.RefreshToken, got.Token.RefreshToken) + assert.True(t, want.Expiry.Equal(got.Token.Expiry)) + }) + } +} + +// CLI versions without split entries read the entry as {"token": ...}. They +// need a token there because releases before v1.24 dereference it, and it +// must not carry the refresh token: they would rotate it and then fail to +// store the oversized result, which logs the user out. +func TestKeyringStore_Put_SplitEntryGivesOlderVersionsNoCredentials(t *testing.T) { + c, backend := newLimitedTestStore() + + require.NoError(t, c.Put("my-profile", Entry{Token: testToken(issue7707LargeAccessTokenBytes)})) + + var old struct { + Token *oauth2.Token `json:"token"` + } + require.NoError(t, json.Unmarshal([]byte(backend.items[itemKey(keyringServiceName, "my-profile")]), &old)) + require.NotNil(t, old.Token) + assert.Empty(t, old.Token.AccessToken) + assert.Empty(t, old.Token.RefreshToken) +} + +// Entry holds only a token and a pending login today, and the two never come +// together. The pending login stands in for fields that Entry may get later: +// a split token must keep them, as a token that fits does. +func TestKeyringStore_SplitEntryKeepsOtherEntryFields(t *testing.T) { + c, _ := newLimitedTestStore() + pending := &PendingLogin{DeviceCode: "device-code", UserCode: "WDJBMJHT"} + tok := testToken(issue7707LargeAccessTokenBytes) + + require.NoError(t, c.Put("my-profile", Entry{Token: tok, PendingLogin: pending})) + + got, err := c.Lookup("my-profile") + require.NoError(t, err) + assert.Equal(t, tok.AccessToken, got.Token.AccessToken) + assert.Equal(t, pending, got.PendingLogin) +} + +// The entry is written before the access token parts because it holds the +// refresh token, which the server replaces on every refresh. When a part write +// fails, whatever the key held before, the new refresh token is stored and +// read back without an access token. +func TestKeyringStore_Put_StoresRefreshTokenBeforeAccessToken(t *testing.T) { + boom := errors.New("backend boom") + before := []struct { + name string + accessTokenSize int + }{ + {name: "no entry"}, + {name: "token that fits", accessTokenSize: issue7707SmallAccessTokenBytes}, + {name: "split token", accessTokenSize: 5000}, + } + + for _, b := range before { + for _, failingPart := range []int{0, 1} { + t.Run(fmt.Sprintf("%s, part %d fails", b.name, failingPart), func(t *testing.T) { + c, backend := newLimitedTestStore() + if b.accessTokenSize > 0 { + old := testToken(b.accessTokenSize) + old.RefreshToken = "old-refresh-token" + require.NoError(t, c.Put("my-profile", Entry{Token: old})) + } + backend.setErr = boom + backend.errAccount = accessTokenPartKey("my-profile", failingPart) + tok := testToken(issue7707LargeAccessTokenBytes) + + err := c.Put("my-profile", Entry{Token: tok}) + require.ErrorIs(t, err, boom) + assert.ErrorIs(t, err, ErrAccessTokenNotStored) + + got, err := c.Lookup("my-profile") + require.NoError(t, err) + assert.Equal(t, tok.RefreshToken, got.Token.RefreshToken) + assert.Empty(t, got.Token.AccessToken) + }) + } + } +} + +// Only the access token is split. Anything else that is too big for one item +// still fails, and nothing is written. +func TestKeyringStore_Put_SplitsOnlyTheAccessToken(t *testing.T) { + cases := []struct { + name string + key string + entry Entry + }{ + { + name: "pending login", + key: "my-profile#device-code", + entry: Entry{PendingLogin: &PendingLogin{DeviceCode: strings.Repeat("x", 5000)}}, + }, + { + name: "refresh token", + key: "my-profile", + entry: Entry{Token: &oauth2.Token{AccessToken: testAccessToken(issue7707LargeAccessTokenBytes), RefreshToken: strings.Repeat("x", 5000)}}, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + c, backend := newLimitedTestStore() + + err := c.Put(tc.key, tc.entry) + assert.ErrorIs(t, err, keyring.ErrSetDataTooBig) + assert.NotErrorIs(t, err, ErrAccessTokenNotStored) + assert.Empty(t, backend.items) + }) + } +} + +// editSplitEntry rewrites the split entry under my-profile. +func editSplitEntry(t *testing.T, backend *fakeBackend, edit func(*splitToken)) { + t.Helper() + k := itemKey(keyringServiceName, "my-profile") + var e keyringEntry + require.NoError(t, json.Unmarshal([]byte(backend.items[k]), &e)) + require.NotNil(t, e.Split) + edit(e.Split) + raw, err := json.Marshal(e) + require.NoError(t, err) + backend.items[k] = string(raw) +} + +// The access token parts are written one at a time, so a reader can find them +// incomplete or belonging to another write. The refresh token is enough to +// recover, so the lookup succeeds without an access token. +func TestKeyringStore_Lookup_SplitEntryWithoutMatchingAccessToken(t *testing.T) { + second := itemKey(keyringServiceName, "my-profile#access-token-1") + cases := []struct { + name string + change func(t *testing.T, backend *fakeBackend) + }{ + { + name: "part missing", + change: func(t *testing.T, backend *fakeBackend) { delete(backend.items, second) }, + }, + { + name: "part of another write", + change: func(t *testing.T, backend *fakeBackend) { backend.items[second] = "another access token" }, + }, + { + name: "no parts counted", + change: func(t *testing.T, backend *fakeBackend) { + editSplitEntry(t, backend, func(s *splitToken) { s.AccessTokenParts = 0 }) + }, + }, + { + name: "fewer parts counted than written", + change: func(t *testing.T, backend *fakeBackend) { + editSplitEntry(t, backend, func(s *splitToken) { s.AccessTokenParts = 1 }) + }, + }, + { + name: "more parts counted than written", + change: func(t *testing.T, backend *fakeBackend) { + editSplitEntry(t, backend, func(s *splitToken) { s.AccessTokenParts = 3 }) + }, + }, + { + name: "digest of another access token", + change: func(t *testing.T, backend *fakeBackend) { + editSplitEntry(t, backend, func(s *splitToken) { s.AccessTokenSHA256 = strings.Repeat("0", 64) }) + }, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + c, backend := newLimitedTestStore() + tok := testToken(issue7707LargeAccessTokenBytes) + require.NoError(t, c.Put("my-profile", Entry{Token: tok})) + require.Equal(t, itemsOf("my-profile", 2), backend.accounts()) + + tc.change(t, backend) + + got, err := c.Lookup("my-profile") + require.NoError(t, err) + assert.Empty(t, got.Token.AccessToken) + assert.Equal(t, tok.RefreshToken, got.Token.RefreshToken) + assert.Equal(t, tok.TokenType, got.Token.TokenType) + assert.True(t, tok.Expiry.Equal(got.Token.Expiry)) + }) + } +} + +// splitItems returns the items that the store writes for tok under +// my-profile. +func splitItems(t *testing.T, tok *oauth2.Token) map[string]string { + t.Helper() + c, backend := newLimitedTestStore() + require.NoError(t, c.Put("my-profile", Entry{Token: tok})) + return backend.items +} + +// Another process can replace a split token while Lookup reads it. Here it +// writes its entry and first part after Lookup read the old entry, and its +// second part after Lookup read the old one. Lookup reads the entry again and +// returns the new token, rather than the old refresh token, which the other +// process has already used up. +func TestKeyringStore_Lookup_WaitsForConcurrentWrite(t *testing.T) { + c, backend := newLimitedTestStore() + old := testToken(issue7707LargeAccessTokenBytes) + old.RefreshToken = "old-refresh-token" + require.NoError(t, c.Put("my-profile", Entry{Token: old})) + + updated := testToken(issue7707LargeAccessTokenBytes) + updated.AccessToken = strings.ReplaceAll(updated.AccessToken, ".", "-") + updated.RefreshToken = "new-refresh-token" + written := splitItems(t, updated) + entry := itemKey(keyringServiceName, "my-profile") + part0 := itemKey(keyringServiceName, "my-profile#access-token-0") + part1 := itemKey(keyringServiceName, "my-profile#access-token-1") + reads := 0 + backend.afterGet = func(string) { + reads++ + switch reads { + case 1: + backend.items[entry] = written[entry] + backend.items[part0] = written[part0] + case 3: + backend.items[part1] = written[part1] + } + } + + got, err := c.Lookup("my-profile") + require.NoError(t, err) + assert.Equal(t, updated.AccessToken, got.Token.AccessToken) + assert.Equal(t, updated.RefreshToken, got.Token.RefreshToken) +} + +// auth logout can delete a split token while Lookup reads it. Lookup reads the +// entry again and finds it gone, rather than returning the deleted refresh +// token, which the caller would refresh and store again. +func TestKeyringStore_Lookup_SplitEntryDeletedDuringLookup(t *testing.T) { + c, backend := newLimitedTestStore() + require.NoError(t, c.Put("my-profile", Entry{Token: testToken(issue7707LargeAccessTokenBytes)})) + backend.afterGet = func(account string) { + if account == "my-profile#access-token-0" { + clear(backend.items) + } + } + + _, err := c.Lookup("my-profile") + assert.ErrorIs(t, err, ErrNotFound) +} + +// A split entry whose parts stay incomplete, as after an interrupted write, is +// read splitLookupAttempts times and then returned without its access token. +func TestKeyringStore_Lookup_GivesUpOnIncompleteSplitEntry(t *testing.T) { + c, backend := newLimitedTestStore() + tok := testToken(issue7707LargeAccessTokenBytes) + require.NoError(t, c.Put("my-profile", Entry{Token: tok})) + delete(backend.items, itemKey(keyringServiceName, "my-profile#access-token-1")) + entryReads := 0 + backend.afterGet = func(account string) { + if account == "my-profile" { + entryReads++ + } + } + + got, err := c.Lookup("my-profile") + require.NoError(t, err) + assert.Empty(t, got.Token.AccessToken) + assert.Equal(t, tok.RefreshToken, got.Token.RefreshToken) + assert.Equal(t, splitLookupAttempts, entryReads) +} + +// A store without a retry delay, as NewKeyringStore builds it, waits 25, 50, +// 100 and 200 milliseconds between its reads of an incomplete split entry, to +// give a concurrent writer time to finish. +func TestKeyringStore_Lookup_WaitsBetweenReadsByDefault(t *testing.T) { + backend := newFakeBackend() + backend.secretTooBig = tooBigForAnyOS + c := &keyringStore{backend: backend, timeout: testTimeout, keyringSvcName: keyringServiceName} + require.NoError(t, c.Put("my-profile", Entry{Token: testToken(issue7707LargeAccessTokenBytes)})) + delete(backend.items, itemKey(keyringServiceName, "my-profile#access-token-1")) + + start := time.Now() + _, err := c.Lookup("my-profile") + require.NoError(t, err) + assert.GreaterOrEqual(t, time.Since(start), 375*time.Millisecond) +} + +func TestKeyringStore_Lookup_SplitEntryPropagatesAccessTokenErrors(t *testing.T) { + boom := errors.New("backend boom") + c, backend := newLimitedTestStore() + require.NoError(t, c.Put("my-profile", Entry{Token: testToken(issue7707LargeAccessTokenBytes)})) + backend.getErr = boom + backend.errAccount = AccessTokenPartMarker + + _, err := c.Lookup("my-profile") + require.ErrorIs(t, err, boom) + assert.Contains(t, err.Error(), "OS keyring unreachable") +} + +func TestKeyringStore_Delete_PropagatesAccessTokenPartErrors(t *testing.T) { + boom := errors.New("backend boom") + c, backend := newLimitedTestStore() + require.NoError(t, c.Put("my-profile", Entry{Token: testToken(issue7707LargeAccessTokenBytes)})) + backend.deleteErr = boom + backend.errAccount = AccessTokenPartMarker + + err := c.Delete("my-profile") + assert.ErrorIs(t, err, boom) +} + +// Every access token part is read, written and deleted under the timeout of +// a single keyring operation. +func TestKeyringStore_AccessTokenPartTimesOut(t *testing.T) { + cases := []struct { + name string + block func(*fakeBackend) + run func(*keyringStore) error + }{ + { + name: "lookup", + block: func(b *fakeBackend) { b.getBlock = true }, + run: func(c *keyringStore) error { + _, err := c.Lookup("my-profile") + return err + }, + }, + { + name: "put", + block: func(b *fakeBackend) { b.setBlock = true }, + run: func(c *keyringStore) error { + return c.Put("my-profile", Entry{Token: testToken(5000)}) + }, + }, + { + name: "delete", + block: func(b *fakeBackend) { b.deleteBlock = true }, + run: func(c *keyringStore) error { + return c.Delete("my-profile") + }, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + c, backend := newLimitedTestStore() + require.NoError(t, c.Put("my-profile", Entry{Token: testToken(issue7707LargeAccessTokenBytes)})) + backend.errAccount = AccessTokenPartMarker + tc.block(backend) + c.timeout = blockedTimeout + + err := tc.run(c) + + var timeoutErr *TimeoutError + assert.ErrorAs(t, err, &timeoutErr, "expected TimeoutError, got %T: %v", err, err) + }) + } +} + +// assertTokenSizeTransitions stores tokens of changing sizes under key, checks +// that each one reads back and which items exist, and then deletes key. +func assertTokenSizeTransitions(t *testing.T, c *keyringStore, backend keyringBackend, key string) { + t.Helper() + steps := []struct { + name string + accessTokenSize int + wantParts int + }{ + {name: "token that fits", accessTokenSize: issue7707SmallAccessTokenBytes, wantParts: 0}, + {name: "split into three parts", accessTokenSize: 5000, wantParts: 3}, + // The third part stays behind, unused: the entry counts two. + {name: "split into two parts", accessTokenSize: issue7707LargeAccessTokenBytes, wantParts: 3}, + // All parts stay behind until Delete. + {name: "token that fits again", accessTokenSize: issue7707SmallAccessTokenBytes, wantParts: 3}, + } + + for i, s := range steps { + tok := testToken(s.accessTokenSize) + tok.RefreshToken = fmt.Sprintf("refresh-token-%d", i) + require.NoError(t, c.Put(key, Entry{Token: tok}), s.name) + + got, err := c.Lookup(key) + require.NoError(t, err, s.name) + assert.Equal(t, tok.AccessToken, got.Token.AccessToken, s.name) + assert.Equal(t, tok.RefreshToken, got.Token.RefreshToken, s.name) + assert.Equal(t, itemsOf(key, s.wantParts), presentItems(t, backend, key), s.name) + } + + require.NoError(t, c.Delete(key)) + assert.Empty(t, presentItems(t, backend, key)) +} + +// The access token's size changes between refreshes, so one key moves between +// a token that fits and a split one. Delete also removes the parts that the +// entry no longer counts. +func TestKeyringStore_TokenSizeTransitions(t *testing.T) { + c, backend := newLimitedTestStore() + assertTokenSizeTransitions(t, c, backend, "my-profile") + assert.Empty(t, backend.items) +} + +// An older CLI version that logs out deletes only the entry, so Delete also +// removes access token parts without one. +func TestKeyringStore_Delete_RemovesPartsWithoutEntry(t *testing.T) { + c, backend := newLimitedTestStore() + require.NoError(t, c.Put("my-profile", Entry{Token: testToken(5000)})) + delete(backend.items, itemKey(keyringServiceName, "my-profile")) + + require.NoError(t, c.Delete("my-profile")) + assert.Empty(t, backend.items) +} + +func TestKeyringStore_Delete_MissingEntryIsNoOp(t *testing.T) { + c, backend := newLimitedTestStore() + require.NoError(t, c.Put("other-profile", Entry{Token: testToken(5000)})) + + require.NoError(t, c.Delete("my-profile")) + assert.Equal(t, itemsOf("other-profile", 3), backend.accounts()) +} + +// A Delete that fails part way, here on the second of three parts, leaves +// parts that the next Delete still finds, so a retried logout removes them. +func TestKeyringStore_Delete_RetryAfterFailedPartDelete(t *testing.T) { + boom := errors.New("backend boom") + c, backend := newLimitedTestStore() + require.NoError(t, c.Put("my-profile", Entry{Token: testToken(5000)})) + backend.deleteErr = boom + backend.errAccount = accessTokenPartKey("my-profile", 1) + require.ErrorIs(t, c.Delete("my-profile"), boom) + + backend.deleteErr = nil + require.NoError(t, c.Delete("my-profile")) + assert.Empty(t, backend.items) +} + func TestProbeKeyring(t *testing.T) { boom := errors.New("backend boom") cases := []struct { diff --git a/libs/auth/storage/storage.go b/libs/auth/storage/storage.go index c820d732134..2b9d118c4a6 100644 --- a/libs/auth/storage/storage.go +++ b/libs/auth/storage/storage.go @@ -22,7 +22,8 @@ var ErrNotFound = errors.New("token not found") // Entry is the value held in the CLI token store. It wraps the credential so // the schema can grow additive metadata (e.g. a config fingerprint, scopes) // without changing the Store interface. Backends persist it verbatim and never -// interpret its contents. +// interpret its contents. The exception is the keyring store, which keeps an +// access token too large for one item in separate items. type Entry struct { // Token is the cached OAuth token. Set for every stored entry except a // pending login. @@ -61,9 +62,16 @@ type PendingLogin struct { // metadata without changing the Store interface. type Store interface { // Put writes e under key, replacing any existing entry. + // + // The keyring store can fail with ErrAccessTokenNotStored after it stored + // everything but the access token. Put(key string, e Entry) error // Lookup returns the entry stored under key, or ErrNotFound. + // + // The keyring store returns a token without its access token when the + // items that hold the access token are missing or stale. Such a token is + // not valid, so the caller refreshes it. Lookup(key string) (Entry, error) // Delete removes the entry under key. Deleting a missing entry is not an diff --git a/libs/auth/u2m/persistent_auth.go b/libs/auth/u2m/persistent_auth.go index 6039fb02f47..6339d134294 100644 --- a/libs/auth/u2m/persistent_auth.go +++ b/libs/auth/u2m/persistent_auth.go @@ -15,6 +15,7 @@ import ( "github.com/databricks/cli/libs/auth/storage" "github.com/databricks/cli/libs/browser" + "github.com/databricks/cli/libs/log" "github.com/databricks/databricks-sdk-go/httpclient" "github.com/databricks/databricks-sdk-go/logger" "golang.org/x/oauth2" @@ -456,6 +457,12 @@ func (a *PersistentAuth) refresh(oldToken *oauth2.Token) (*oauth2.Token, error) return nil, oauthError(err) } err = a.store.Put(a.oAuthArgument.GetCacheKey(), storage.Entry{Token: t}) + if errors.Is(err, storage.ErrAccessTokenNotStored) { + // The new refresh token is stored, which is what the next command + // needs. Without the access token, it refreshes again. + log.Debugf(a.ctx, "token refresh: %v", err) + return t, nil + } if err != nil { if cached := a.recoverStoreUpdate(oldToken, t); cached != nil { return cached, nil diff --git a/libs/auth/u2m/persistent_auth_keyring_os_test.go b/libs/auth/u2m/persistent_auth_keyring_os_test.go new file mode 100644 index 00000000000..7ca50edd3eb --- /dev/null +++ b/libs/auth/u2m/persistent_auth_keyring_os_test.go @@ -0,0 +1,91 @@ +//go:build darwin || windows + +package u2m + +import ( + "net/http" + "net/url" + "os" + "strconv" + "strings" + "testing" + "time" + "uuid" + + "github.com/databricks/cli/libs/auth/storage" + "github.com/databricks/databricks-sdk-go/httpclient/fixtures" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/zalando/go-keyring" + "golang.org/x/oauth2" +) + +// TestToken_RefreshesLargeTokenInOSKeyring refreshes an expired token that the +// real OS keyring holds in several items, because its access token is too +// large for one, and reads the new access token and refresh token back. It +// uses the store that the CLI uses, with its 3-second timeout per keyring +// operation. +// +// It writes to the real OS keyring and therefore runs on CI only (GitHub +// Actions sets CI=true). +func TestToken_RefreshesLargeTokenInOSKeyring(t *testing.T) { + if os.Getenv("CI") == "" { + t.Skip("touches the real OS keyring; runs on CI only") + } + + // The keyring store keeps every item in this service. + const service = "databricks-cli" + profile := "__probe_" + uuid.New().String() + t.Cleanup(func() { + _ = keyring.Delete(service, profile) + for i := range 4 { + _ = keyring.Delete(service, profile+storage.AccessTokenPartMarker+strconv.Itoa(i)) + } + }) + + // The size of the access tokens in https://github.com/databricks/cli/issues/7707. + const accessTokenSize = 3250 + tokenStore := storage.NewKeyringStore() + err := tokenStore.Put(profile, storage.Entry{Token: &oauth2.Token{ + AccessToken: strings.Repeat("a", accessTokenSize), + RefreshToken: "first", + Expiry: time.Now().Add(-time.Minute), + }}) + require.NoError(t, err) + + refreshed := strings.Repeat("b", accessTokenSize) + arg, err := NewProfileWorkspaceOAuthArgument("https://keyring-refresh.cloud.databricks.test", profile) + require.NoError(t, err) + p, err := NewPersistentAuth( + t.Context(), + WithTokenStore(tokenStore), + WithHttpClient(&http.Client{ + Transport: fixtures.SliceTransport{ + { + Method: "POST", + Resource: "/oidc/v1/token", + ExpectedRequest: url.Values{"client_id": {"databricks-cli"}, "grant_type": {"refresh_token"}, "refresh_token": {"first"}}, + Response: "access_token=" + refreshed + "&refresh_token=second&expires_in=3600", + ResponseHeaders: map[string][]string{ + "Content-Type": {"application/x-www-form-urlencoded"}, + }, + }, + }, + }), + WithOAuthEndpointSupplier(MockOAuthEndpointSupplier{}), + WithOAuthArgument(arg), + ) + require.NoError(t, err) + defer p.Close() + + tok, err := p.Token() + require.NoError(t, err) + assert.Equal(t, refreshed, tok.AccessToken) + + _, err = keyring.Get(service, profile+storage.AccessTokenPartMarker+"1") + assert.NoError(t, err, "the refreshed token was not split") + stored, err := storage.NewKeyringStore().Lookup(profile) + require.NoError(t, err) + assert.Equal(t, refreshed, stored.Token.AccessToken) + assert.Equal(t, "second", stored.Token.RefreshToken) +} diff --git a/libs/auth/u2m/persistent_auth_test.go b/libs/auth/u2m/persistent_auth_test.go index 9b28b754487..17e5a40f2a6 100644 --- a/libs/auth/u2m/persistent_auth_test.go +++ b/libs/auth/u2m/persistent_auth_test.go @@ -371,21 +371,25 @@ func TestToken_RefreshesTokenExpiringSoon(t *testing.T) { func TestToken_WithoutRefresh(t *testing.T) { tests := []struct { - name string - expiry time.Duration - wantTok string - wantErr error + name string + accessToken string + expiry time.Duration + wantTok string + wantErr error }{ - {name: "valid token is returned", expiry: time.Hour, wantTok: "cached"}, - {name: "token expiring soon is returned, not refreshed", expiry: 4 * time.Minute, wantTok: "cached"}, - {name: "expired token fails, not refreshed", expiry: -time.Minute, wantErr: ErrRefreshDisabled}, + {name: "valid token is returned", accessToken: "cached", expiry: time.Hour, wantTok: "cached"}, + {name: "token expiring soon is returned, not refreshed", accessToken: "cached", expiry: 4 * time.Minute, wantTok: "cached"}, + {name: "expired token fails, not refreshed", accessToken: "cached", expiry: -time.Minute, wantErr: ErrRefreshDisabled}, + // The keyring store returns a token without its access token when the + // parts of a split access token are missing or stale. + {name: "token without access token fails, not refreshed", expiry: time.Hour, wantErr: ErrRefreshDisabled}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { cache := &tokenStoreMock{ lookup: func(string) (*oauth2.Token, error) { return &oauth2.Token{ - AccessToken: "cached", + AccessToken: tt.accessToken, RefreshToken: "cde", Expiry: time.Now().Add(tt.expiry), }, nil @@ -430,6 +434,160 @@ func TestToken_WithoutRefresh(t *testing.T) { } } +// The keyring store returns a token without its access token when the parts +// of a split access token are missing or stale. Such a token is not valid, so +// Token refreshes it although it has not expired. +func TestToken_RefreshesTokenWithoutAccessToken(t *testing.T) { + const key = "https://accounts.cloud.databricks.test/oidc/accounts/xyz" + tokenStore := storage.NewMemoryStore() + err := tokenStore.Put(key, storage.Entry{Token: &oauth2.Token{ + RefreshToken: "cde", + Expiry: time.Now().Add(time.Hour), + }}) + if err != nil { + t.Fatalf("Put(): want no error, got %v", err) + } + arg, err := NewBasicAccountOAuthArgument("https://accounts.cloud.databricks.test", "xyz") + if err != nil { + t.Fatalf("NewBasicAccountOAuthArgument(): want no error, got %v", err) + } + p, err := NewPersistentAuth( + t.Context(), + WithTokenStore(tokenStore), + WithHttpClient(&http.Client{ + Transport: fixtures.SliceTransport{ + { + Method: "POST", + Resource: "/oidc/accounts/xyz/v1/token", + ExpectedRequest: url.Values{"client_id": {"databricks-cli"}, "grant_type": {"refresh_token"}, "refresh_token": {"cde"}}, + Response: `access_token=refreshed&refresh_token=def&expires_in=3600`, + ResponseHeaders: map[string][]string{ + "Content-Type": {"application/x-www-form-urlencoded"}, + }, + }, + }, + }), + WithOAuthEndpointSupplier(MockOAuthEndpointSupplier{}), + WithOAuthArgument(arg), + ) + if err != nil { + t.Fatalf("NewPersistentAuth(): want no error, got %v", err) + } + defer p.Close() + + tok, err := p.Token() + if err != nil { + t.Fatalf("p.Token(): want no error, got %v", err) + } + if tok.AccessToken != "refreshed" { + t.Errorf("p.Token(): want access token 'refreshed', got %q", tok.AccessToken) + } + stored, err := tokenStore.Lookup(key) + if err != nil { + t.Fatalf("Lookup(): want no error, got %v", err) + } + if stored.Token.AccessToken != "refreshed" || stored.Token.RefreshToken != "def" { + t.Errorf("Lookup(): want the refreshed token stored, got access token %q and refresh token %q", stored.Token.AccessToken, stored.Token.RefreshToken) + } +} + +// accessTokenPartFailingStore stands in for the keyring store when it fails +// to write a part of a split access token: the entry keeps the new refresh +// token but not the access token, and Put returns err wrapped in +// storage.ErrAccessTokenNotStored. +type accessTokenPartFailingStore struct { + storage.Store + err error +} + +func (s *accessTokenPartFailingStore) Put(key string, e storage.Entry) error { + if s.err == nil || e.Token == nil { + return s.Store.Put(key, e) + } + rest := *e.Token + rest.AccessToken = "" + if err := s.Store.Put(key, storage.Entry{Token: &rest}); err != nil { + return err + } + return fmt.Errorf("%w: %w", storage.ErrAccessTokenNotStored, s.err) +} + +// A refresh whose access token cannot be stored still succeeds, because the +// new refresh token was stored, and the next call refreshes with it. +// Refreshing with the old one would fail, because the server replaces the +// refresh token on every refresh. +func TestToken_RefreshesWithRefreshTokenStoredByFailedStoreUpdate(t *testing.T) { + const key = "https://accounts.cloud.databricks.test/oidc/accounts/xyz" + memory := storage.NewMemoryStore() + err := memory.Put(key, storage.Entry{Token: &oauth2.Token{ + AccessToken: "expired", + RefreshToken: "first", + Expiry: time.Now().Add(-time.Minute), + }}) + if err != nil { + t.Fatalf("Put(): want no error, got %v", err) + } + storeErr := errors.New("keyring set timed out") + tokenStore := &accessTokenPartFailingStore{Store: memory, err: storeErr} + arg, err := NewBasicAccountOAuthArgument("https://accounts.cloud.databricks.test", "xyz") + if err != nil { + t.Fatalf("NewBasicAccountOAuthArgument(): want no error, got %v", err) + } + headers := map[string][]string{"Content-Type": {"application/x-www-form-urlencoded"}} + p, err := NewPersistentAuth( + t.Context(), + WithTokenStore(tokenStore), + WithHttpClient(&http.Client{ + Transport: fixtures.SliceTransport{ + { + Method: "POST", + Resource: "/oidc/accounts/xyz/v1/token", + ExpectedRequest: url.Values{"client_id": {"databricks-cli"}, "grant_type": {"refresh_token"}, "refresh_token": {"first"}}, + Response: `access_token=second-access&refresh_token=second&expires_in=3600`, + ResponseHeaders: headers, + }, + { + Method: "POST", + Resource: "/oidc/accounts/xyz/v1/token", + ExpectedRequest: url.Values{"client_id": {"databricks-cli"}, "grant_type": {"refresh_token"}, "refresh_token": {"second"}}, + Response: `access_token=third-access&refresh_token=third&expires_in=3600`, + ResponseHeaders: headers, + }, + }, + }), + WithOAuthEndpointSupplier(MockOAuthEndpointSupplier{}), + WithOAuthArgument(arg), + ) + if err != nil { + t.Fatalf("NewPersistentAuth(): want no error, got %v", err) + } + defer p.Close() + + tok, err := p.Token() + if err != nil { + t.Fatalf("first p.Token(): want no error, got %v", err) + } + if tok.AccessToken != "second-access" { + t.Errorf("first p.Token(): want access token 'second-access', got %q", tok.AccessToken) + } + + tokenStore.err = nil + tok, err = p.Token() + if err != nil { + t.Fatalf("second p.Token(): want no error, got %v", err) + } + if tok.AccessToken != "third-access" { + t.Errorf("second p.Token(): want access token 'third-access', got %q", tok.AccessToken) + } + stored, err := memory.Lookup(key) + if err != nil { + t.Fatalf("Lookup(): want no error, got %v", err) + } + if stored.Token.RefreshToken != "third" { + t.Errorf("Lookup(): want refresh token 'third', got %q", stored.Token.RefreshToken) + } +} + func TestToken_ReturnsStillValidTokenWhenProactiveRefreshFails(t *testing.T) { ctx := t.Context() expectedKey := "https://accounts.cloud.databricks.test/oidc/accounts/xyz"