From 95d8fc90da5133e5e678a58008489f9488aa0e97 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 18 Nov 2026 11:34:06 +0000 Subject: [PATCH 1/3] acc: make the suite runnable against DMS in Litebox Two cases cannot run when Litebox serves DMS and the workspace API: one stubs a DMS route that Litebox never routes to the stub, and one creates a snapshot through the testserver-only snapshots API that Litebox's workspace fake cannot see. Add a `Litebox = false` test.toml setting that skips such a case when DMS_CLI_ENDPOINTS_FILE settings are found, and materialize it in out.test.toml. Litebox also allocates two-digit tree ids, so a literal replacement for a deployment id matched inside UUIDs and cut them up. Word-bound all-digit literal replacements in libs/testdiff and in the repls.py mirror. Co-authored-by: Isaac --- acceptance/acceptance_test.go | 12 +++++++++--- acceptance/bin/repls.py | 10 ++++++++-- .../bundle/dms/immutable-folder/out.test.toml | 1 + .../bundle/dms/immutable-folder/test.toml | 3 +++ .../dms/version-never-created/out.test.toml | 1 + .../dms/version-never-created/test.toml | 2 ++ acceptance/internal/config.go | 5 +++++ acceptance/internal/dms_litebox_settings.go | 6 ++++++ acceptance/internal/materialized_config.go | 1 + libs/testdiff/replacement.go | 19 +++++++++++++++++-- libs/testdiff/replacement_test.go | 7 +++++++ 11 files changed, 60 insertions(+), 7 deletions(-) diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index 661112fef63..348c61fe4cf 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -445,8 +445,10 @@ func testAccept(t *testing.T, inprocessMode bool, selectedTests []string, skipTo cloudEnv := os.Getenv("CLOUD_ENV") + liteboxConfigured := false if cloudEnv == "" { internal.StartDefaultServer(t, LogRequests) + liteboxConfigured = internal.DMSLiteboxConfigured(t) if os.Getenv("TEST_DEFAULT_WAREHOUSE_ID") == "" { t.Setenv("TEST_DEFAULT_WAREHOUSE_ID", testserver.TestDefaultWarehouseId) } @@ -620,7 +622,7 @@ func testAccept(t *testing.T, inprocessMode bool, selectedTests []string, skipTo t.Skip("Skipping test execution (only regenerating out.test.toml)") } - skipReason := getSkipReason(&config, configPath, namedTests.Exact(dir)) + skipReason := getSkipReason(&config, configPath, namedTests.Exact(dir), liteboxConfigured) if skipReason != "" { skippedDirs += 1 t.Skip(skipReason) @@ -756,8 +758,8 @@ func isSelectedVariant(changedTests map[string][]string, namedTests selection.Na // Return a reason to skip the test. Empty string means "don't skip". named is set for a test // the last commit message names by its own name (not by a path above it), which runs on cloud -// even if it is CloudSlow. -func getSkipReason(config *internal.TestConfig, configPath string, named bool) string { +// even if it is CloudSlow. litebox is set when DMS connection settings were found. +func getSkipReason(config *internal.TestConfig, configPath string, named, litebox bool) string { if Forcerun { return "" } @@ -771,6 +773,10 @@ func getSkipReason(config *internal.TestConfig, configPath string, named bool) s return fmt.Sprintf("Disabled via GOOS.%s setting in %s", runtime.GOOS, configPath) } + if litebox && config.Litebox != nil && !*config.Litebox { + return "Disabled via Litebox setting in " + configPath + } + cloudEnv := os.Getenv("CLOUD_ENV") isRunningOnCloud := cloudEnv != "" diff --git a/acceptance/bin/repls.py b/acceptance/bin/repls.py index f24da6ce638..08bb3b1612e 100644 --- a/acceptance/bin/repls.py +++ b/acceptance/bin/repls.py @@ -28,6 +28,12 @@ def read_entries(): return result +def literal_pattern(value): + """Mirror appendLiteral in libs/testdiff: a short numeric id must not match inside UUIDs.""" + pattern = re.escape(value) + return rf"\b{pattern}\b" if value.isdigit() else pattern + + def read_repls(): """Return (pattern, replacement, distinct) tuples in application order.""" entries = [] @@ -45,8 +51,8 @@ def read_repls(): # values with quotes or backslashes are replaced inside JSON output as well. encoded = json.dumps(literal, ensure_ascii=False)[1:-1] if encoded != literal: - entries.append((order, re.escape(encoded), new, False)) - entries.append((order, re.escape(literal), new, False)) + entries.append((order, literal_pattern(encoded), new, False)) + entries.append((order, literal_pattern(literal), new, False)) # Stable sort: replacements with the same order are applied in the order they were added. entries.sort(key=lambda entry: entry[0]) diff --git a/acceptance/bundle/dms/immutable-folder/out.test.toml b/acceptance/bundle/dms/immutable-folder/out.test.toml index 0cedb09be0f..d6beb94843e 100644 --- a/acceptance/bundle/dms/immutable-folder/out.test.toml +++ b/acceptance/bundle/dms/immutable-folder/out.test.toml @@ -1,3 +1,4 @@ Cloud = false +Litebox = false EnvMatrix.DMS = ["true"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/dms/immutable-folder/test.toml b/acceptance/bundle/dms/immutable-folder/test.toml index 0aabadfa242..8477a3ebeaa 100644 --- a/acceptance/bundle/dms/immutable-folder/test.toml +++ b/acceptance/bundle/dms/immutable-folder/test.toml @@ -1,5 +1,8 @@ # The immutable folder API is not available against a real workspace yet. Cloud = false +# The fake snapshots API stores the snapshot in the testserver workspace, which Litebox's +# workspace fake cannot see, so the recorded path reads as deleted there. +Litebox = false EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/dms/version-never-created/out.test.toml b/acceptance/bundle/dms/version-never-created/out.test.toml index 88e6fbd847c..0948a368b3b 100644 --- a/acceptance/bundle/dms/version-never-created/out.test.toml +++ b/acceptance/bundle/dms/version-never-created/out.test.toml @@ -1,2 +1,3 @@ Cloud = false +Litebox = false EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/version-never-created/test.toml b/acceptance/bundle/dms/version-never-created/test.toml index 6553e7c09bb..c8929f23e66 100644 --- a/acceptance/bundle/dms/version-never-created/test.toml +++ b/acceptance/bundle/dms/version-never-created/test.toml @@ -2,6 +2,8 @@ # that precedes it, which fault.py's trailing-* pattern cannot express, so this case # keeps a [[Server]] stub - and the cloud harness ignores stubs. Cloud = false +# Litebox serves DMS itself, so the stub below never fires there either. +Litebox = false # The first version fails, so the deployment record is never created - only the # workspace node CreateDeployment registered. The next deploy resolves the ID from diff --git a/acceptance/internal/config.go b/acceptance/internal/config.go index 6ea79a9578a..4cb43c67ddd 100644 --- a/acceptance/internal/config.go +++ b/acceptance/internal/config.go @@ -56,6 +56,11 @@ type TestConfig struct { // This also sets -tail when -v is passed. It does not enable the cloud run on its own. CloudSlow *bool + // If false, skip this test when DMS and the workspace API are served by Litebox + // (DMS_CLI_ENDPOINTS_FILE): a [[Server]] stub on one of those routes never fires, and + // objects created through a testserver-only API are invisible to Litebox's workspace. + Litebox *bool + // If set, current user will be set to a service principal-like UUID instead of email (default is false) IsServicePrincipal *bool diff --git a/acceptance/internal/dms_litebox_settings.go b/acceptance/internal/dms_litebox_settings.go index f36ada02976..46bae7b8984 100644 --- a/acceptance/internal/dms_litebox_settings.go +++ b/acceptance/internal/dms_litebox_settings.go @@ -18,6 +18,12 @@ type dmsLiteboxSettings struct { CA string `json:"ca"` } +// DMSLiteboxConfigured reports whether local DMS connection settings were found. +func DMSLiteboxConfigured(t *testing.T) bool { + t.Helper() + return loadDMSLiteboxSettings(t) != nil +} + func loadDMSLiteboxSettings(t *testing.T) *dmsLiteboxSettings { t.Helper() acceptanceDir, err := os.Getwd() diff --git a/acceptance/internal/materialized_config.go b/acceptance/internal/materialized_config.go index 41fed59eeeb..25ddc992c23 100644 --- a/acceptance/internal/materialized_config.go +++ b/acceptance/internal/materialized_config.go @@ -42,6 +42,7 @@ func GenerateMaterializedConfig(config *TestConfig) string { writeBool(&buf, "Cloud", config.Cloud) writeBool(&buf, "CloudSlow", config.CloudSlow) + writeBool(&buf, "Litebox", config.Litebox) writeBool(&buf, "RunsOnDbr", config.RunsOnDbr) if config.Phase != 0 { fmt.Fprintf(&buf, "Phase = %d\n", config.Phase) diff --git a/libs/testdiff/replacement.go b/libs/testdiff/replacement.go index 6df2ade8434..b4ae901d36e 100644 --- a/libs/testdiff/replacement.go +++ b/libs/testdiff/replacement.go @@ -91,9 +91,15 @@ func (r *ReplacementsContext) append(pattern *regexp.Regexp, replacement string, } func (r *ReplacementsContext) appendLiteral(old, new string, order int) { + // Transform the input strings such that they can be used as literal strings in regular expressions. + pattern := regexp.QuoteMeta(old) + // A short numeric id (Litebox allocates two-digit ones) would otherwise also match inside + // UUIDs, timestamps and ports and cut them up. + if isAllDigits(old) { + pattern = `\b` + pattern + `\b` + } r.append( - // Transform the input strings such that they can be used as literal strings in regular expressions. - regexp.MustCompile(regexp.QuoteMeta(old)), + regexp.MustCompile(pattern), // Transform the replacement string such that `$` is interpreted as a literal dollar sign. // For more information about how the replacement string is used, see [regexp.Regexp.Expand]. strings.ReplaceAll(new, `$`, `$$`), @@ -101,6 +107,15 @@ func (r *ReplacementsContext) appendLiteral(old, new string, order int) { ) } +func isAllDigits(s string) bool { + for _, c := range s { + if c < '0' || c > '9' { + return false + } + } + return s != "" +} + func (r *ReplacementsContext) Set(old, new string) { r.SetWithOrder(old, new, 0) } diff --git a/libs/testdiff/replacement_test.go b/libs/testdiff/replacement_test.go index 53e025b4ff5..95836cf843e 100644 --- a/libs/testdiff/replacement_test.go +++ b/libs/testdiff/replacement_test.go @@ -14,6 +14,13 @@ func TestReplacement_Literal(t *testing.T) { assert.Equal(t, `[replacement]`, repls.Replace(`foobar`)) } +func TestReplacement_NumericLiteralIsWordBounded(t *testing.T) { + var repls ReplacementsContext + + repls.Set(`21`, `[DEPLOYMENT_ID]`) + assert.Equal(t, `deployments/[DEPLOYMENT_ID] "[DEPLOYMENT_ID]" 1d2f21aa 3210`, repls.Replace(`deployments/21 "21" 1d2f21aa 3210`)) +} + func TestReplacement_Encoded(t *testing.T) { var repls ReplacementsContext From 69c11ac0230be23e893a5731d3933d33a931a613 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 18 Nov 2026 12:18:47 +0000 Subject: [PATCH 2/3] Move the numeric replacement change to its own PR Co-authored-by: Isaac --- acceptance/bin/repls.py | 10 ++-------- libs/testdiff/replacement.go | 19 ++----------------- libs/testdiff/replacement_test.go | 7 ------- 3 files changed, 4 insertions(+), 32 deletions(-) diff --git a/acceptance/bin/repls.py b/acceptance/bin/repls.py index 08bb3b1612e..f24da6ce638 100644 --- a/acceptance/bin/repls.py +++ b/acceptance/bin/repls.py @@ -28,12 +28,6 @@ def read_entries(): return result -def literal_pattern(value): - """Mirror appendLiteral in libs/testdiff: a short numeric id must not match inside UUIDs.""" - pattern = re.escape(value) - return rf"\b{pattern}\b" if value.isdigit() else pattern - - def read_repls(): """Return (pattern, replacement, distinct) tuples in application order.""" entries = [] @@ -51,8 +45,8 @@ def read_repls(): # values with quotes or backslashes are replaced inside JSON output as well. encoded = json.dumps(literal, ensure_ascii=False)[1:-1] if encoded != literal: - entries.append((order, literal_pattern(encoded), new, False)) - entries.append((order, literal_pattern(literal), new, False)) + entries.append((order, re.escape(encoded), new, False)) + entries.append((order, re.escape(literal), new, False)) # Stable sort: replacements with the same order are applied in the order they were added. entries.sort(key=lambda entry: entry[0]) diff --git a/libs/testdiff/replacement.go b/libs/testdiff/replacement.go index b4ae901d36e..6df2ade8434 100644 --- a/libs/testdiff/replacement.go +++ b/libs/testdiff/replacement.go @@ -91,15 +91,9 @@ func (r *ReplacementsContext) append(pattern *regexp.Regexp, replacement string, } func (r *ReplacementsContext) appendLiteral(old, new string, order int) { - // Transform the input strings such that they can be used as literal strings in regular expressions. - pattern := regexp.QuoteMeta(old) - // A short numeric id (Litebox allocates two-digit ones) would otherwise also match inside - // UUIDs, timestamps and ports and cut them up. - if isAllDigits(old) { - pattern = `\b` + pattern + `\b` - } r.append( - regexp.MustCompile(pattern), + // Transform the input strings such that they can be used as literal strings in regular expressions. + regexp.MustCompile(regexp.QuoteMeta(old)), // Transform the replacement string such that `$` is interpreted as a literal dollar sign. // For more information about how the replacement string is used, see [regexp.Regexp.Expand]. strings.ReplaceAll(new, `$`, `$$`), @@ -107,15 +101,6 @@ func (r *ReplacementsContext) appendLiteral(old, new string, order int) { ) } -func isAllDigits(s string) bool { - for _, c := range s { - if c < '0' || c > '9' { - return false - } - } - return s != "" -} - func (r *ReplacementsContext) Set(old, new string) { r.SetWithOrder(old, new, 0) } diff --git a/libs/testdiff/replacement_test.go b/libs/testdiff/replacement_test.go index 95836cf843e..53e025b4ff5 100644 --- a/libs/testdiff/replacement_test.go +++ b/libs/testdiff/replacement_test.go @@ -14,13 +14,6 @@ func TestReplacement_Literal(t *testing.T) { assert.Equal(t, `[replacement]`, repls.Replace(`foobar`)) } -func TestReplacement_NumericLiteralIsWordBounded(t *testing.T) { - var repls ReplacementsContext - - repls.Set(`21`, `[DEPLOYMENT_ID]`) - assert.Equal(t, `deployments/[DEPLOYMENT_ID] "[DEPLOYMENT_ID]" 1d2f21aa 3210`, repls.Replace(`deployments/21 "21" 1d2f21aa 3210`)) -} - func TestReplacement_Encoded(t *testing.T) { var repls ReplacementsContext From 435809819cba3e5c01daad30807308005296e377 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 18 Nov 2026 13:07:59 +0000 Subject: [PATCH 3/3] Address review: shorter field comment, list Litebox among local skip reasons Co-authored-by: Isaac --- .agents/rules/testing.md | 2 +- acceptance/README.md | 2 +- acceptance/acceptance_test.go | 6 +++--- acceptance/internal/config.go | 5 ++--- 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/.agents/rules/testing.md b/.agents/rules/testing.md index 1f6af36d68c..7efc095f3e0 100644 --- a/.agents/rules/testing.md +++ b/.agents/rules/testing.md @@ -58,7 +58,7 @@ When writing tests, don't include an explanation in each test case in your respo **RULE: `Cloud = true` adds a cloud run; it never removes the local run.** Every test under `acceptance/` runs locally against the fake server in `libs/testserver`. `Cloud = true` in a `test.toml` or `out.test.toml` means "this test *also* runs against a real workspace when `CLOUD_ENV` is set" — it never means "this test does not run locally". Never tell the user a test doesn't run locally because it is `Cloud = true`. `CloudSlow` does *not* enable a cloud run on its own; it only narrows an existing `Cloud = true` run (see below). -The whole `Cloud*` family lives inside an `if isRunningOnCloud` branch in `getSkipReason` (`acceptance/acceptance_test.go`), so it can only ever subtract from the cloud run: `CloudSlow` (only meaningful when `Cloud = true`) drops it under `-short`, and `CloudEnvs` narrows it to the listed clouds. To find what skips a test *locally*, look at a different set: `GOOS`, `RunsOnDbr`, and `DATABRICKS_TEST_SELECT_CHANGED` (which cloud PR runs set to cover only the tests the branch touches). +The whole `Cloud*` family lives inside an `if isRunningOnCloud` branch in `getSkipReason` (`acceptance/acceptance_test.go`), so it can only ever subtract from the cloud run: `CloudSlow` (only meaningful when `Cloud = true`) drops it under `-short`, and `CloudEnvs` narrows it to the listed clouds. To find what skips a test *locally*, look at a different set: `GOOS`, `RunsOnDbr`, `Litebox` (only when local DMS connection settings are found), and `DATABRICKS_TEST_SELECT_CHANGED` (which cloud PR runs set to cover only the tests the branch touches). `Cloud` is inherited, so a parent `test.toml` can opt a whole subtree in; a leaf `test.toml` with no `Cloud` line is not evidence of anything. Read the generated `out.test.toml` for a test's effective settings. diff --git a/acceptance/README.md b/acceptance/README.md index 69c7391ea02..2bf81e8741b 100644 --- a/acceptance/README.md +++ b/acceptance/README.md @@ -11,7 +11,7 @@ A test can *additionally* opt into running against a real workspace by setting ` - `Cloud` is inherited from parent `test.toml` files, so a whole subtree can be opted in at once. The root `acceptance/test.toml` defaults it to `Cloud = false`, i.e. local only. - Each test's effective settings are visible in its generated `out.test.toml`. -The cloud run is keyed off `CLOUD_ENV`; see `getSkipReason` in `acceptance/acceptance_test.go` for the exact gating. What skips a test *locally* is a different set of settings entirely: `GOOS`, `RunsOnDbr`, and `DATABRICKS_TEST_SELECT_CHANGED` (used by cloud PR runs to cover only the tests the branch touches). +The cloud run is keyed off `CLOUD_ENV`; see `getSkipReason` in `acceptance/acceptance_test.go` for the exact gating. What skips a test *locally* is a different set of settings entirely: `GOOS`, `RunsOnDbr`, `Litebox` (only when local DMS connection settings are found), and `DATABRICKS_TEST_SELECT_CHANGED` (used by cloud PR runs to cover only the tests the branch touches). To run tests against a real workspace: `deco env run -i -n aws-prod-ucws -- ` (requires the `deco` tool and access to a test env). diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index 348c61fe4cf..675ec36d5a7 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -758,8 +758,8 @@ func isSelectedVariant(changedTests map[string][]string, namedTests selection.Na // Return a reason to skip the test. Empty string means "don't skip". named is set for a test // the last commit message names by its own name (not by a path above it), which runs on cloud -// even if it is CloudSlow. litebox is set when DMS connection settings were found. -func getSkipReason(config *internal.TestConfig, configPath string, named, litebox bool) string { +// even if it is CloudSlow. +func getSkipReason(config *internal.TestConfig, configPath string, named, liteboxConfigured bool) string { if Forcerun { return "" } @@ -773,7 +773,7 @@ func getSkipReason(config *internal.TestConfig, configPath string, named, litebo return fmt.Sprintf("Disabled via GOOS.%s setting in %s", runtime.GOOS, configPath) } - if litebox && config.Litebox != nil && !*config.Litebox { + if liteboxConfigured && config.Litebox != nil && !*config.Litebox { return "Disabled via Litebox setting in " + configPath } diff --git a/acceptance/internal/config.go b/acceptance/internal/config.go index 4cb43c67ddd..ffbcb65bb45 100644 --- a/acceptance/internal/config.go +++ b/acceptance/internal/config.go @@ -56,9 +56,8 @@ type TestConfig struct { // This also sets -tail when -v is passed. It does not enable the cloud run on its own. CloudSlow *bool - // If false, skip this test when DMS and the workspace API are served by Litebox - // (DMS_CLI_ENDPOINTS_FILE): a [[Server]] stub on one of those routes never fires, and - // objects created through a testserver-only API are invisible to Litebox's workspace. + // If false, skip this test when Litebox serves DMS and the workspace API. + // Unlike Cloud, an unset value means the test runs. Litebox *bool // If set, current user will be set to a service principal-like UUID instead of email (default is false)