diff --git a/.agents/rules/testing.md b/.agents/rules/testing.md index 1f6af36d68c..7efc095f3e0 100644 --- a/.agents/rules/testing.md +++ b/.agents/rules/testing.md @@ -58,7 +58,7 @@ When writing tests, don't include an explanation in each test case in your respo **RULE: `Cloud = true` adds a cloud run; it never removes the local run.** Every test under `acceptance/` runs locally against the fake server in `libs/testserver`. `Cloud = true` in a `test.toml` or `out.test.toml` means "this test *also* runs against a real workspace when `CLOUD_ENV` is set" — it never means "this test does not run locally". Never tell the user a test doesn't run locally because it is `Cloud = true`. `CloudSlow` does *not* enable a cloud run on its own; it only narrows an existing `Cloud = true` run (see below). -The whole `Cloud*` family lives inside an `if isRunningOnCloud` branch in `getSkipReason` (`acceptance/acceptance_test.go`), so it can only ever subtract from the cloud run: `CloudSlow` (only meaningful when `Cloud = true`) drops it under `-short`, and `CloudEnvs` narrows it to the listed clouds. To find what skips a test *locally*, look at a different set: `GOOS`, `RunsOnDbr`, and `DATABRICKS_TEST_SELECT_CHANGED` (which cloud PR runs set to cover only the tests the branch touches). +The whole `Cloud*` family lives inside an `if isRunningOnCloud` branch in `getSkipReason` (`acceptance/acceptance_test.go`), so it can only ever subtract from the cloud run: `CloudSlow` (only meaningful when `Cloud = true`) drops it under `-short`, and `CloudEnvs` narrows it to the listed clouds. To find what skips a test *locally*, look at a different set: `GOOS`, `RunsOnDbr`, `Litebox` (only when local DMS connection settings are found), and `DATABRICKS_TEST_SELECT_CHANGED` (which cloud PR runs set to cover only the tests the branch touches). `Cloud` is inherited, so a parent `test.toml` can opt a whole subtree in; a leaf `test.toml` with no `Cloud` line is not evidence of anything. Read the generated `out.test.toml` for a test's effective settings. diff --git a/acceptance/README.md b/acceptance/README.md index 69c7391ea02..2bf81e8741b 100644 --- a/acceptance/README.md +++ b/acceptance/README.md @@ -11,7 +11,7 @@ A test can *additionally* opt into running against a real workspace by setting ` - `Cloud` is inherited from parent `test.toml` files, so a whole subtree can be opted in at once. The root `acceptance/test.toml` defaults it to `Cloud = false`, i.e. local only. - Each test's effective settings are visible in its generated `out.test.toml`. -The cloud run is keyed off `CLOUD_ENV`; see `getSkipReason` in `acceptance/acceptance_test.go` for the exact gating. What skips a test *locally* is a different set of settings entirely: `GOOS`, `RunsOnDbr`, and `DATABRICKS_TEST_SELECT_CHANGED` (used by cloud PR runs to cover only the tests the branch touches). +The cloud run is keyed off `CLOUD_ENV`; see `getSkipReason` in `acceptance/acceptance_test.go` for the exact gating. What skips a test *locally* is a different set of settings entirely: `GOOS`, `RunsOnDbr`, `Litebox` (only when local DMS connection settings are found), and `DATABRICKS_TEST_SELECT_CHANGED` (used by cloud PR runs to cover only the tests the branch touches). To run tests against a real workspace: `deco env run -i -n aws-prod-ucws -- ` (requires the `deco` tool and access to a test env). diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index 661112fef63..675ec36d5a7 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -445,8 +445,10 @@ func testAccept(t *testing.T, inprocessMode bool, selectedTests []string, skipTo cloudEnv := os.Getenv("CLOUD_ENV") + liteboxConfigured := false if cloudEnv == "" { internal.StartDefaultServer(t, LogRequests) + liteboxConfigured = internal.DMSLiteboxConfigured(t) if os.Getenv("TEST_DEFAULT_WAREHOUSE_ID") == "" { t.Setenv("TEST_DEFAULT_WAREHOUSE_ID", testserver.TestDefaultWarehouseId) } @@ -620,7 +622,7 @@ func testAccept(t *testing.T, inprocessMode bool, selectedTests []string, skipTo t.Skip("Skipping test execution (only regenerating out.test.toml)") } - skipReason := getSkipReason(&config, configPath, namedTests.Exact(dir)) + skipReason := getSkipReason(&config, configPath, namedTests.Exact(dir), liteboxConfigured) if skipReason != "" { skippedDirs += 1 t.Skip(skipReason) @@ -757,7 +759,7 @@ func isSelectedVariant(changedTests map[string][]string, namedTests selection.Na // Return a reason to skip the test. Empty string means "don't skip". named is set for a test // the last commit message names by its own name (not by a path above it), which runs on cloud // even if it is CloudSlow. -func getSkipReason(config *internal.TestConfig, configPath string, named bool) string { +func getSkipReason(config *internal.TestConfig, configPath string, named, liteboxConfigured bool) string { if Forcerun { return "" } @@ -771,6 +773,10 @@ func getSkipReason(config *internal.TestConfig, configPath string, named bool) s return fmt.Sprintf("Disabled via GOOS.%s setting in %s", runtime.GOOS, configPath) } + if liteboxConfigured && config.Litebox != nil && !*config.Litebox { + return "Disabled via Litebox setting in " + configPath + } + cloudEnv := os.Getenv("CLOUD_ENV") isRunningOnCloud := cloudEnv != "" diff --git a/acceptance/bundle/dms/immutable-folder/out.test.toml b/acceptance/bundle/dms/immutable-folder/out.test.toml index 0cedb09be0f..d6beb94843e 100644 --- a/acceptance/bundle/dms/immutable-folder/out.test.toml +++ b/acceptance/bundle/dms/immutable-folder/out.test.toml @@ -1,3 +1,4 @@ Cloud = false +Litebox = false EnvMatrix.DMS = ["true"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/dms/immutable-folder/test.toml b/acceptance/bundle/dms/immutable-folder/test.toml index 0aabadfa242..8477a3ebeaa 100644 --- a/acceptance/bundle/dms/immutable-folder/test.toml +++ b/acceptance/bundle/dms/immutable-folder/test.toml @@ -1,5 +1,8 @@ # The immutable folder API is not available against a real workspace yet. Cloud = false +# The fake snapshots API stores the snapshot in the testserver workspace, which Litebox's +# workspace fake cannot see, so the recorded path reads as deleted there. +Litebox = false EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/dms/version-never-created/out.test.toml b/acceptance/bundle/dms/version-never-created/out.test.toml index 88e6fbd847c..0948a368b3b 100644 --- a/acceptance/bundle/dms/version-never-created/out.test.toml +++ b/acceptance/bundle/dms/version-never-created/out.test.toml @@ -1,2 +1,3 @@ Cloud = false +Litebox = false EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/version-never-created/test.toml b/acceptance/bundle/dms/version-never-created/test.toml index 6553e7c09bb..c8929f23e66 100644 --- a/acceptance/bundle/dms/version-never-created/test.toml +++ b/acceptance/bundle/dms/version-never-created/test.toml @@ -2,6 +2,8 @@ # that precedes it, which fault.py's trailing-* pattern cannot express, so this case # keeps a [[Server]] stub - and the cloud harness ignores stubs. Cloud = false +# Litebox serves DMS itself, so the stub below never fires there either. +Litebox = false # The first version fails, so the deployment record is never created - only the # workspace node CreateDeployment registered. The next deploy resolves the ID from diff --git a/acceptance/internal/config.go b/acceptance/internal/config.go index 6ea79a9578a..ffbcb65bb45 100644 --- a/acceptance/internal/config.go +++ b/acceptance/internal/config.go @@ -56,6 +56,10 @@ type TestConfig struct { // This also sets -tail when -v is passed. It does not enable the cloud run on its own. CloudSlow *bool + // If false, skip this test when Litebox serves DMS and the workspace API. + // Unlike Cloud, an unset value means the test runs. + Litebox *bool + // If set, current user will be set to a service principal-like UUID instead of email (default is false) IsServicePrincipal *bool diff --git a/acceptance/internal/dms_litebox_settings.go b/acceptance/internal/dms_litebox_settings.go index f36ada02976..46bae7b8984 100644 --- a/acceptance/internal/dms_litebox_settings.go +++ b/acceptance/internal/dms_litebox_settings.go @@ -18,6 +18,12 @@ type dmsLiteboxSettings struct { CA string `json:"ca"` } +// DMSLiteboxConfigured reports whether local DMS connection settings were found. +func DMSLiteboxConfigured(t *testing.T) bool { + t.Helper() + return loadDMSLiteboxSettings(t) != nil +} + func loadDMSLiteboxSettings(t *testing.T) *dmsLiteboxSettings { t.Helper() acceptanceDir, err := os.Getwd() diff --git a/acceptance/internal/materialized_config.go b/acceptance/internal/materialized_config.go index 41fed59eeeb..25ddc992c23 100644 --- a/acceptance/internal/materialized_config.go +++ b/acceptance/internal/materialized_config.go @@ -42,6 +42,7 @@ func GenerateMaterializedConfig(config *TestConfig) string { writeBool(&buf, "Cloud", config.Cloud) writeBool(&buf, "CloudSlow", config.CloudSlow) + writeBool(&buf, "Litebox", config.Litebox) writeBool(&buf, "RunsOnDbr", config.RunsOnDbr) if config.Phase != 0 { fmt.Fprintf(&buf, "Phase = %d\n", config.Phase)