bundle: Add genie_space resource (direct engine only)

NEXT_CHANGELOG.md

@@ -7,6 +7,7 @@
 ### CLI
 
 ### Bundles
+* Add the `genie_spaces` bundle resource for managing Databricks Genie spaces as code, plus `bundle generate genie-space` to import an existing space. Direct deployment engine only ([#5282](https://github.com/databricks/cli/pull/5282)).
 
 ### Dependency updates
 

acceptance/.gitattributes

@@ -4,6 +4,7 @@
 # uploading the file's content to a workspace.
 *.txt text eol=lf
 *.lvdash.json text eol=lf
+*.geniespace.json text eol=lf
 
 # The out.test.toml file is autogenerated based on the merged test.toml view.
 out.test.toml linguist-generated=true

acceptance/bundle/deployment/bind/genie_space/databricks.yml.tmpl

@@ -0,0 +1,13 @@
+bundle:
+  name: test-bundle-$UNIQUE_NAME
+
+resources:
+  genie_spaces:
+    genie_space1:
+      title: $GENIE_SPACE_TITLE
+      warehouse_id: "test-warehouse-id"
+      parent_path: /Users/$CURRENT_USER_NAME
+      # Inline body matches the out-of-band-created space byte-for-byte, so the
+      # only possible post-bind drift signal is the etag.
+      serialized_space:
+        version: 1

acceptance/bundle/deployment/bind/genie_space/output.txt

@@ -0,0 +1,37 @@
+
+>>> [CLI] bundle deployment bind genie_space1 [GENIE_SPACE_ID] --auto-approve
+Updating deployment state...
+Successfully bound genie_space with an id '[GENIE_SPACE_ID]'
+Run 'bundle deploy' to deploy changes to your workspace
+
+>>> [CLI] bundle plan
+Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> [CLI] genie get-space [GENIE_SPACE_ID]
+{
+  "parent_path": "/Users/[USERNAME]",
+  "title": "test genie space [UNIQUE_NAME]",
+  "warehouse_id": "test-warehouse-id"
+}
+
+>>> [CLI] bundle deployment unbind genie_space1
+Updating deployment state...
+
+>>> [CLI] bundle destroy --auto-approve
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default
+
+Deleting files...
+Destroy complete!
+
+>>> [CLI] genie get-space [GENIE_SPACE_ID]
+{
+  "parent_path": "/Users/[USERNAME]",
+  "title": "test genie space [UNIQUE_NAME]",
+  "warehouse_id": "test-warehouse-id"
+}

acceptance/bundle/deployment/bind/genie_space/script

@@ -0,0 +1,30 @@
+GENIE_SPACE_TITLE="test genie space $UNIQUE_NAME"
+
+export GENIE_SPACE_TITLE
+envsubst < databricks.yml.tmpl > databricks.yml
+
+# Create a Genie space out of band, then bind the bundle resource to it.
+GENIE_SPACE_ID=$($CLI genie create-space "test-warehouse-id" '{"version":1}' --title "${GENIE_SPACE_TITLE}" | jq -r '.space_id')
+
+cleanupRemoveGenieSpace() {
+    $CLI genie trash-space "${GENIE_SPACE_ID}"
+}
+trap cleanupRemoveGenieSpace EXIT
+
+trace $CLI bundle deployment bind genie_space1 "${GENIE_SPACE_ID}" --auto-approve
+
+# Bind must copy the remote etag into state, so the first plan after bind is
+# clean. Without that, the etag drift signal (empty stored vs remote) would
+# produce a bogus update here.
+trace $CLI bundle plan
+
+trace $CLI bundle deploy
+
+trace $CLI genie get-space "${GENIE_SPACE_ID}" | jq --sort-keys '{title, parent_path, warehouse_id}'
+
+trace $CLI bundle deployment unbind genie_space1
+
+trace $CLI bundle destroy --auto-approve
+
+# Read the Genie space again (expecting it still exists and is not deleted):
+trace $CLI genie get-space "${GENIE_SPACE_ID}" | jq --sort-keys '{title, parent_path, warehouse_id}'

acceptance/bundle/deployment/bind/genie_space/test.toml

@@ -0,0 +1,13 @@
+Local = true
+
+# Genie spaces are only deployed via the direct deployment engine.
+EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]
+
+# Uses a literal warehouse id and a version-sensitive serialized_space, so this
+# lifecycle test runs against the local mock server only (overrides the Cloud=true
+# inherited from acceptance/bundle/deployment/test.toml).
+Cloud = false
+
+[[Repls]]
+Old = "[0-9a-f]{32}"
+New = "[GENIE_SPACE_ID]"

acceptance/bundle/generate/genie_space/databricks.yml

@@ -0,0 +1,2 @@
+bundle:
+  name: genie-space-generate

acceptance/bundle/generate/genie_space/genie_space.json.tmpl

@@ -0,0 +1,7 @@
+{
+    "title": "test genie space",
+    "description": "test description",
+    "parent_path": "/Workspace/test-$UNIQUE_NAME",
+    "warehouse_id": "test-warehouse-id",
+    "serialized_space": "{\"tables\":[],\"questions\":[]}"
+}

acceptance/bundle/generate/genie_space/out/genie_space/test_genie_space.geniespace.json

@@ -0,0 +1,4 @@
+{
+  "questions": [],
+  "tables": []
+}

acceptance/bundle/generate/genie_space/out/resource/test_genie_space.genie_space.yml

@@ -0,0 +1,8 @@
+resources:
+  genie_spaces:
+    test_genie_space:
+      title: "test genie space"
+      warehouse_id: test-warehouse-id
+      file_path: ../genie_space/test_genie_space.geniespace.json
+      description: test description
+      parent_path: /Workspace/test-[UNIQUE_NAME]

acceptance/bundle/generate/genie_space/output.txt

@@ -0,0 +1,6 @@
+
+>>> [CLI] workspace mkdirs /Workspace/test-[UNIQUE_NAME]
+
+>>> [CLI] bundle generate genie-space --existing-id [GENIE_SPACE_ID] --genie-space-dir out/genie_space --resource-dir out/resource
+Writing genie space to out/genie_space/test_genie_space.geniespace.json
+Writing configuration to out/resource/test_genie_space.genie_space.yml

acceptance/bundle/generate/genie_space/script

@@ -0,0 +1,8 @@
+trace $CLI workspace mkdirs /Workspace/test-$UNIQUE_NAME
+
+# create a genie space to import
+envsubst < genie_space.json.tmpl > genie_space.json
+genie_space_id=$($CLI genie create-space --json @genie_space.json | jq -r '.space_id')
+rm genie_space.json
+
+trace $CLI bundle generate genie-space --existing-id $genie_space_id --genie-space-dir out/genie_space --resource-dir out/resource

acceptance/bundle/generate/genie_space/test.toml

@@ -0,0 +1,10 @@
+[[Repls]]
+Old = '\\\\'
+New = '/'
+
+[[Repls]]
+Old = "[0-9a-f]{32}"
+New = "[GENIE_SPACE_ID]"
+
+[Env]
+MSYS_NO_PATHCONV = "1"

acceptance/bundle/generate/genie_space_existing_id_not_found/databricks.yml

@@ -0,0 +1,2 @@
+bundle:
+  name: test-bundle

acceptance/bundle/generate/genie_space_existing_id_not_found/output.txt

@@ -0,0 +1,4 @@
+Error: genie space with ID f00dcafe not found
+
+
+Exit code: 1

acceptance/bundle/generate/genie_space_existing_id_not_found/script

@@ -0,0 +1,2 @@
+# Test that bundle generate genie-space fails when the existing ID is not found
+exec $CLI bundle generate genie-space --existing-id f00dcafe

acceptance/bundle/generate/genie_space_inplace/databricks.yml

@@ -0,0 +1,9 @@
+bundle:
+  name: genie space update inplace
+
+resources:
+  genie_spaces:
+    test_genie_space:
+      title: "test genie space"
+      warehouse_id: "my-warehouse-1234"
+      file_path: ./space.geniespace.json

acceptance/bundle/generate/genie_space_inplace/output.txt

@@ -0,0 +1,30 @@
+
+>>> cat space.geniespace.json
+{}
+
+=== deploy initial genie space
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/genie space update inplace/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+=== update the genie space
+>>> [CLI] genie update-space [GENIE_SPACE_ID] --serialized-space {"a":"b"}
+{
+  "etag": "2",
+  "parent_path": "/Users/[USERNAME]/.bundle/genie space update inplace/default/resources",
+  "serialized_space": "{\"a\":\"b\"}",
+  "space_id": "[GENIE_SPACE_ID]",
+  "title": "test genie space",
+  "warehouse_id": "my-warehouse-1234"
+}
+
+=== update the genie space file using bundle generate
+>>> [CLI] bundle generate genie-space --resource test_genie_space --force
+Writing genie space to space.geniespace.json
+
+>>> cat space.geniespace.json
+{
+  "a": "b"
+}

acceptance/bundle/generate/genie_space_inplace/script

@@ -0,0 +1,13 @@
+trace cat space.geniespace.json
+
+title "deploy initial genie space"
+trace $CLI bundle deploy
+genie_space_id=$($CLI bundle summary --output json | jq -r '.resources.genie_spaces.test_genie_space.id')
+
+title "update the genie space"
+trace $CLI genie update-space $genie_space_id --serialized-space '{"a":"b"}'
+
+title "update the genie space file using bundle generate"
+trace $CLI bundle generate genie-space --resource test_genie_space --force
+
+trace cat space.geniespace.json

acceptance/bundle/generate/genie_space_inplace/space.geniespace.json

@@ -0,0 +1 @@
+{}

acceptance/bundle/generate/genie_space_inplace/test.toml

@@ -0,0 +1,6 @@
+# Genie spaces are only deployed via the direct deployment engine.
+EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]
+
+[[Repls]]
+Old = "[0-9a-f]{32}"
+New = "[GENIE_SPACE_ID]"

acceptance/bundle/help/bundle-generate/output.txt

@@ -30,6 +30,7 @@ Available Commands:
   alert       Generate configuration for an alert
   app         Generate bundle configuration for a Databricks app
   dashboard   Generate configuration for a dashboard
+  genie-space Generate configuration for a Genie space
   job         Generate bundle configuration for a job
   pipeline    Generate bundle configuration for a pipeline
 

acceptance/bundle/invariant/configs/genie_space.yml.tmpl

@@ -0,0 +1,14 @@
+bundle:
+  name: test-bundle-$UNIQUE_NAME
+
+resources:
+  genie_spaces:
+    foo:
+      warehouse_id: $TEST_DEFAULT_WAREHOUSE_ID
+      title: test-genie-space-$UNIQUE_NAME
+      # Inline (structured) serialized_space is marshalled to a JSON string by
+      # ConfigureGenieSpaceSerializedSpace; this config doubles as a regression
+      # guard that the normalization produces a drift-free deploy.
+      serialized_space:
+        version: 1
+        display_name: test-genie-space-$UNIQUE_NAME

acceptance/bundle/invariant/continue_293/test.toml

@@ -9,6 +9,9 @@ EnvMatrixExclude.no_model_with_permissions = ["INPUT_CONFIG=model_with_permissio
 # vector_search_endpoints resource is not supported on v0.293.0
 EnvMatrixExclude.no_vector_search_endpoint = ["INPUT_CONFIG=vector_search_endpoint.yml.tmpl"]
 
+# genie_spaces resource is not supported on v0.293.0
+EnvMatrixExclude.no_genie_space = ["INPUT_CONFIG=genie_space.yml.tmpl"]
+
 # Dotted pipeline configuration keys are not supported on v0.293.0
 EnvMatrixExclude.no_pipeline_config_dots = ["INPUT_CONFIG=pipeline_config_dots.yml.tmpl"]
 

acceptance/bundle/invariant/migrate/test.toml

@@ -5,6 +5,8 @@ EnvMatrixExclude.no_vector_search_index = ["INPUT_CONFIG=vector_search_index.yml
 # Error: Catalog resources are only supported with direct deployment mode
 EnvMatrixExclude.no_catalog = ["INPUT_CONFIG=catalog.yml.tmpl"]
 EnvMatrixExclude.no_external_location = ["INPUT_CONFIG=external_location.yml.tmpl"]
+# Genie spaces are direct-only too; the terraform deploy that seeds the migration fails for them.
+EnvMatrixExclude.no_genie_space = ["INPUT_CONFIG=genie_space.yml.tmpl"]
 
 # Cross-resource permission references (e.g. ${resources.jobs.job_b.permissions[0].level})
 # don't work in terraform mode: the terraform interpolator converts the path to

acceptance/bundle/invariant/test.toml

@@ -30,6 +30,7 @@ EnvMatrix.INPUT_CONFIG = [
     "database_instance.yml.tmpl",
     "experiment.yml.tmpl",
     "external_location.yml.tmpl",
+    "genie_space.yml.tmpl",
     "job.yml.tmpl",
     "job_pydabs_10_tasks.yml.tmpl",
     "job_pydabs_1000_tasks.yml.tmpl",
@@ -80,6 +81,10 @@ no_external_location_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=external_loc
 # External volumes reference external locations; excluded from cloud for the same reason
 no_external_volume_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=volume_external.yml.tmpl"]
 
+# Genie space serialized_space schema is workspace-version-sensitive; we only
+# exercise it against the local mock server, not a real cloud workspace.
+no_genie_space_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=genie_space.yml.tmpl"]
+
 # Fake SQL endpoint for local tests
 [[Server]]
 Pattern = "POST /api/2.0/sql/statements/"

acceptance/bundle/refschema/out.fields.txt

@@ -734,6 +734,25 @@ resources.external_locations.*.grants[*]	catalog.PrivilegeAssignment	ALL
 resources.external_locations.*.grants[*].principal	string	ALL
 resources.external_locations.*.grants[*].privileges	[]catalog.Privilege	ALL
 resources.external_locations.*.grants[*].privileges[*]	catalog.Privilege	ALL
+resources.genie_spaces.*.description	string	ALL
+resources.genie_spaces.*.etag	string	ALL
+resources.genie_spaces.*.file_path	string	INPUT
+resources.genie_spaces.*.id	string	INPUT
+resources.genie_spaces.*.lifecycle	resources.Lifecycle	INPUT
+resources.genie_spaces.*.lifecycle.prevent_destroy	bool	INPUT
+resources.genie_spaces.*.modified_status	string	INPUT
+resources.genie_spaces.*.parent_path	string	ALL
+resources.genie_spaces.*.serialized_space	any	ALL
+resources.genie_spaces.*.space_id	string	ALL
+resources.genie_spaces.*.title	string	ALL
+resources.genie_spaces.*.url	string	INPUT
+resources.genie_spaces.*.warehouse_id	string	ALL
+resources.genie_spaces.*.permissions.object_id	string	ALL
+resources.genie_spaces.*.permissions[*]	dresources.StatePermission	ALL
+resources.genie_spaces.*.permissions[*].group_name	string	ALL
+resources.genie_spaces.*.permissions[*].level	iam.PermissionLevel	ALL
+resources.genie_spaces.*.permissions[*].service_principal_name	string	ALL
+resources.genie_spaces.*.permissions[*].user_name	string	ALL
 resources.jobs.*.budget_policy_id	string	ALL
 resources.jobs.*.continuous	*jobs.Continuous	ALL
 resources.jobs.*.continuous.pause_status	jobs.PauseStatus	ALL

acceptance/bundle/resources/genie_spaces/inline/databricks.yml.tmpl

@@ -0,0 +1,22 @@
+bundle:
+  name: deploy-genie-space-inline-$UNIQUE_NAME
+
+resources:
+  genie_spaces:
+    sales_analytics:
+      title: "Sales Analytics Inline Genie"
+      description: "Inline serialized_space test"
+      warehouse_id: "test-warehouse-id"
+      parent_path: /Users/$CURRENT_USER_NAME
+      serialized_space:
+        version: 1
+        config:
+          sample_questions:
+            - id: "sq-001"
+              question: ["What is the total revenue?"]
+        data_sources:
+          tables:
+            - identifier: "main.sales.orders"
+              column_configs:
+                - column_name: "amount"
+                  get_example_values: true