From 7a296e74d5447b52997de00bb8743c4aa5225fc3 Mon Sep 17 00:00:00 2001 From: Daniel Bai Date: Thu, 19 Nov 2026 00:33:42 +0000 Subject: [PATCH 1/2] PyDABs: keep empty nested objects, as YAML does Python resources now emit a nested object that is set but empty, such as StreamingMode() or ExternalTrigger(), as {} instead of dropping it like an unset field. Unset fields and empty lists and maps are still omitted. Co-authored-by: Isaac --- .../data_quality_monitors-support/output.txt | 3 +- .../databricks.yml | 28 +++++++ .../materialization_sets-support/output.txt | 80 +++++++++++++++++++ .../materialization_sets-support/resources.py | 61 ++++++++++++++ .../bundles/core/_transform_to_json.py | 10 ++- .../databricks_tests/core/test_transform.py | 15 ++++ .../core/test_transform_to_json.py | 29 +++++++ 7 files changed, 221 insertions(+), 5 deletions(-) diff --git a/acceptance/bundle/python/data_quality_monitors-support/output.txt b/acceptance/bundle/python/data_quality_monitors-support/output.txt index 99b29fe4cb5..03c2ea62f78 100644 --- a/acceptance/bundle/python/data_quality_monitors-support/output.txt +++ b/acceptance/bundle/python/data_quality_monitors-support/output.txt @@ -25,7 +25,8 @@ "my_data_quality_monitor_2": { "data_profiling_config": { "assets_dir": "/Workspace/Users/user@example.com/assets", - "output_schema_id": "[UUID][0]" + "output_schema_id": "[UUID][0]", + "snapshot": {} }, "object_id": "[UUID][1] (updated)", "object_type": "table" diff --git a/acceptance/bundle/python/materialization_sets-support/databricks.yml b/acceptance/bundle/python/materialization_sets-support/databricks.yml index 443e1853d07..84c30ca51b6 100644 --- a/acceptance/bundle/python/materialization_sets-support/databricks.yml +++ b/acceptance/bundle/python/materialization_sets-support/databricks.yml @@ -20,3 +20,31 @@ resources: table_name_prefix: my_feature_1_off cron_schedule_trigger: mode: DERIVED + + yaml_streaming_mode: + materialized_features: + - feature_name: main.default.yaml_streaming_feature + online_store_config: + online_store_name: my_online_store + catalog_name: main + schema_name: default + table_name_prefix: yaml_streaming_feature_on + streaming_mode: {} + + yaml_table_trigger: + materialized_features: + - feature_name: main.default.yaml_table_feature + offline_store_config: + catalog_name: main + schema_name: default + table_name_prefix: yaml_table_feature_off + table_trigger: {} + + yaml_external_trigger: + materialized_features: + - feature_name: main.default.yaml_external_feature + offline_store_config: + catalog_name: main + schema_name: default + table_name_prefix: yaml_external_feature_off + external_trigger: {} diff --git a/acceptance/bundle/python/materialization_sets-support/output.txt b/acceptance/bundle/python/materialization_sets-support/output.txt index 32a5aeb735d..e4f6c05d2ab 100644 --- a/acceptance/bundle/python/materialization_sets-support/output.txt +++ b/acceptance/bundle/python/materialization_sets-support/output.txt @@ -42,6 +42,86 @@ } } ] + }, + "python_external_trigger": { + "materialized_features": [ + { + "external_trigger": {}, + "feature_name": "main.default.python_external_feature_updated", + "offline_store_config": { + "catalog_name": "main", + "schema_name": "default", + "table_name_prefix": "python_external_feature_off" + } + } + ] + }, + "python_streaming_mode": { + "materialized_features": [ + { + "feature_name": "main.default.python_streaming_feature_updated", + "online_store_config": { + "catalog_name": "main", + "online_store_name": "my_online_store", + "schema_name": "default", + "table_name_prefix": "python_streaming_feature_on" + }, + "streaming_mode": {} + } + ] + }, + "python_table_trigger": { + "materialized_features": [ + { + "feature_name": "main.default.python_table_feature_updated", + "offline_store_config": { + "catalog_name": "main", + "schema_name": "default", + "table_name_prefix": "python_table_feature_off" + }, + "table_trigger": {} + } + ] + }, + "yaml_external_trigger": { + "materialized_features": [ + { + "external_trigger": {}, + "feature_name": "main.default.yaml_external_feature_updated", + "offline_store_config": { + "catalog_name": "main", + "schema_name": "default", + "table_name_prefix": "yaml_external_feature_off" + } + } + ] + }, + "yaml_streaming_mode": { + "materialized_features": [ + { + "feature_name": "main.default.yaml_streaming_feature_updated", + "online_store_config": { + "catalog_name": "main", + "online_store_name": "my_online_store", + "schema_name": "default", + "table_name_prefix": "yaml_streaming_feature_on" + }, + "streaming_mode": {} + } + ] + }, + "yaml_table_trigger": { + "materialized_features": [ + { + "feature_name": "main.default.yaml_table_feature_updated", + "offline_store_config": { + "catalog_name": "main", + "schema_name": "default", + "table_name_prefix": "yaml_table_feature_off" + }, + "table_trigger": {} + } + ] } } } diff --git a/acceptance/bundle/python/materialization_sets-support/resources.py b/acceptance/bundle/python/materialization_sets-support/resources.py index 44b87f3bc23..8e96110d870 100644 --- a/acceptance/bundle/python/materialization_sets-support/resources.py +++ b/acceptance/bundle/python/materialization_sets-support/resources.py @@ -1,4 +1,13 @@ from databricks.bundles.core import Resources +from databricks.bundles.materialization_sets import ( + ExternalTrigger, + MaterializationSet, + MaterializedFeature, + OfflineStoreConfig, + OnlineStoreConfig, + StreamingMode, + TableTrigger, +) def load_resources() -> Resources: @@ -21,4 +30,56 @@ def load_resources() -> Resources: }, ) + resources.add_materialization_set( + "python_streaming_mode", + MaterializationSet( + materialized_features=[ + MaterializedFeature( + feature_name="main.default.python_streaming_feature", + online_store_config=OnlineStoreConfig( + online_store_name="my_online_store", + catalog_name="main", + schema_name="default", + table_name_prefix="python_streaming_feature_on", + ), + streaming_mode=StreamingMode(), + ), + ], + ), + ) + + resources.add_materialization_set( + "python_table_trigger", + MaterializationSet( + materialized_features=[ + MaterializedFeature( + feature_name="main.default.python_table_feature", + offline_store_config=OfflineStoreConfig( + catalog_name="main", + schema_name="default", + table_name_prefix="python_table_feature_off", + ), + table_trigger=TableTrigger(), + ), + ], + ), + ) + + resources.add_materialization_set( + "python_external_trigger", + MaterializationSet( + materialized_features=[ + MaterializedFeature( + feature_name="main.default.python_external_feature", + offline_store_config=OfflineStoreConfig( + catalog_name="main", + schema_name="default", + table_name_prefix="python_external_feature_off", + ), + external_trigger=ExternalTrigger(), + ), + ], + ), + ) + return resources diff --git a/python/databricks/bundles/core/_transform_to_json.py b/python/databricks/bundles/core/_transform_to_json.py index a55c33fdc1f..42aad375dac 100644 --- a/python/databricks/bundles/core/_transform_to_json.py +++ b/python/databricks/bundles/core/_transform_to_json.py @@ -43,11 +43,13 @@ def _transform_to_json_object(value: Any) -> dict[str, Any]: out = {} for field_name, _ in _stable_fields(value): - field_value = _transform_to_json_value(getattr(value, field_name)) + python_value = getattr(value, field_name) + json_value = _transform_to_json_value(python_value) - # we implement omittempty semantics here skipping None, [], and {} - if field_value not in [None, [], {}]: - out[field_name] = field_value + # we implement omitempty semantics here skipping None, [], and {}, but keep an + # empty dataclass, as Go keeps a non-nil pointer to an empty struct + if is_dataclass(python_value) or json_value not in [None, [], {}]: + out[field_name] = json_value return out diff --git a/python/databricks_tests/core/test_transform.py b/python/databricks_tests/core/test_transform.py index c350fe2b7b8..433e15b76bd 100644 --- a/python/databricks_tests/core/test_transform.py +++ b/python/databricks_tests/core/test_transform.py @@ -289,6 +289,21 @@ def test_complex_cluster_spec_roundtrip(): assert cluster_spec_dict == cluster_spec_dict_2 +def test_empty_dataclass_roundtrip(): + @dataclass + class Nested: + field: Optional[int] = None + + @dataclass + class Fake: + nested: Optional[Nested] = None + + out = _transform(Fake, {"nested": {}}) + + assert out == Fake(nested=Nested()) + assert _transform_to_json_object(out) == {"nested": {}} + + def test_cron_schedule(): cron_schedule = _transform( CronSchedule, diff --git a/python/databricks_tests/core/test_transform_to_json.py b/python/databricks_tests/core/test_transform_to_json.py index e2acf7a3f0a..fb811ff01f6 100644 --- a/python/databricks_tests/core/test_transform_to_json.py +++ b/python/databricks_tests/core/test_transform_to_json.py @@ -21,6 +21,22 @@ class Fake: dict_field: dict[str, int] = field(default_factory=dict) +@dataclass +class FakeNested: + int_field: Optional[int] = None + + +@dataclass +class FakeMarker: + pass + + +@dataclass +class FakeParent: + nested_field: Optional[FakeNested] = None + marker_field: Optional[FakeMarker] = None + + @pytest.mark.parametrize( ("input", "expected"), [ @@ -37,6 +53,19 @@ class Fake: Fake(), {}, ), + ( + # empty dataclasses are kept + FakeParent(nested_field=FakeNested(), marker_field=FakeMarker()), + {"nested_field": {}, "marker_field": {}}, + ), + ( + FakeParent(nested_field=FakeNested(int_field=1)), + {"nested_field": {"int_field": 1}}, + ), + ( + FakeParent(), + {}, + ), ( [{"a": Fake(enum_field=Color.RED)}], [{"a": {"enum_field": "red"}}], From 575456c24ee40c856b88885ab7d1e0852ab2495e Mon Sep 17 00:00:00 2001 From: Daniel Bai Date: Thu, 19 Nov 2026 21:43:19 +0000 Subject: [PATCH 2/2] Check for an empty dataclass only once the value is empty A Variable is also a dataclass, but it serializes to a non-empty string, so it no longer reaches the dataclass check. Also add the changelog fragment. Co-authored-by: Isaac --- .nextchanges/bundles/7790.md | 1 + python/databricks/bundles/core/_transform_to_json.py | 2 +- python/databricks_tests/core/test_transform_to_json.py | 8 ++++++-- 3 files changed, 8 insertions(+), 3 deletions(-) create mode 100644 .nextchanges/bundles/7790.md diff --git a/.nextchanges/bundles/7790.md b/.nextchanges/bundles/7790.md new file mode 100644 index 00000000000..b31c61e99db --- /dev/null +++ b/.nextchanges/bundles/7790.md @@ -0,0 +1 @@ +* Python resources now keep an empty nested object such as `StreamingMode()` instead of dropping it, as YAML does. A job with `continuous=Continuous()` now runs continuously. ([#7790](https://github.com/databricks/cli/pull/7790)) diff --git a/python/databricks/bundles/core/_transform_to_json.py b/python/databricks/bundles/core/_transform_to_json.py index 42aad375dac..fbf32af18a6 100644 --- a/python/databricks/bundles/core/_transform_to_json.py +++ b/python/databricks/bundles/core/_transform_to_json.py @@ -48,7 +48,7 @@ def _transform_to_json_object(value: Any) -> dict[str, Any]: # we implement omitempty semantics here skipping None, [], and {}, but keep an # empty dataclass, as Go keeps a non-nil pointer to an empty struct - if is_dataclass(python_value) or json_value not in [None, [], {}]: + if json_value not in [None, [], {}] or is_dataclass(python_value): out[field_name] = json_value return out diff --git a/python/databricks_tests/core/test_transform_to_json.py b/python/databricks_tests/core/test_transform_to_json.py index fb811ff01f6..af4332e17db 100644 --- a/python/databricks_tests/core/test_transform_to_json.py +++ b/python/databricks_tests/core/test_transform_to_json.py @@ -4,7 +4,7 @@ import pytest -from databricks.bundles.core import Variable +from databricks.bundles.core import Variable, VariableOrOptional from databricks.bundles.core._transform_to_json import _transform_to_json_value @@ -33,7 +33,7 @@ class FakeMarker: @dataclass class FakeParent: - nested_field: Optional[FakeNested] = None + nested_field: VariableOrOptional[FakeNested] = None marker_field: Optional[FakeMarker] = None @@ -66,6 +66,10 @@ class FakeParent: FakeParent(), {}, ), + ( + FakeParent(nested_field=Variable(path="var.my_var", type=FakeNested)), + {"nested_field": "${var.my_var}"}, + ), ( [{"a": Fake(enum_field=Color.RED)}], [{"a": {"enum_field": "red"}}],