diff --git a/.nextchanges/bundles/7790.md b/.nextchanges/bundles/7790.md new file mode 100644 index 00000000000..b31c61e99db --- /dev/null +++ b/.nextchanges/bundles/7790.md @@ -0,0 +1 @@ +* Python resources now keep an empty nested object such as `StreamingMode()` instead of dropping it, as YAML does. A job with `continuous=Continuous()` now runs continuously. ([#7790](https://github.com/databricks/cli/pull/7790)) diff --git a/acceptance/bundle/python/data_quality_monitors-support/output.txt b/acceptance/bundle/python/data_quality_monitors-support/output.txt index 99b29fe4cb5..03c2ea62f78 100644 --- a/acceptance/bundle/python/data_quality_monitors-support/output.txt +++ b/acceptance/bundle/python/data_quality_monitors-support/output.txt @@ -25,7 +25,8 @@ "my_data_quality_monitor_2": { "data_profiling_config": { "assets_dir": "/Workspace/Users/user@example.com/assets", - "output_schema_id": "[UUID][0]" + "output_schema_id": "[UUID][0]", + "snapshot": {} }, "object_id": "[UUID][1] (updated)", "object_type": "table" diff --git a/acceptance/bundle/python/materialization_sets-support/databricks.yml b/acceptance/bundle/python/materialization_sets-support/databricks.yml index 443e1853d07..84c30ca51b6 100644 --- a/acceptance/bundle/python/materialization_sets-support/databricks.yml +++ b/acceptance/bundle/python/materialization_sets-support/databricks.yml @@ -20,3 +20,31 @@ resources: table_name_prefix: my_feature_1_off cron_schedule_trigger: mode: DERIVED + + yaml_streaming_mode: + materialized_features: + - feature_name: main.default.yaml_streaming_feature + online_store_config: + online_store_name: my_online_store + catalog_name: main + schema_name: default + table_name_prefix: yaml_streaming_feature_on + streaming_mode: {} + + yaml_table_trigger: + materialized_features: + - feature_name: main.default.yaml_table_feature + offline_store_config: + catalog_name: main + schema_name: default + table_name_prefix: yaml_table_feature_off + table_trigger: {} + + yaml_external_trigger: + materialized_features: + - feature_name: main.default.yaml_external_feature + offline_store_config: + catalog_name: main + schema_name: default + table_name_prefix: yaml_external_feature_off + external_trigger: {} diff --git a/acceptance/bundle/python/materialization_sets-support/output.txt b/acceptance/bundle/python/materialization_sets-support/output.txt index 32a5aeb735d..e4f6c05d2ab 100644 --- a/acceptance/bundle/python/materialization_sets-support/output.txt +++ b/acceptance/bundle/python/materialization_sets-support/output.txt @@ -42,6 +42,86 @@ } } ] + }, + "python_external_trigger": { + "materialized_features": [ + { + "external_trigger": {}, + "feature_name": "main.default.python_external_feature_updated", + "offline_store_config": { + "catalog_name": "main", + "schema_name": "default", + "table_name_prefix": "python_external_feature_off" + } + } + ] + }, + "python_streaming_mode": { + "materialized_features": [ + { + "feature_name": "main.default.python_streaming_feature_updated", + "online_store_config": { + "catalog_name": "main", + "online_store_name": "my_online_store", + "schema_name": "default", + "table_name_prefix": "python_streaming_feature_on" + }, + "streaming_mode": {} + } + ] + }, + "python_table_trigger": { + "materialized_features": [ + { + "feature_name": "main.default.python_table_feature_updated", + "offline_store_config": { + "catalog_name": "main", + "schema_name": "default", + "table_name_prefix": "python_table_feature_off" + }, + "table_trigger": {} + } + ] + }, + "yaml_external_trigger": { + "materialized_features": [ + { + "external_trigger": {}, + "feature_name": "main.default.yaml_external_feature_updated", + "offline_store_config": { + "catalog_name": "main", + "schema_name": "default", + "table_name_prefix": "yaml_external_feature_off" + } + } + ] + }, + "yaml_streaming_mode": { + "materialized_features": [ + { + "feature_name": "main.default.yaml_streaming_feature_updated", + "online_store_config": { + "catalog_name": "main", + "online_store_name": "my_online_store", + "schema_name": "default", + "table_name_prefix": "yaml_streaming_feature_on" + }, + "streaming_mode": {} + } + ] + }, + "yaml_table_trigger": { + "materialized_features": [ + { + "feature_name": "main.default.yaml_table_feature_updated", + "offline_store_config": { + "catalog_name": "main", + "schema_name": "default", + "table_name_prefix": "yaml_table_feature_off" + }, + "table_trigger": {} + } + ] } } } diff --git a/acceptance/bundle/python/materialization_sets-support/resources.py b/acceptance/bundle/python/materialization_sets-support/resources.py index 44b87f3bc23..8e96110d870 100644 --- a/acceptance/bundle/python/materialization_sets-support/resources.py +++ b/acceptance/bundle/python/materialization_sets-support/resources.py @@ -1,4 +1,13 @@ from databricks.bundles.core import Resources +from databricks.bundles.materialization_sets import ( + ExternalTrigger, + MaterializationSet, + MaterializedFeature, + OfflineStoreConfig, + OnlineStoreConfig, + StreamingMode, + TableTrigger, +) def load_resources() -> Resources: @@ -21,4 +30,56 @@ def load_resources() -> Resources: }, ) + resources.add_materialization_set( + "python_streaming_mode", + MaterializationSet( + materialized_features=[ + MaterializedFeature( + feature_name="main.default.python_streaming_feature", + online_store_config=OnlineStoreConfig( + online_store_name="my_online_store", + catalog_name="main", + schema_name="default", + table_name_prefix="python_streaming_feature_on", + ), + streaming_mode=StreamingMode(), + ), + ], + ), + ) + + resources.add_materialization_set( + "python_table_trigger", + MaterializationSet( + materialized_features=[ + MaterializedFeature( + feature_name="main.default.python_table_feature", + offline_store_config=OfflineStoreConfig( + catalog_name="main", + schema_name="default", + table_name_prefix="python_table_feature_off", + ), + table_trigger=TableTrigger(), + ), + ], + ), + ) + + resources.add_materialization_set( + "python_external_trigger", + MaterializationSet( + materialized_features=[ + MaterializedFeature( + feature_name="main.default.python_external_feature", + offline_store_config=OfflineStoreConfig( + catalog_name="main", + schema_name="default", + table_name_prefix="python_external_feature_off", + ), + external_trigger=ExternalTrigger(), + ), + ], + ), + ) + return resources diff --git a/python/databricks/bundles/core/_transform_to_json.py b/python/databricks/bundles/core/_transform_to_json.py index a55c33fdc1f..fbf32af18a6 100644 --- a/python/databricks/bundles/core/_transform_to_json.py +++ b/python/databricks/bundles/core/_transform_to_json.py @@ -43,11 +43,13 @@ def _transform_to_json_object(value: Any) -> dict[str, Any]: out = {} for field_name, _ in _stable_fields(value): - field_value = _transform_to_json_value(getattr(value, field_name)) + python_value = getattr(value, field_name) + json_value = _transform_to_json_value(python_value) - # we implement omittempty semantics here skipping None, [], and {} - if field_value not in [None, [], {}]: - out[field_name] = field_value + # we implement omitempty semantics here skipping None, [], and {}, but keep an + # empty dataclass, as Go keeps a non-nil pointer to an empty struct + if json_value not in [None, [], {}] or is_dataclass(python_value): + out[field_name] = json_value return out diff --git a/python/databricks_tests/core/test_transform.py b/python/databricks_tests/core/test_transform.py index c350fe2b7b8..433e15b76bd 100644 --- a/python/databricks_tests/core/test_transform.py +++ b/python/databricks_tests/core/test_transform.py @@ -289,6 +289,21 @@ def test_complex_cluster_spec_roundtrip(): assert cluster_spec_dict == cluster_spec_dict_2 +def test_empty_dataclass_roundtrip(): + @dataclass + class Nested: + field: Optional[int] = None + + @dataclass + class Fake: + nested: Optional[Nested] = None + + out = _transform(Fake, {"nested": {}}) + + assert out == Fake(nested=Nested()) + assert _transform_to_json_object(out) == {"nested": {}} + + def test_cron_schedule(): cron_schedule = _transform( CronSchedule, diff --git a/python/databricks_tests/core/test_transform_to_json.py b/python/databricks_tests/core/test_transform_to_json.py index e2acf7a3f0a..af4332e17db 100644 --- a/python/databricks_tests/core/test_transform_to_json.py +++ b/python/databricks_tests/core/test_transform_to_json.py @@ -4,7 +4,7 @@ import pytest -from databricks.bundles.core import Variable +from databricks.bundles.core import Variable, VariableOrOptional from databricks.bundles.core._transform_to_json import _transform_to_json_value @@ -21,6 +21,22 @@ class Fake: dict_field: dict[str, int] = field(default_factory=dict) +@dataclass +class FakeNested: + int_field: Optional[int] = None + + +@dataclass +class FakeMarker: + pass + + +@dataclass +class FakeParent: + nested_field: VariableOrOptional[FakeNested] = None + marker_field: Optional[FakeMarker] = None + + @pytest.mark.parametrize( ("input", "expected"), [ @@ -37,6 +53,23 @@ class Fake: Fake(), {}, ), + ( + # empty dataclasses are kept + FakeParent(nested_field=FakeNested(), marker_field=FakeMarker()), + {"nested_field": {}, "marker_field": {}}, + ), + ( + FakeParent(nested_field=FakeNested(int_field=1)), + {"nested_field": {"int_field": 1}}, + ), + ( + FakeParent(), + {}, + ), + ( + FakeParent(nested_field=Variable(path="var.my_var", type=FakeNested)), + {"nested_field": "${var.my_var}"}, + ), ( [{"a": Fake(enum_field=Color.RED)}], [{"a": {"enum_field": "red"}}],