diff --git a/.agents/rules/telemetry.md b/.agents/rules/telemetry.md index 5526dfb2007..b2448ec9434 100644 --- a/.agents/rules/telemetry.md +++ b/.agents/rules/telemetry.md @@ -34,7 +34,7 @@ The deploy payload is declared twice: the proto in the universe repo (`proto/log Other things worth knowing: -- A one-off boolean needs no proto change: add a key constant to `bundle/metrics/metrics.go` (snake_case, matching the constant name) and call `b.Metrics.SetBoolValue(...)`. It lands in `experimental.bool_values`. +- A one-off boolean needs no proto change: add a field to `bundle.Telemetry` (`bundle/telemetry.go`) with a snake_case `metric:"..."` tag and set it. Pick the type by what is measured: `bool` for a flag reported only when true, `*bool` for a measured value (nil when not measured), `BoolSet` for a measurement taken several times (e.g. once per artifact; record each with `SetBool`, and each value taken is reported once). It lands in `experimental.bool_values` and in the bundle bitmap (`bundle_deploy_event.bundle_bitmap`); run `./task generate-bitmap-schema` to append it to the bitmap schema. - Metrics that may be removed later belong in `BundleDeployExperimental`, which carries no compatibility guarantees. - Positional encodings such as the `upload_file_sizes` histogram are frozen once the metric has adoption: changing a bucket bound silently changes the meaning of every previously reported entry. - Per-resource state sizes come from the direct engine's state in `resources.json`. diff --git a/Taskfile.yml b/Taskfile.yml index 0c4184c9778..1f845900b09 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -824,6 +824,7 @@ tasks: # Refreshes acceptance/bundle/refschema/output/*.txt, which feeds # generate-direct-apitypes and generate-direct-resources below. - task: generate-refschema + - task: generate-bitmap-schema - task: generate-schema - task: generate-validation - task: generate-direct @@ -842,6 +843,7 @@ tasks: cmds: - task: generate-cligen - task: generate-refschema + - task: generate-bitmap-schema - task: generate-schema - task: generate-validation - task: generate-direct @@ -1018,6 +1020,37 @@ tasks: cmds: - go test ./acceptance -run TestAccept/bundle/refschema -update + # Appends any newly added bundle config fields to the bitmap schema (see + # cmd/bundle/bitmap). The schema is append-only, so update-schema reads the + # currently embedded schema and only appends. genschema compresses it into + # bundle/bitmap/schema.deflate, which the CLI embeds, and an identical copy in + # the databricks-bundles decoder (python/databricks/bundles/_bitmap). The + # compressed files are not reviewable; the listing in + # acceptance/bundle/bitmap/full shows every schema line with its bit. + generate-bitmap-schema: + desc: Regenerate the bitmap schema and the acceptance goldens that depend on it + sources: + - bundle/**/*.go + - exclude: bundle/**/*_test.go + - acceptance/bundle/bitmap/full/script + - acceptance/bundle/bitmap/full/split.py + - acceptance/bundle/bitmap/presence/script + - acceptance/bundle/bitmap/presence/databricks.yml + - go.mod + - go.sum + - "{{.EMBED_SOURCES}}" + generates: + - bundle/bitmap/schema.deflate + - acceptance/bundle/bitmap/full/output/*.txt + - acceptance/bundle/bitmap/presence/out.bitmap.b64 + - python/databricks/bundles/_bitmap/schema.deflate + cmds: + - go run . bundle bitmap update-schema > bundle/bitmap/schema.txt.tmp + - go run ./bundle/bitmap/genschema bundle/bitmap/schema.txt.tmp bundle/bitmap/schema.deflate python/databricks/bundles/_bitmap/schema.deflate + - rm bundle/bitmap/schema.txt.tmp + # The encoded bitmap in presence carries the schema size. + - go test ./acceptance -run 'TestAccept/bundle/bitmap/(full|presence)' -update + # Upstream field documentation comes from the checked-in .codegen/cli.json; # bundle/internal/schema/annotations.yml carries the CLI-owned docs and # overrides and is rewritten in place (synced with the config structure). diff --git a/acceptance/bundle/bitmap/full/out.test.toml b/acceptance/bundle/bitmap/full/out.test.toml new file mode 100644 index 00000000000..2ab253adc73 --- /dev/null +++ b/acceptance/bundle/bitmap/full/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = [] diff --git a/acceptance/bundle/bitmap/full/output.txt b/acceptance/bundle/bitmap/full/output.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/acceptance/bundle/bitmap/full/output/alerts.txt b/acceptance/bundle/bitmap/full/output/alerts.txt new file mode 100644 index 00000000000..6e74b70a1da --- /dev/null +++ b/acceptance/bundle/bitmap/full/output/alerts.txt @@ -0,0 +1,75 @@ +4293 resources.alerts +4294 resources.alerts.* +4295 resources.alerts.*.url +4296 resources.alerts.*.modified_status +4297 resources.alerts.*.lifecycle +4298 resources.alerts.*.lifecycle.prevent_destroy +4299 resources.alerts.*.lifecycle.prevent_destroy=true +4300 resources.alerts.*.lifecycle.prevent_destroy=false +4301 resources.alerts.*.lifecycle.depends_on +4302 resources.alerts.*.lifecycle.depends_on[*] +4303 resources.alerts.*.lifecycle.depends_on[*].resource +4304 resources.alerts.*.create_time +4305 resources.alerts.*.custom_description +4306 resources.alerts.*.custom_summary +4307 resources.alerts.*.custom_template_format +4308 resources.alerts.*.display_name +4309 resources.alerts.*.effective_run_as +4310 resources.alerts.*.effective_run_as.service_principal_name +4311 resources.alerts.*.effective_run_as.user_name +4312 resources.alerts.*.evaluation +4313 resources.alerts.*.evaluation.comparison_operator +4314 resources.alerts.*.evaluation.empty_result_state +4315 resources.alerts.*.evaluation.last_evaluated_at +4316 resources.alerts.*.evaluation.notification +4317 resources.alerts.*.evaluation.notification.notify_on_ok +4318 resources.alerts.*.evaluation.notification.notify_on_ok=true +4319 resources.alerts.*.evaluation.notification.notify_on_ok=false +4320 resources.alerts.*.evaluation.notification.retrigger_seconds +4321 resources.alerts.*.evaluation.notification.subscriptions +4322 resources.alerts.*.evaluation.notification.subscriptions[*] +4323 resources.alerts.*.evaluation.notification.subscriptions[*].destination_id +4324 resources.alerts.*.evaluation.notification.subscriptions[*].user_email +4325 resources.alerts.*.evaluation.source +4326 resources.alerts.*.evaluation.source.aggregation +4327 resources.alerts.*.evaluation.source.display +4328 resources.alerts.*.evaluation.source.name +4329 resources.alerts.*.evaluation.state +4330 resources.alerts.*.evaluation.threshold +4331 resources.alerts.*.evaluation.threshold.column +4332 resources.alerts.*.evaluation.threshold.column.aggregation +4333 resources.alerts.*.evaluation.threshold.column.display +4334 resources.alerts.*.evaluation.threshold.column.name +4335 resources.alerts.*.evaluation.threshold.value +4336 resources.alerts.*.evaluation.threshold.value.bool_value +4337 resources.alerts.*.evaluation.threshold.value.bool_value=true +4338 resources.alerts.*.evaluation.threshold.value.bool_value=false +4339 resources.alerts.*.evaluation.threshold.value.double_value +4340 resources.alerts.*.evaluation.threshold.value.string_value +4341 resources.alerts.*.id +4342 resources.alerts.*.lifecycle_state +4343 resources.alerts.*.owner_user_name +4344 resources.alerts.*.parameters +4345 resources.alerts.*.parameters[*] +4346 resources.alerts.*.parameters[*].name +4347 resources.alerts.*.parameters[*].type +4348 resources.alerts.*.parameters[*].value +4349 resources.alerts.*.parent_path +4350 resources.alerts.*.query_text +4351 resources.alerts.*.run_as +4352 resources.alerts.*.run_as.service_principal_name +4353 resources.alerts.*.run_as.user_name +4354 resources.alerts.*.run_as_user_name +4355 resources.alerts.*.schedule +4356 resources.alerts.*.schedule.pause_status +4357 resources.alerts.*.schedule.quartz_cron_schedule +4358 resources.alerts.*.schedule.timezone_id +4359 resources.alerts.*.update_time +4360 resources.alerts.*.warehouse_id +4361 resources.alerts.*.permissions +4362 resources.alerts.*.permissions[*] +4363 resources.alerts.*.permissions[*].level +4364 resources.alerts.*.permissions[*].user_name +4365 resources.alerts.*.permissions[*].service_principal_name +4366 resources.alerts.*.permissions[*].group_name +4367 resources.alerts.*.file_path diff --git a/acceptance/bundle/bitmap/full/output/apps.txt b/acceptance/bundle/bitmap/full/output/apps.txt new file mode 100644 index 00000000000..2c4e29ac030 --- /dev/null +++ b/acceptance/bundle/bitmap/full/output/apps.txt @@ -0,0 +1,265 @@ +4005 resources.apps +4006 resources.apps.* +4007 resources.apps.*.url +4008 resources.apps.*.modified_status +4009 resources.apps.*.lifecycle +4010 resources.apps.*.lifecycle.prevent_destroy +4011 resources.apps.*.lifecycle.prevent_destroy=true +4012 resources.apps.*.lifecycle.prevent_destroy=false +4013 resources.apps.*.lifecycle.depends_on +4014 resources.apps.*.lifecycle.depends_on[*] +4015 resources.apps.*.lifecycle.depends_on[*].resource +4016 resources.apps.*.active_deployment +4017 resources.apps.*.active_deployment.command +4018 resources.apps.*.active_deployment.command[*] +4019 resources.apps.*.active_deployment.create_time +4020 resources.apps.*.active_deployment.creator +4021 resources.apps.*.active_deployment.deployment_artifacts +4022 resources.apps.*.active_deployment.deployment_artifacts.source_code_path +4023 resources.apps.*.active_deployment.deployment_id +4024 resources.apps.*.active_deployment.env_vars +4025 resources.apps.*.active_deployment.env_vars[*] +4026 resources.apps.*.active_deployment.env_vars[*].name +4027 resources.apps.*.active_deployment.env_vars[*].value +4028 resources.apps.*.active_deployment.env_vars[*].value_from +4029 resources.apps.*.active_deployment.git_source +4030 resources.apps.*.active_deployment.git_source.branch +4031 resources.apps.*.active_deployment.git_source.commit +4032 resources.apps.*.active_deployment.git_source.git_repository +4033 resources.apps.*.active_deployment.git_source.git_repository.auto_deploy +4034 resources.apps.*.active_deployment.git_source.git_repository.auto_deploy=true +4035 resources.apps.*.active_deployment.git_source.git_repository.auto_deploy=false +4036 resources.apps.*.active_deployment.git_source.git_repository.caller_credential_id +4037 resources.apps.*.active_deployment.git_source.git_repository.provider +4038 resources.apps.*.active_deployment.git_source.git_repository.url +4039 resources.apps.*.active_deployment.git_source.resolved_commit +4040 resources.apps.*.active_deployment.git_source.source_code_path +4041 resources.apps.*.active_deployment.git_source.tag +4042 resources.apps.*.active_deployment.health_check +4043 resources.apps.*.active_deployment.health_check.path +4044 resources.apps.*.active_deployment.health_check.timeout +4045 resources.apps.*.active_deployment.mode +4046 resources.apps.*.active_deployment.scan_omitted_count_is_lower_bound +4047 resources.apps.*.active_deployment.scan_omitted_count_is_lower_bound=true +4048 resources.apps.*.active_deployment.scan_omitted_count_is_lower_bound=false +4049 resources.apps.*.active_deployment.scan_omitted_findings_count +4050 resources.apps.*.active_deployment.scan_status +4051 resources.apps.*.active_deployment.source_code_path +4052 resources.apps.*.active_deployment.status +4053 resources.apps.*.active_deployment.status.message +4054 resources.apps.*.active_deployment.status.state +4055 resources.apps.*.active_deployment.update_time +4056 resources.apps.*.app_status +4057 resources.apps.*.app_status.message +4058 resources.apps.*.app_status.running_instances +4059 resources.apps.*.app_status.state +4060 resources.apps.*.budget_policy_id +4061 resources.apps.*.compute_max_instances +4062 resources.apps.*.compute_min_instances +4063 resources.apps.*.compute_size +4064 resources.apps.*.compute_status +4065 resources.apps.*.compute_status.active_instances +4066 resources.apps.*.compute_status.message +4067 resources.apps.*.compute_status.state +4068 resources.apps.*.create_time +4069 resources.apps.*.creator +4070 resources.apps.*.default_git_source +4071 resources.apps.*.default_git_source.branch +4072 resources.apps.*.default_git_source.commit +4073 resources.apps.*.default_git_source.git_repository +4074 resources.apps.*.default_git_source.git_repository.auto_deploy +4075 resources.apps.*.default_git_source.git_repository.auto_deploy=true +4076 resources.apps.*.default_git_source.git_repository.auto_deploy=false +4077 resources.apps.*.default_git_source.git_repository.caller_credential_id +4078 resources.apps.*.default_git_source.git_repository.provider +4079 resources.apps.*.default_git_source.git_repository.url +4080 resources.apps.*.default_git_source.resolved_commit +4081 resources.apps.*.default_git_source.source_code_path +4082 resources.apps.*.default_git_source.tag +4083 resources.apps.*.default_source_code_path +4084 resources.apps.*.description +4085 resources.apps.*.effective_budget_policy_id +4086 resources.apps.*.effective_resources +4087 resources.apps.*.effective_resources[*] +4088 resources.apps.*.effective_resources[*].app +4089 resources.apps.*.effective_resources[*].app.name +4090 resources.apps.*.effective_resources[*].app.permission +4091 resources.apps.*.effective_resources[*].database +4092 resources.apps.*.effective_resources[*].database.database_name +4093 resources.apps.*.effective_resources[*].database.instance_name +4094 resources.apps.*.effective_resources[*].database.permission +4095 resources.apps.*.effective_resources[*].description +4096 resources.apps.*.effective_resources[*].experiment +4097 resources.apps.*.effective_resources[*].experiment.experiment_id +4098 resources.apps.*.effective_resources[*].experiment.permission +4099 resources.apps.*.effective_resources[*].genie_space +4100 resources.apps.*.effective_resources[*].genie_space.name +4101 resources.apps.*.effective_resources[*].genie_space.permission +4102 resources.apps.*.effective_resources[*].genie_space.space_id +4103 resources.apps.*.effective_resources[*].job +4104 resources.apps.*.effective_resources[*].job.id +4105 resources.apps.*.effective_resources[*].job.permission +4106 resources.apps.*.effective_resources[*].name +4107 resources.apps.*.effective_resources[*].postgres +4108 resources.apps.*.effective_resources[*].postgres.branch +4109 resources.apps.*.effective_resources[*].postgres.database +4110 resources.apps.*.effective_resources[*].postgres.permission +4111 resources.apps.*.effective_resources[*].secret +4112 resources.apps.*.effective_resources[*].secret.key +4113 resources.apps.*.effective_resources[*].secret.permission +4114 resources.apps.*.effective_resources[*].secret.scope +4115 resources.apps.*.effective_resources[*].serving_endpoint +4116 resources.apps.*.effective_resources[*].serving_endpoint.name +4117 resources.apps.*.effective_resources[*].serving_endpoint.permission +4118 resources.apps.*.effective_resources[*].sql_warehouse +4119 resources.apps.*.effective_resources[*].sql_warehouse.id +4120 resources.apps.*.effective_resources[*].sql_warehouse.permission +4121 resources.apps.*.effective_resources[*].uc_securable +4122 resources.apps.*.effective_resources[*].uc_securable.permission +4123 resources.apps.*.effective_resources[*].uc_securable.securable_full_name +4124 resources.apps.*.effective_resources[*].uc_securable.securable_kind +4125 resources.apps.*.effective_resources[*].uc_securable.securable_type +4126 resources.apps.*.effective_usage_policy_id +4127 resources.apps.*.effective_user_api_scopes +4128 resources.apps.*.effective_user_api_scopes[*] +4129 resources.apps.*.forward_user_access_token +4130 resources.apps.*.forward_user_access_token=true +4131 resources.apps.*.forward_user_access_token=false +4132 resources.apps.*.git_repository +4133 resources.apps.*.git_repository.auto_deploy +4134 resources.apps.*.git_repository.auto_deploy=true +4135 resources.apps.*.git_repository.auto_deploy=false +4136 resources.apps.*.git_repository.caller_credential_id +4137 resources.apps.*.git_repository.provider +4138 resources.apps.*.git_repository.url +4139 resources.apps.*.git_source +4140 resources.apps.*.git_source.branch +4141 resources.apps.*.git_source.commit +4142 resources.apps.*.git_source.git_repository +4143 resources.apps.*.git_source.git_repository.auto_deploy +4144 resources.apps.*.git_source.git_repository.auto_deploy=true +4145 resources.apps.*.git_source.git_repository.auto_deploy=false +4146 resources.apps.*.git_source.git_repository.caller_credential_id +4147 resources.apps.*.git_source.git_repository.provider +4148 resources.apps.*.git_source.git_repository.url +4149 resources.apps.*.git_source.resolved_commit +4150 resources.apps.*.git_source.source_code_path +4151 resources.apps.*.git_source.tag +4152 resources.apps.*.id +4153 resources.apps.*.name +4154 resources.apps.*.oauth2_app_client_id +4155 resources.apps.*.oauth2_app_integration_id +4156 resources.apps.*.pending_deployment +4157 resources.apps.*.pending_deployment.command +4158 resources.apps.*.pending_deployment.command[*] +4159 resources.apps.*.pending_deployment.create_time +4160 resources.apps.*.pending_deployment.creator +4161 resources.apps.*.pending_deployment.deployment_artifacts +4162 resources.apps.*.pending_deployment.deployment_artifacts.source_code_path +4163 resources.apps.*.pending_deployment.deployment_id +4164 resources.apps.*.pending_deployment.env_vars +4165 resources.apps.*.pending_deployment.env_vars[*] +4166 resources.apps.*.pending_deployment.env_vars[*].name +4167 resources.apps.*.pending_deployment.env_vars[*].value +4168 resources.apps.*.pending_deployment.env_vars[*].value_from +4169 resources.apps.*.pending_deployment.git_source +4170 resources.apps.*.pending_deployment.git_source.branch +4171 resources.apps.*.pending_deployment.git_source.commit +4172 resources.apps.*.pending_deployment.git_source.git_repository +4173 resources.apps.*.pending_deployment.git_source.git_repository.auto_deploy +4174 resources.apps.*.pending_deployment.git_source.git_repository.auto_deploy=true +4175 resources.apps.*.pending_deployment.git_source.git_repository.auto_deploy=false +4176 resources.apps.*.pending_deployment.git_source.git_repository.caller_credential_id +4177 resources.apps.*.pending_deployment.git_source.git_repository.provider +4178 resources.apps.*.pending_deployment.git_source.git_repository.url +4179 resources.apps.*.pending_deployment.git_source.resolved_commit +4180 resources.apps.*.pending_deployment.git_source.source_code_path +4181 resources.apps.*.pending_deployment.git_source.tag +4182 resources.apps.*.pending_deployment.health_check +4183 resources.apps.*.pending_deployment.health_check.path +4184 resources.apps.*.pending_deployment.health_check.timeout +4185 resources.apps.*.pending_deployment.mode +4186 resources.apps.*.pending_deployment.scan_omitted_count_is_lower_bound +4187 resources.apps.*.pending_deployment.scan_omitted_count_is_lower_bound=true +4188 resources.apps.*.pending_deployment.scan_omitted_count_is_lower_bound=false +4189 resources.apps.*.pending_deployment.scan_omitted_findings_count +4190 resources.apps.*.pending_deployment.scan_status +4191 resources.apps.*.pending_deployment.source_code_path +4192 resources.apps.*.pending_deployment.status +4193 resources.apps.*.pending_deployment.status.message +4194 resources.apps.*.pending_deployment.status.state +4195 resources.apps.*.pending_deployment.update_time +4196 resources.apps.*.resources +4197 resources.apps.*.resources[*] +4198 resources.apps.*.resources[*].app +4199 resources.apps.*.resources[*].app.name +4200 resources.apps.*.resources[*].app.permission +4201 resources.apps.*.resources[*].database +4202 resources.apps.*.resources[*].database.database_name +4203 resources.apps.*.resources[*].database.instance_name +4204 resources.apps.*.resources[*].database.permission +4205 resources.apps.*.resources[*].description +4206 resources.apps.*.resources[*].experiment +4207 resources.apps.*.resources[*].experiment.experiment_id +4208 resources.apps.*.resources[*].experiment.permission +4209 resources.apps.*.resources[*].genie_space +4210 resources.apps.*.resources[*].genie_space.name +4211 resources.apps.*.resources[*].genie_space.permission +4212 resources.apps.*.resources[*].genie_space.space_id +4213 resources.apps.*.resources[*].job +4214 resources.apps.*.resources[*].job.id +4215 resources.apps.*.resources[*].job.permission +4216 resources.apps.*.resources[*].name +4217 resources.apps.*.resources[*].postgres +4218 resources.apps.*.resources[*].postgres.branch +4219 resources.apps.*.resources[*].postgres.database +4220 resources.apps.*.resources[*].postgres.permission +4221 resources.apps.*.resources[*].secret +4222 resources.apps.*.resources[*].secret.key +4223 resources.apps.*.resources[*].secret.permission +4224 resources.apps.*.resources[*].secret.scope +4225 resources.apps.*.resources[*].serving_endpoint +4226 resources.apps.*.resources[*].serving_endpoint.name +4227 resources.apps.*.resources[*].serving_endpoint.permission +4228 resources.apps.*.resources[*].sql_warehouse +4229 resources.apps.*.resources[*].sql_warehouse.id +4230 resources.apps.*.resources[*].sql_warehouse.permission +4231 resources.apps.*.resources[*].uc_securable +4232 resources.apps.*.resources[*].uc_securable.permission +4233 resources.apps.*.resources[*].uc_securable.securable_full_name +4234 resources.apps.*.resources[*].uc_securable.securable_kind +4235 resources.apps.*.resources[*].uc_securable.securable_type +4236 resources.apps.*.service_principal_client_id +4237 resources.apps.*.service_principal_id +4238 resources.apps.*.service_principal_name +4239 resources.apps.*.source_code_path +4240 resources.apps.*.space +4241 resources.apps.*.telemetry_export_destinations +4242 resources.apps.*.telemetry_export_destinations[*] +4243 resources.apps.*.telemetry_export_destinations[*].unity_catalog +4244 resources.apps.*.telemetry_export_destinations[*].unity_catalog.logs_table +4245 resources.apps.*.telemetry_export_destinations[*].unity_catalog.metrics_table +4246 resources.apps.*.telemetry_export_destinations[*].unity_catalog.traces_table +4247 resources.apps.*.thumbnail_url +4248 resources.apps.*.update_time +4249 resources.apps.*.updater +4250 resources.apps.*.usage_policy_id +4251 resources.apps.*.user_api_scopes +4252 resources.apps.*.user_api_scopes[*] +4253 resources.apps.*.lifecycle.started +4254 resources.apps.*.lifecycle.started=true +4255 resources.apps.*.lifecycle.started=false +4256 resources.apps.*.config +4257 resources.apps.*.config.command +4258 resources.apps.*.config.command[*] +4259 resources.apps.*.config.env +4260 resources.apps.*.config.env[*] +4261 resources.apps.*.config.env[*].name +4262 resources.apps.*.config.env[*].value +4263 resources.apps.*.config.env[*].value_from +4264 resources.apps.*.permissions +4265 resources.apps.*.permissions[*] +4266 resources.apps.*.permissions[*].level +4267 resources.apps.*.permissions[*].user_name +4268 resources.apps.*.permissions[*].service_principal_name +4269 resources.apps.*.permissions[*].group_name diff --git a/acceptance/bundle/bitmap/full/output/catalogs.txt b/acceptance/bundle/bitmap/full/output/catalogs.txt new file mode 100644 index 00000000000..18ea283b209 --- /dev/null +++ b/acceptance/bundle/bitmap/full/output/catalogs.txt @@ -0,0 +1,36 @@ +3607 resources.catalogs +3608 resources.catalogs.* +3609 resources.catalogs.*.url +3610 resources.catalogs.*.modified_status +3611 resources.catalogs.*.lifecycle +3612 resources.catalogs.*.lifecycle.prevent_destroy +3613 resources.catalogs.*.lifecycle.prevent_destroy=true +3614 resources.catalogs.*.lifecycle.prevent_destroy=false +3615 resources.catalogs.*.lifecycle.depends_on +3616 resources.catalogs.*.lifecycle.depends_on[*] +3617 resources.catalogs.*.lifecycle.depends_on[*].resource +3618 resources.catalogs.*.id +3619 resources.catalogs.*.comment +3620 resources.catalogs.*.connection_name +3621 resources.catalogs.*.custom_max_retention_hours +3622 resources.catalogs.*.managed_encryption_settings +3623 resources.catalogs.*.managed_encryption_settings.azure_encryption_settings +3624 resources.catalogs.*.managed_encryption_settings.azure_encryption_settings.azure_cmk_access_connector_id +3625 resources.catalogs.*.managed_encryption_settings.azure_encryption_settings.azure_cmk_managed_identity_id +3626 resources.catalogs.*.managed_encryption_settings.azure_encryption_settings.azure_tenant_id +3627 resources.catalogs.*.managed_encryption_settings.azure_key_vault_key_id +3628 resources.catalogs.*.managed_encryption_settings.customer_managed_key_id +3629 resources.catalogs.*.name +3630 resources.catalogs.*.options +3631 resources.catalogs.*.options.* +3632 resources.catalogs.*.properties +3633 resources.catalogs.*.properties.* +3634 resources.catalogs.*.provider_name +3635 resources.catalogs.*.share_name +3636 resources.catalogs.*.storage_root +3637 resources.catalogs.*.isolation_mode +3638 resources.catalogs.*.grants +3639 resources.catalogs.*.grants[*] +3640 resources.catalogs.*.grants[*].principal +3641 resources.catalogs.*.grants[*].privileges +3642 resources.catalogs.*.grants[*].privileges[*] diff --git a/acceptance/bundle/bitmap/full/output/cluster_policies.txt b/acceptance/bundle/bitmap/full/output/cluster_policies.txt new file mode 100644 index 00000000000..d7ca99fc487 --- /dev/null +++ b/acceptance/bundle/bitmap/full/output/cluster_policies.txt @@ -0,0 +1,41 @@ +5033 resources.cluster_policies +5034 resources.cluster_policies.* +5035 resources.cluster_policies.*.url +5036 resources.cluster_policies.*.modified_status +5037 resources.cluster_policies.*.lifecycle +5038 resources.cluster_policies.*.lifecycle.prevent_destroy +5039 resources.cluster_policies.*.lifecycle.prevent_destroy=true +5040 resources.cluster_policies.*.lifecycle.prevent_destroy=false +5041 resources.cluster_policies.*.lifecycle.depends_on +5042 resources.cluster_policies.*.lifecycle.depends_on[*] +5043 resources.cluster_policies.*.lifecycle.depends_on[*].resource +5044 resources.cluster_policies.*.id +5045 resources.cluster_policies.*.definition +5046 resources.cluster_policies.*.description +5047 resources.cluster_policies.*.libraries +5048 resources.cluster_policies.*.libraries[*] +5049 resources.cluster_policies.*.libraries[*].cran +5050 resources.cluster_policies.*.libraries[*].cran.package +5051 resources.cluster_policies.*.libraries[*].cran.repo +5052 resources.cluster_policies.*.libraries[*].egg +5053 resources.cluster_policies.*.libraries[*].jar +5054 resources.cluster_policies.*.libraries[*].maven +5055 resources.cluster_policies.*.libraries[*].maven.coordinates +5056 resources.cluster_policies.*.libraries[*].maven.exclusions +5057 resources.cluster_policies.*.libraries[*].maven.exclusions[*] +5058 resources.cluster_policies.*.libraries[*].maven.repo +5059 resources.cluster_policies.*.libraries[*].pypi +5060 resources.cluster_policies.*.libraries[*].pypi.package +5061 resources.cluster_policies.*.libraries[*].pypi.repo +5062 resources.cluster_policies.*.libraries[*].requirements +5063 resources.cluster_policies.*.libraries[*].whl +5064 resources.cluster_policies.*.max_clusters_per_user +5065 resources.cluster_policies.*.name +5066 resources.cluster_policies.*.policy_family_definition_overrides +5067 resources.cluster_policies.*.policy_family_id +5068 resources.cluster_policies.*.permissions +5069 resources.cluster_policies.*.permissions[*] +5070 resources.cluster_policies.*.permissions[*].level +5071 resources.cluster_policies.*.permissions[*].user_name +5072 resources.cluster_policies.*.permissions[*].service_principal_name +5073 resources.cluster_policies.*.permissions[*].group_name diff --git a/acceptance/bundle/bitmap/full/output/clusters.txt b/acceptance/bundle/bitmap/full/output/clusters.txt new file mode 100644 index 00000000000..60de699bdbd --- /dev/null +++ b/acceptance/bundle/bitmap/full/output/clusters.txt @@ -0,0 +1,170 @@ +3776 resources.clusters +3777 resources.clusters.* +3778 resources.clusters.*.url +3779 resources.clusters.*.modified_status +3780 resources.clusters.*.lifecycle +3781 resources.clusters.*.lifecycle.prevent_destroy +3782 resources.clusters.*.lifecycle.prevent_destroy=true +3783 resources.clusters.*.lifecycle.prevent_destroy=false +3784 resources.clusters.*.lifecycle.depends_on +3785 resources.clusters.*.lifecycle.depends_on[*] +3786 resources.clusters.*.lifecycle.depends_on[*].resource +3787 resources.clusters.*.id +3788 resources.clusters.*.apply_policy_default_values +3789 resources.clusters.*.apply_policy_default_values=true +3790 resources.clusters.*.apply_policy_default_values=false +3791 resources.clusters.*.autoscale +3792 resources.clusters.*.autoscale.max_workers +3793 resources.clusters.*.autoscale.min_workers +3794 resources.clusters.*.autotermination_minutes +3795 resources.clusters.*.aws_attributes +3796 resources.clusters.*.aws_attributes.availability +3797 resources.clusters.*.aws_attributes.ebs_volume_count +3798 resources.clusters.*.aws_attributes.ebs_volume_iops +3799 resources.clusters.*.aws_attributes.ebs_volume_size +3800 resources.clusters.*.aws_attributes.ebs_volume_throughput +3801 resources.clusters.*.aws_attributes.ebs_volume_type +3802 resources.clusters.*.aws_attributes.first_on_demand +3803 resources.clusters.*.aws_attributes.instance_profile_arn +3804 resources.clusters.*.aws_attributes.spot_bid_price_percent +3805 resources.clusters.*.aws_attributes.zone_id +3806 resources.clusters.*.azure_attributes +3807 resources.clusters.*.azure_attributes.availability +3808 resources.clusters.*.azure_attributes.capacity_reservation_group +3809 resources.clusters.*.azure_attributes.first_on_demand +3810 resources.clusters.*.azure_attributes.log_analytics_info +3811 resources.clusters.*.azure_attributes.log_analytics_info.log_analytics_primary_key +3812 resources.clusters.*.azure_attributes.log_analytics_info.log_analytics_workspace_id +3813 resources.clusters.*.azure_attributes.spot_bid_max_price +3814 resources.clusters.*.cluster_log_conf +3815 resources.clusters.*.cluster_log_conf.dbfs +3816 resources.clusters.*.cluster_log_conf.dbfs.destination +3817 resources.clusters.*.cluster_log_conf.s3 +3818 resources.clusters.*.cluster_log_conf.s3.canned_acl +3819 resources.clusters.*.cluster_log_conf.s3.destination +3820 resources.clusters.*.cluster_log_conf.s3.enable_encryption +3821 resources.clusters.*.cluster_log_conf.s3.enable_encryption=true +3822 resources.clusters.*.cluster_log_conf.s3.enable_encryption=false +3823 resources.clusters.*.cluster_log_conf.s3.encryption_type +3824 resources.clusters.*.cluster_log_conf.s3.endpoint +3825 resources.clusters.*.cluster_log_conf.s3.kms_key +3826 resources.clusters.*.cluster_log_conf.s3.region +3827 resources.clusters.*.cluster_log_conf.volumes +3828 resources.clusters.*.cluster_log_conf.volumes.destination +3829 resources.clusters.*.cluster_name +3830 resources.clusters.*.custom_tags +3831 resources.clusters.*.custom_tags.* +3832 resources.clusters.*.data_security_mode +3833 resources.clusters.*.dependency_mode +3834 resources.clusters.*.docker_image +3835 resources.clusters.*.docker_image.basic_auth +3836 resources.clusters.*.docker_image.basic_auth.password +3837 resources.clusters.*.docker_image.basic_auth.username +3838 resources.clusters.*.docker_image.url +3839 resources.clusters.*.driver_instance_pool_id +3840 resources.clusters.*.driver_node_type_flexibility +3841 resources.clusters.*.driver_node_type_flexibility.alternate_node_type_ids +3842 resources.clusters.*.driver_node_type_flexibility.alternate_node_type_ids[*] +3843 resources.clusters.*.driver_node_type_flexibility.aws_context_id +3844 resources.clusters.*.driver_node_type_id +3845 resources.clusters.*.enable_elastic_disk +3846 resources.clusters.*.enable_elastic_disk=true +3847 resources.clusters.*.enable_elastic_disk=false +3848 resources.clusters.*.enable_local_disk_encryption +3849 resources.clusters.*.enable_local_disk_encryption=true +3850 resources.clusters.*.enable_local_disk_encryption=false +3851 resources.clusters.*.gcp_attributes +3852 resources.clusters.*.gcp_attributes.availability +3853 resources.clusters.*.gcp_attributes.boot_disk_size +3854 resources.clusters.*.gcp_attributes.confidential_compute_type +3855 resources.clusters.*.gcp_attributes.first_on_demand +3856 resources.clusters.*.gcp_attributes.google_service_account +3857 resources.clusters.*.gcp_attributes.local_ssd_count +3858 resources.clusters.*.gcp_attributes.use_preemptible_executors +3859 resources.clusters.*.gcp_attributes.use_preemptible_executors=true +3860 resources.clusters.*.gcp_attributes.use_preemptible_executors=false +3861 resources.clusters.*.gcp_attributes.zone_id +3862 resources.clusters.*.init_scripts +3863 resources.clusters.*.init_scripts[*] +3864 resources.clusters.*.init_scripts[*].abfss +3865 resources.clusters.*.init_scripts[*].abfss.destination +3866 resources.clusters.*.init_scripts[*].dbfs +3867 resources.clusters.*.init_scripts[*].dbfs.destination +3868 resources.clusters.*.init_scripts[*].file +3869 resources.clusters.*.init_scripts[*].file.destination +3870 resources.clusters.*.init_scripts[*].gcs +3871 resources.clusters.*.init_scripts[*].gcs.destination +3872 resources.clusters.*.init_scripts[*].s3 +3873 resources.clusters.*.init_scripts[*].s3.canned_acl +3874 resources.clusters.*.init_scripts[*].s3.destination +3875 resources.clusters.*.init_scripts[*].s3.enable_encryption +3876 resources.clusters.*.init_scripts[*].s3.enable_encryption=true +3877 resources.clusters.*.init_scripts[*].s3.enable_encryption=false +3878 resources.clusters.*.init_scripts[*].s3.encryption_type +3879 resources.clusters.*.init_scripts[*].s3.endpoint +3880 resources.clusters.*.init_scripts[*].s3.kms_key +3881 resources.clusters.*.init_scripts[*].s3.region +3882 resources.clusters.*.init_scripts[*].volumes +3883 resources.clusters.*.init_scripts[*].volumes.destination +3884 resources.clusters.*.init_scripts[*].workspace +3885 resources.clusters.*.init_scripts[*].workspace.destination +3886 resources.clusters.*.instance_pool_id +3887 resources.clusters.*.is_single_node +3888 resources.clusters.*.is_single_node=true +3889 resources.clusters.*.is_single_node=false +3890 resources.clusters.*.kind +3891 resources.clusters.*.node_type_id +3892 resources.clusters.*.num_workers +3893 resources.clusters.*.policy_id +3894 resources.clusters.*.remote_disk_throughput +3895 resources.clusters.*.runtime_engine +3896 resources.clusters.*.single_user_name +3897 resources.clusters.*.spark_conf +3898 resources.clusters.*.spark_conf.* +3899 resources.clusters.*.spark_env_vars +3900 resources.clusters.*.spark_env_vars.* +3901 resources.clusters.*.spark_version +3902 resources.clusters.*.ssh_public_keys +3903 resources.clusters.*.ssh_public_keys[*] +3904 resources.clusters.*.total_initial_remote_disk_size +3905 resources.clusters.*.use_ml_runtime +3906 resources.clusters.*.use_ml_runtime=true +3907 resources.clusters.*.use_ml_runtime=false +3908 resources.clusters.*.worker_node_type_flexibility +3909 resources.clusters.*.worker_node_type_flexibility.alternate_node_type_ids +3910 resources.clusters.*.worker_node_type_flexibility.alternate_node_type_ids[*] +3911 resources.clusters.*.worker_node_type_flexibility.aws_context_id +3912 resources.clusters.*.workload_type +3913 resources.clusters.*.workload_type.clients +3914 resources.clusters.*.workload_type.clients.jobs +3915 resources.clusters.*.workload_type.clients.jobs=true +3916 resources.clusters.*.workload_type.clients.jobs=false +3917 resources.clusters.*.workload_type.clients.notebooks +3918 resources.clusters.*.workload_type.clients.notebooks=true +3919 resources.clusters.*.workload_type.clients.notebooks=false +3920 resources.clusters.*.lifecycle.started +3921 resources.clusters.*.lifecycle.started=true +3922 resources.clusters.*.lifecycle.started=false +3923 resources.clusters.*.libraries +3924 resources.clusters.*.libraries[*] +3925 resources.clusters.*.libraries[*].cran +3926 resources.clusters.*.libraries[*].cran.package +3927 resources.clusters.*.libraries[*].cran.repo +3928 resources.clusters.*.libraries[*].egg +3929 resources.clusters.*.libraries[*].jar +3930 resources.clusters.*.libraries[*].maven +3931 resources.clusters.*.libraries[*].maven.coordinates +3932 resources.clusters.*.libraries[*].maven.exclusions +3933 resources.clusters.*.libraries[*].maven.exclusions[*] +3934 resources.clusters.*.libraries[*].maven.repo +3935 resources.clusters.*.libraries[*].pypi +3936 resources.clusters.*.libraries[*].pypi.package +3937 resources.clusters.*.libraries[*].pypi.repo +3938 resources.clusters.*.libraries[*].requirements +3939 resources.clusters.*.libraries[*].whl +3940 resources.clusters.*.permissions +3941 resources.clusters.*.permissions[*] +3942 resources.clusters.*.permissions[*].level +3943 resources.clusters.*.permissions[*].user_name +3944 resources.clusters.*.permissions[*].service_principal_name +3945 resources.clusters.*.permissions[*].group_name diff --git a/acceptance/bundle/bitmap/full/output/dashboards.txt b/acceptance/bundle/bitmap/full/output/dashboards.txt new file mode 100644 index 00000000000..3710b0334b4 --- /dev/null +++ b/acceptance/bundle/bitmap/full/output/dashboards.txt @@ -0,0 +1,34 @@ +3946 resources.dashboards +3947 resources.dashboards.* +3948 resources.dashboards.*.url +3949 resources.dashboards.*.modified_status +3950 resources.dashboards.*.lifecycle +3951 resources.dashboards.*.lifecycle.prevent_destroy +3952 resources.dashboards.*.lifecycle.prevent_destroy=true +3953 resources.dashboards.*.lifecycle.prevent_destroy=false +3954 resources.dashboards.*.lifecycle.depends_on +3955 resources.dashboards.*.lifecycle.depends_on[*] +3956 resources.dashboards.*.lifecycle.depends_on[*].resource +3957 resources.dashboards.*.id +3958 resources.dashboards.*.create_time +3959 resources.dashboards.*.dashboard_id +3960 resources.dashboards.*.display_name +3961 resources.dashboards.*.etag +3962 resources.dashboards.*.lifecycle_state +3963 resources.dashboards.*.parent_path +3964 resources.dashboards.*.path +3965 resources.dashboards.*.update_time +3966 resources.dashboards.*.warehouse_id +3967 resources.dashboards.*.serialized_dashboard +3968 resources.dashboards.*.embed_credentials +3969 resources.dashboards.*.embed_credentials=true +3970 resources.dashboards.*.embed_credentials=false +3971 resources.dashboards.*.dataset_catalog +3972 resources.dashboards.*.dataset_schema +3973 resources.dashboards.*.permissions +3974 resources.dashboards.*.permissions[*] +3975 resources.dashboards.*.permissions[*].level +3976 resources.dashboards.*.permissions[*].user_name +3977 resources.dashboards.*.permissions[*].service_principal_name +3978 resources.dashboards.*.permissions[*].group_name +3979 resources.dashboards.*.file_path diff --git a/acceptance/bundle/bitmap/full/output/data_quality_monitors.txt b/acceptance/bundle/bitmap/full/output/data_quality_monitors.txt new file mode 100644 index 00000000000..2e30f4c78ad --- /dev/null +++ b/acceptance/bundle/bitmap/full/output/data_quality_monitors.txt @@ -0,0 +1,90 @@ +3517 resources.data_quality_monitors +3518 resources.data_quality_monitors.* +3519 resources.data_quality_monitors.*.url +3520 resources.data_quality_monitors.*.modified_status +3521 resources.data_quality_monitors.*.lifecycle +3522 resources.data_quality_monitors.*.lifecycle.prevent_destroy +3523 resources.data_quality_monitors.*.lifecycle.prevent_destroy=true +3524 resources.data_quality_monitors.*.lifecycle.prevent_destroy=false +3525 resources.data_quality_monitors.*.lifecycle.depends_on +3526 resources.data_quality_monitors.*.lifecycle.depends_on[*] +3527 resources.data_quality_monitors.*.lifecycle.depends_on[*].resource +3528 resources.data_quality_monitors.*.id +3529 resources.data_quality_monitors.*.anomaly_detection_config +3530 resources.data_quality_monitors.*.anomaly_detection_config.excluded_table_full_names +3531 resources.data_quality_monitors.*.anomaly_detection_config.excluded_table_full_names[*] +3532 resources.data_quality_monitors.*.anomaly_detection_config.notification_settings +3533 resources.data_quality_monitors.*.anomaly_detection_config.notification_settings.on_failure +3534 resources.data_quality_monitors.*.anomaly_detection_config.notification_settings.on_failure.destination_ids +3535 resources.data_quality_monitors.*.anomaly_detection_config.notification_settings.on_failure.destination_ids[*] +3536 resources.data_quality_monitors.*.anomaly_detection_config.notification_settings.on_failure.destination_user_ids +3537 resources.data_quality_monitors.*.anomaly_detection_config.notification_settings.on_failure.destination_user_ids[*] +3538 resources.data_quality_monitors.*.anomaly_detection_config.notification_settings.on_failure.email_addresses +3539 resources.data_quality_monitors.*.anomaly_detection_config.notification_settings.on_failure.email_addresses[*] +3540 resources.data_quality_monitors.*.anomaly_detection_config.notification_settings.on_unhealthy_table +3541 resources.data_quality_monitors.*.anomaly_detection_config.notification_settings.on_unhealthy_table.destination_ids +3542 resources.data_quality_monitors.*.anomaly_detection_config.notification_settings.on_unhealthy_table.destination_ids[*] +3543 resources.data_quality_monitors.*.anomaly_detection_config.notification_settings.on_unhealthy_table.destination_user_ids +3544 resources.data_quality_monitors.*.anomaly_detection_config.notification_settings.on_unhealthy_table.destination_user_ids[*] +3545 resources.data_quality_monitors.*.anomaly_detection_config.notification_settings.on_unhealthy_table.email_addresses +3546 resources.data_quality_monitors.*.anomaly_detection_config.notification_settings.on_unhealthy_table.email_addresses[*] +3547 resources.data_quality_monitors.*.data_profiling_config +3548 resources.data_quality_monitors.*.data_profiling_config.assets_dir +3549 resources.data_quality_monitors.*.data_profiling_config.baseline_table_name +3550 resources.data_quality_monitors.*.data_profiling_config.custom_metrics +3551 resources.data_quality_monitors.*.data_profiling_config.custom_metrics[*] +3552 resources.data_quality_monitors.*.data_profiling_config.custom_metrics[*].definition +3553 resources.data_quality_monitors.*.data_profiling_config.custom_metrics[*].input_columns +3554 resources.data_quality_monitors.*.data_profiling_config.custom_metrics[*].input_columns[*] +3555 resources.data_quality_monitors.*.data_profiling_config.custom_metrics[*].name +3556 resources.data_quality_monitors.*.data_profiling_config.custom_metrics[*].output_data_type +3557 resources.data_quality_monitors.*.data_profiling_config.custom_metrics[*].type +3558 resources.data_quality_monitors.*.data_profiling_config.dashboard_id +3559 resources.data_quality_monitors.*.data_profiling_config.drift_metrics_table_name +3560 resources.data_quality_monitors.*.data_profiling_config.effective_warehouse_id +3561 resources.data_quality_monitors.*.data_profiling_config.inference_log +3562 resources.data_quality_monitors.*.data_profiling_config.inference_log.granularities +3563 resources.data_quality_monitors.*.data_profiling_config.inference_log.granularities[*] +3564 resources.data_quality_monitors.*.data_profiling_config.inference_log.label_column +3565 resources.data_quality_monitors.*.data_profiling_config.inference_log.model_id_column +3566 resources.data_quality_monitors.*.data_profiling_config.inference_log.prediction_column +3567 resources.data_quality_monitors.*.data_profiling_config.inference_log.problem_type +3568 resources.data_quality_monitors.*.data_profiling_config.inference_log.timestamp_column +3569 resources.data_quality_monitors.*.data_profiling_config.latest_monitor_failure_message +3570 resources.data_quality_monitors.*.data_profiling_config.monitor_version +3571 resources.data_quality_monitors.*.data_profiling_config.monitored_table_name +3572 resources.data_quality_monitors.*.data_profiling_config.notification_settings +3573 resources.data_quality_monitors.*.data_profiling_config.notification_settings.on_failure +3574 resources.data_quality_monitors.*.data_profiling_config.notification_settings.on_failure.destination_ids +3575 resources.data_quality_monitors.*.data_profiling_config.notification_settings.on_failure.destination_ids[*] +3576 resources.data_quality_monitors.*.data_profiling_config.notification_settings.on_failure.destination_user_ids +3577 resources.data_quality_monitors.*.data_profiling_config.notification_settings.on_failure.destination_user_ids[*] +3578 resources.data_quality_monitors.*.data_profiling_config.notification_settings.on_failure.email_addresses +3579 resources.data_quality_monitors.*.data_profiling_config.notification_settings.on_failure.email_addresses[*] +3580 resources.data_quality_monitors.*.data_profiling_config.notification_settings.on_unhealthy_table +3581 resources.data_quality_monitors.*.data_profiling_config.notification_settings.on_unhealthy_table.destination_ids +3582 resources.data_quality_monitors.*.data_profiling_config.notification_settings.on_unhealthy_table.destination_ids[*] +3583 resources.data_quality_monitors.*.data_profiling_config.notification_settings.on_unhealthy_table.destination_user_ids +3584 resources.data_quality_monitors.*.data_profiling_config.notification_settings.on_unhealthy_table.destination_user_ids[*] +3585 resources.data_quality_monitors.*.data_profiling_config.notification_settings.on_unhealthy_table.email_addresses +3586 resources.data_quality_monitors.*.data_profiling_config.notification_settings.on_unhealthy_table.email_addresses[*] +3587 resources.data_quality_monitors.*.data_profiling_config.output_schema_id +3588 resources.data_quality_monitors.*.data_profiling_config.profile_metrics_table_name +3589 resources.data_quality_monitors.*.data_profiling_config.schedule +3590 resources.data_quality_monitors.*.data_profiling_config.schedule.pause_status +3591 resources.data_quality_monitors.*.data_profiling_config.schedule.quartz_cron_expression +3592 resources.data_quality_monitors.*.data_profiling_config.schedule.timezone_id +3593 resources.data_quality_monitors.*.data_profiling_config.skip_builtin_dashboard +3594 resources.data_quality_monitors.*.data_profiling_config.skip_builtin_dashboard=true +3595 resources.data_quality_monitors.*.data_profiling_config.skip_builtin_dashboard=false +3596 resources.data_quality_monitors.*.data_profiling_config.slicing_exprs +3597 resources.data_quality_monitors.*.data_profiling_config.slicing_exprs[*] +3598 resources.data_quality_monitors.*.data_profiling_config.snapshot +3599 resources.data_quality_monitors.*.data_profiling_config.status +3600 resources.data_quality_monitors.*.data_profiling_config.time_series +3601 resources.data_quality_monitors.*.data_profiling_config.time_series.granularities +3602 resources.data_quality_monitors.*.data_profiling_config.time_series.granularities[*] +3603 resources.data_quality_monitors.*.data_profiling_config.time_series.timestamp_column +3604 resources.data_quality_monitors.*.data_profiling_config.warehouse_id +3605 resources.data_quality_monitors.*.object_id +3606 resources.data_quality_monitors.*.object_type diff --git a/acceptance/bundle/bitmap/full/output/database_catalogs.txt b/acceptance/bundle/bitmap/full/output/database_catalogs.txt new file mode 100644 index 00000000000..52fb48563fd --- /dev/null +++ b/acceptance/bundle/bitmap/full/output/database_catalogs.txt @@ -0,0 +1,19 @@ +4489 resources.database_catalogs +4490 resources.database_catalogs.* +4491 resources.database_catalogs.*.url +4492 resources.database_catalogs.*.modified_status +4493 resources.database_catalogs.*.lifecycle +4494 resources.database_catalogs.*.lifecycle.prevent_destroy +4495 resources.database_catalogs.*.lifecycle.prevent_destroy=true +4496 resources.database_catalogs.*.lifecycle.prevent_destroy=false +4497 resources.database_catalogs.*.lifecycle.depends_on +4498 resources.database_catalogs.*.lifecycle.depends_on[*] +4499 resources.database_catalogs.*.lifecycle.depends_on[*].resource +4500 resources.database_catalogs.*.id +4501 resources.database_catalogs.*.create_database_if_not_exists +4502 resources.database_catalogs.*.create_database_if_not_exists=true +4503 resources.database_catalogs.*.create_database_if_not_exists=false +4504 resources.database_catalogs.*.database_instance_name +4505 resources.database_catalogs.*.database_name +4506 resources.database_catalogs.*.name +4507 resources.database_catalogs.*.uid diff --git a/acceptance/bundle/bitmap/full/output/database_instances.txt b/acceptance/bundle/bitmap/full/output/database_instances.txt new file mode 100644 index 00000000000..586acc61d2e --- /dev/null +++ b/acceptance/bundle/bitmap/full/output/database_instances.txt @@ -0,0 +1,73 @@ +4416 resources.database_instances +4417 resources.database_instances.* +4418 resources.database_instances.*.url +4419 resources.database_instances.*.modified_status +4420 resources.database_instances.*.lifecycle +4421 resources.database_instances.*.lifecycle.prevent_destroy +4422 resources.database_instances.*.lifecycle.prevent_destroy=true +4423 resources.database_instances.*.lifecycle.prevent_destroy=false +4424 resources.database_instances.*.lifecycle.depends_on +4425 resources.database_instances.*.lifecycle.depends_on[*] +4426 resources.database_instances.*.lifecycle.depends_on[*].resource +4427 resources.database_instances.*.id +4428 resources.database_instances.*.capacity +4429 resources.database_instances.*.child_instance_refs +4430 resources.database_instances.*.child_instance_refs[*] +4431 resources.database_instances.*.child_instance_refs[*].branch_time +4432 resources.database_instances.*.child_instance_refs[*].effective_lsn +4433 resources.database_instances.*.child_instance_refs[*].lsn +4434 resources.database_instances.*.child_instance_refs[*].name +4435 resources.database_instances.*.child_instance_refs[*].uid +4436 resources.database_instances.*.creation_time +4437 resources.database_instances.*.creator +4438 resources.database_instances.*.custom_tags +4439 resources.database_instances.*.custom_tags[*] +4440 resources.database_instances.*.custom_tags[*].key +4441 resources.database_instances.*.custom_tags[*].value +4442 resources.database_instances.*.effective_capacity +4443 resources.database_instances.*.effective_custom_tags +4444 resources.database_instances.*.effective_custom_tags[*] +4445 resources.database_instances.*.effective_custom_tags[*].key +4446 resources.database_instances.*.effective_custom_tags[*].value +4447 resources.database_instances.*.effective_enable_pg_native_login +4448 resources.database_instances.*.effective_enable_pg_native_login=true +4449 resources.database_instances.*.effective_enable_pg_native_login=false +4450 resources.database_instances.*.effective_enable_readable_secondaries +4451 resources.database_instances.*.effective_enable_readable_secondaries=true +4452 resources.database_instances.*.effective_enable_readable_secondaries=false +4453 resources.database_instances.*.effective_node_count +4454 resources.database_instances.*.effective_retention_window_in_days +4455 resources.database_instances.*.effective_stopped +4456 resources.database_instances.*.effective_stopped=true +4457 resources.database_instances.*.effective_stopped=false +4458 resources.database_instances.*.effective_usage_policy_id +4459 resources.database_instances.*.enable_pg_native_login +4460 resources.database_instances.*.enable_pg_native_login=true +4461 resources.database_instances.*.enable_pg_native_login=false +4462 resources.database_instances.*.enable_readable_secondaries +4463 resources.database_instances.*.enable_readable_secondaries=true +4464 resources.database_instances.*.enable_readable_secondaries=false +4465 resources.database_instances.*.name +4466 resources.database_instances.*.node_count +4467 resources.database_instances.*.parent_instance_ref +4468 resources.database_instances.*.parent_instance_ref.branch_time +4469 resources.database_instances.*.parent_instance_ref.effective_lsn +4470 resources.database_instances.*.parent_instance_ref.lsn +4471 resources.database_instances.*.parent_instance_ref.name +4472 resources.database_instances.*.parent_instance_ref.uid +4473 resources.database_instances.*.pg_version +4474 resources.database_instances.*.read_only_dns +4475 resources.database_instances.*.read_write_dns +4476 resources.database_instances.*.retention_window_in_days +4477 resources.database_instances.*.state +4478 resources.database_instances.*.stopped +4479 resources.database_instances.*.stopped=true +4480 resources.database_instances.*.stopped=false +4481 resources.database_instances.*.uid +4482 resources.database_instances.*.usage_policy_id +4483 resources.database_instances.*.permissions +4484 resources.database_instances.*.permissions[*] +4485 resources.database_instances.*.permissions[*].level +4486 resources.database_instances.*.permissions[*].user_name +4487 resources.database_instances.*.permissions[*].service_principal_name +4488 resources.database_instances.*.permissions[*].group_name diff --git a/acceptance/bundle/bitmap/full/output/experiments.txt b/acceptance/bundle/bitmap/full/output/experiments.txt new file mode 100644 index 00000000000..a4b6951de37 --- /dev/null +++ b/acceptance/bundle/bitmap/full/output/experiments.txt @@ -0,0 +1,30 @@ +2919 resources.experiments +2920 resources.experiments.* +2921 resources.experiments.*.url +2922 resources.experiments.*.modified_status +2923 resources.experiments.*.lifecycle +2924 resources.experiments.*.lifecycle.prevent_destroy +2925 resources.experiments.*.lifecycle.prevent_destroy=true +2926 resources.experiments.*.lifecycle.prevent_destroy=false +2927 resources.experiments.*.lifecycle.depends_on +2928 resources.experiments.*.lifecycle.depends_on[*] +2929 resources.experiments.*.lifecycle.depends_on[*].resource +2930 resources.experiments.*.id +2931 resources.experiments.*.artifact_location +2932 resources.experiments.*.name +2933 resources.experiments.*.tags +2934 resources.experiments.*.tags[*] +2935 resources.experiments.*.tags[*].key +2936 resources.experiments.*.tags[*].value +2937 resources.experiments.*.trace_location +2938 resources.experiments.*.trace_location.uc_trace_location +2939 resources.experiments.*.trace_location.uc_trace_location.catalog +2940 resources.experiments.*.trace_location.uc_trace_location.effective_table_prefix +2941 resources.experiments.*.trace_location.uc_trace_location.schema +2942 resources.experiments.*.trace_location.uc_trace_location.table_prefix +2943 resources.experiments.*.permissions +2944 resources.experiments.*.permissions[*] +2945 resources.experiments.*.permissions[*].level +2946 resources.experiments.*.permissions[*].user_name +2947 resources.experiments.*.permissions[*].service_principal_name +2948 resources.experiments.*.permissions[*].group_name diff --git a/acceptance/bundle/bitmap/full/output/external_locations.txt b/acceptance/bundle/bitmap/full/output/external_locations.txt new file mode 100644 index 00000000000..016975b1720 --- /dev/null +++ b/acceptance/bundle/bitmap/full/output/external_locations.txt @@ -0,0 +1,85 @@ +3691 resources.external_locations +3692 resources.external_locations.* +3693 resources.external_locations.*.id +3694 resources.external_locations.*.modified_status +3695 resources.external_locations.*.lifecycle +3696 resources.external_locations.*.lifecycle.prevent_destroy +3697 resources.external_locations.*.lifecycle.prevent_destroy=true +3698 resources.external_locations.*.lifecycle.prevent_destroy=false +3699 resources.external_locations.*.lifecycle.depends_on +3700 resources.external_locations.*.lifecycle.depends_on[*] +3701 resources.external_locations.*.lifecycle.depends_on[*].resource +3702 resources.external_locations.*.comment +3703 resources.external_locations.*.credential_name +3704 resources.external_locations.*.effective_enable_file_events +3705 resources.external_locations.*.effective_enable_file_events=true +3706 resources.external_locations.*.effective_enable_file_events=false +3707 resources.external_locations.*.effective_file_event_queue +3708 resources.external_locations.*.effective_file_event_queue.managed_aqs +3709 resources.external_locations.*.effective_file_event_queue.managed_aqs.managed_resource_id +3710 resources.external_locations.*.effective_file_event_queue.managed_aqs.queue_url +3711 resources.external_locations.*.effective_file_event_queue.managed_aqs.resource_group +3712 resources.external_locations.*.effective_file_event_queue.managed_aqs.subscription_id +3713 resources.external_locations.*.effective_file_event_queue.managed_pubsub +3714 resources.external_locations.*.effective_file_event_queue.managed_pubsub.managed_resource_id +3715 resources.external_locations.*.effective_file_event_queue.managed_pubsub.subscription_name +3716 resources.external_locations.*.effective_file_event_queue.managed_sqs +3717 resources.external_locations.*.effective_file_event_queue.managed_sqs.managed_resource_id +3718 resources.external_locations.*.effective_file_event_queue.managed_sqs.queue_url +3719 resources.external_locations.*.effective_file_event_queue.provided_aqs +3720 resources.external_locations.*.effective_file_event_queue.provided_aqs.managed_resource_id +3721 resources.external_locations.*.effective_file_event_queue.provided_aqs.queue_url +3722 resources.external_locations.*.effective_file_event_queue.provided_aqs.resource_group +3723 resources.external_locations.*.effective_file_event_queue.provided_aqs.subscription_id +3724 resources.external_locations.*.effective_file_event_queue.provided_pubsub +3725 resources.external_locations.*.effective_file_event_queue.provided_pubsub.managed_resource_id +3726 resources.external_locations.*.effective_file_event_queue.provided_pubsub.subscription_name +3727 resources.external_locations.*.effective_file_event_queue.provided_sqs +3728 resources.external_locations.*.effective_file_event_queue.provided_sqs.managed_resource_id +3729 resources.external_locations.*.effective_file_event_queue.provided_sqs.queue_url +3730 resources.external_locations.*.enable_file_events +3731 resources.external_locations.*.enable_file_events=true +3732 resources.external_locations.*.enable_file_events=false +3733 resources.external_locations.*.encryption_details +3734 resources.external_locations.*.encryption_details.sse_encryption_details +3735 resources.external_locations.*.encryption_details.sse_encryption_details.algorithm +3736 resources.external_locations.*.encryption_details.sse_encryption_details.aws_kms_key_arn +3737 resources.external_locations.*.fallback +3738 resources.external_locations.*.fallback=true +3739 resources.external_locations.*.fallback=false +3740 resources.external_locations.*.file_event_queue +3741 resources.external_locations.*.file_event_queue.managed_aqs +3742 resources.external_locations.*.file_event_queue.managed_aqs.managed_resource_id +3743 resources.external_locations.*.file_event_queue.managed_aqs.queue_url +3744 resources.external_locations.*.file_event_queue.managed_aqs.resource_group +3745 resources.external_locations.*.file_event_queue.managed_aqs.subscription_id +3746 resources.external_locations.*.file_event_queue.managed_pubsub +3747 resources.external_locations.*.file_event_queue.managed_pubsub.managed_resource_id +3748 resources.external_locations.*.file_event_queue.managed_pubsub.subscription_name +3749 resources.external_locations.*.file_event_queue.managed_sqs +3750 resources.external_locations.*.file_event_queue.managed_sqs.managed_resource_id +3751 resources.external_locations.*.file_event_queue.managed_sqs.queue_url +3752 resources.external_locations.*.file_event_queue.provided_aqs +3753 resources.external_locations.*.file_event_queue.provided_aqs.managed_resource_id +3754 resources.external_locations.*.file_event_queue.provided_aqs.queue_url +3755 resources.external_locations.*.file_event_queue.provided_aqs.resource_group +3756 resources.external_locations.*.file_event_queue.provided_aqs.subscription_id +3757 resources.external_locations.*.file_event_queue.provided_pubsub +3758 resources.external_locations.*.file_event_queue.provided_pubsub.managed_resource_id +3759 resources.external_locations.*.file_event_queue.provided_pubsub.subscription_name +3760 resources.external_locations.*.file_event_queue.provided_sqs +3761 resources.external_locations.*.file_event_queue.provided_sqs.managed_resource_id +3762 resources.external_locations.*.file_event_queue.provided_sqs.queue_url +3763 resources.external_locations.*.name +3764 resources.external_locations.*.read_only +3765 resources.external_locations.*.read_only=true +3766 resources.external_locations.*.read_only=false +3767 resources.external_locations.*.skip_validation +3768 resources.external_locations.*.skip_validation=true +3769 resources.external_locations.*.skip_validation=false +3770 resources.external_locations.*.url +3771 resources.external_locations.*.grants +3772 resources.external_locations.*.grants[*] +3773 resources.external_locations.*.grants[*].principal +3774 resources.external_locations.*.grants[*].privileges +3775 resources.external_locations.*.grants[*].privileges[*] diff --git a/acceptance/bundle/bitmap/full/output/feature_specs.txt b/acceptance/bundle/bitmap/full/output/feature_specs.txt new file mode 100644 index 00000000000..194ee562bbd --- /dev/null +++ b/acceptance/bundle/bitmap/full/output/feature_specs.txt @@ -0,0 +1,51 @@ +2844 resources.feature_specs +2845 resources.feature_specs.* +2846 resources.feature_specs.*.url +2847 resources.feature_specs.*.modified_status +2848 resources.feature_specs.*.lifecycle +2849 resources.feature_specs.*.lifecycle.prevent_destroy +2850 resources.feature_specs.*.lifecycle.prevent_destroy=true +2851 resources.feature_specs.*.lifecycle.prevent_destroy=false +2852 resources.feature_specs.*.lifecycle.depends_on +2853 resources.feature_specs.*.lifecycle.depends_on[*] +2854 resources.feature_specs.*.lifecycle.depends_on[*].resource +2855 resources.feature_specs.*.id +2856 resources.feature_specs.*.exclude_columns +2857 resources.feature_specs.*.exclude_columns[*] +2858 resources.feature_specs.*.extra_pip_requirements +2859 resources.feature_specs.*.extra_pip_requirements.requirements +2860 resources.feature_specs.*.extra_pip_requirements.requirements[*] +2861 resources.feature_specs.*.features +2862 resources.feature_specs.*.features[*] +2863 resources.feature_specs.*.features[*].declarative_feature +2864 resources.feature_specs.*.features[*].declarative_feature.feature +2865 resources.feature_specs.*.features[*].feature_function +2866 resources.feature_specs.*.features[*].feature_function.input_bindings +2867 resources.feature_specs.*.features[*].feature_function.input_bindings[*] +2868 resources.feature_specs.*.features[*].feature_function.input_bindings[*].bound_feature +2869 resources.feature_specs.*.features[*].feature_function.input_bindings[*].parameter +2870 resources.feature_specs.*.features[*].feature_function.output +2871 resources.feature_specs.*.features[*].feature_function.udf +2872 resources.feature_specs.*.features[*].feature_lookup +2873 resources.feature_specs.*.features[*].feature_lookup.default_values +2874 resources.feature_specs.*.features[*].feature_lookup.default_values[*] +2875 resources.feature_specs.*.features[*].feature_lookup.default_values[*].default_value +2876 resources.feature_specs.*.features[*].feature_lookup.default_values[*].feature +2877 resources.feature_specs.*.features[*].feature_lookup.feature_names +2878 resources.feature_specs.*.features[*].feature_lookup.feature_names[*] +2879 resources.feature_specs.*.features[*].feature_lookup.lookback_window +2880 resources.feature_specs.*.features[*].feature_lookup.lookup_keys +2881 resources.feature_specs.*.features[*].feature_lookup.lookup_keys[*] +2882 resources.feature_specs.*.features[*].feature_lookup.rename_outputs +2883 resources.feature_specs.*.features[*].feature_lookup.rename_outputs[*] +2884 resources.feature_specs.*.features[*].feature_lookup.rename_outputs[*].feature +2885 resources.feature_specs.*.features[*].feature_lookup.rename_outputs[*].output +2886 resources.feature_specs.*.features[*].feature_lookup.table +2887 resources.feature_specs.*.features[*].feature_lookup.timestamp_lookup_keys +2888 resources.feature_specs.*.features[*].feature_lookup.timestamp_lookup_keys[*] +2889 resources.feature_specs.*.name +2890 resources.feature_specs.*.grants +2891 resources.feature_specs.*.grants[*] +2892 resources.feature_specs.*.grants[*].principal +2893 resources.feature_specs.*.grants[*].privileges +2894 resources.feature_specs.*.grants[*].privileges[*] diff --git a/acceptance/bundle/bitmap/full/output/features.txt b/acceptance/bundle/bitmap/full/output/features.txt new file mode 100644 index 00000000000..9ab7199d13a --- /dev/null +++ b/acceptance/bundle/bitmap/full/output/features.txt @@ -0,0 +1,180 @@ +2577 resources.features +2578 resources.features.* +2579 resources.features.*.url +2580 resources.features.*.modified_status +2581 resources.features.*.lifecycle +2582 resources.features.*.lifecycle.prevent_destroy +2583 resources.features.*.lifecycle.prevent_destroy=true +2584 resources.features.*.lifecycle.prevent_destroy=false +2585 resources.features.*.lifecycle.depends_on +2586 resources.features.*.lifecycle.depends_on[*] +2587 resources.features.*.lifecycle.depends_on[*].resource +2588 resources.features.*.id +2589 resources.features.*.catalog_name +2590 resources.features.*.created_at +2591 resources.features.*.created_by +2592 resources.features.*.description +2593 resources.features.*.entities +2594 resources.features.*.entities[*] +2595 resources.features.*.entities[*].name +2596 resources.features.*.filter_condition +2597 resources.features.*.full_name +2598 resources.features.*.function +2599 resources.features.*.function.aggregation_function +2600 resources.features.*.function.aggregation_function.approx_count_distinct +2601 resources.features.*.function.aggregation_function.approx_count_distinct.input +2602 resources.features.*.function.aggregation_function.approx_count_distinct.relative_error +2603 resources.features.*.function.aggregation_function.approx_count_distinct.relative_sd +2604 resources.features.*.function.aggregation_function.approx_percentile +2605 resources.features.*.function.aggregation_function.approx_percentile.accuracy +2606 resources.features.*.function.aggregation_function.approx_percentile.input +2607 resources.features.*.function.aggregation_function.approx_percentile.percentile +2608 resources.features.*.function.aggregation_function.approx_percentile.relative_error +2609 resources.features.*.function.aggregation_function.avg +2610 resources.features.*.function.aggregation_function.avg.input +2611 resources.features.*.function.aggregation_function.count_function +2612 resources.features.*.function.aggregation_function.count_function.input +2613 resources.features.*.function.aggregation_function.first +2614 resources.features.*.function.aggregation_function.first.input +2615 resources.features.*.function.aggregation_function.first_distinct +2616 resources.features.*.function.aggregation_function.first_distinct.input +2617 resources.features.*.function.aggregation_function.first_distinct.n +2618 resources.features.*.function.aggregation_function.first_n +2619 resources.features.*.function.aggregation_function.first_n.input +2620 resources.features.*.function.aggregation_function.first_n.n +2621 resources.features.*.function.aggregation_function.last +2622 resources.features.*.function.aggregation_function.last.input +2623 resources.features.*.function.aggregation_function.last_distinct +2624 resources.features.*.function.aggregation_function.last_distinct.input +2625 resources.features.*.function.aggregation_function.last_distinct.n +2626 resources.features.*.function.aggregation_function.last_n +2627 resources.features.*.function.aggregation_function.last_n.input +2628 resources.features.*.function.aggregation_function.last_n.n +2629 resources.features.*.function.aggregation_function.max +2630 resources.features.*.function.aggregation_function.max.input +2631 resources.features.*.function.aggregation_function.min +2632 resources.features.*.function.aggregation_function.min.input +2633 resources.features.*.function.aggregation_function.stddev_pop +2634 resources.features.*.function.aggregation_function.stddev_pop.input +2635 resources.features.*.function.aggregation_function.stddev_samp +2636 resources.features.*.function.aggregation_function.stddev_samp.input +2637 resources.features.*.function.aggregation_function.sum +2638 resources.features.*.function.aggregation_function.sum.input +2639 resources.features.*.function.aggregation_function.time_window +2640 resources.features.*.function.aggregation_function.time_window.continuous +2641 resources.features.*.function.aggregation_function.time_window.continuous.offset +2642 resources.features.*.function.aggregation_function.time_window.continuous.window_duration +2643 resources.features.*.function.aggregation_function.time_window.rolling +2644 resources.features.*.function.aggregation_function.time_window.rolling.delay +2645 resources.features.*.function.aggregation_function.time_window.rolling.window_duration +2646 resources.features.*.function.aggregation_function.time_window.sawtooth +2647 resources.features.*.function.aggregation_function.time_window.sawtooth.delay +2648 resources.features.*.function.aggregation_function.time_window.sawtooth.window_duration +2649 resources.features.*.function.aggregation_function.time_window.sliding +2650 resources.features.*.function.aggregation_function.time_window.sliding.delay +2651 resources.features.*.function.aggregation_function.time_window.sliding.offset +2652 resources.features.*.function.aggregation_function.time_window.sliding.slide_duration +2653 resources.features.*.function.aggregation_function.time_window.sliding.window_duration +2654 resources.features.*.function.aggregation_function.time_window.start_time +2655 resources.features.*.function.aggregation_function.time_window.tumbling +2656 resources.features.*.function.aggregation_function.time_window.tumbling.delay +2657 resources.features.*.function.aggregation_function.time_window.tumbling.offset +2658 resources.features.*.function.aggregation_function.time_window.tumbling.window_duration +2659 resources.features.*.function.aggregation_function.var_pop +2660 resources.features.*.function.aggregation_function.var_pop.input +2661 resources.features.*.function.aggregation_function.var_samp +2662 resources.features.*.function.aggregation_function.var_samp.input +2663 resources.features.*.function.column_selection +2664 resources.features.*.function.column_selection.column +2665 resources.features.*.function.custom_udf +2666 resources.features.*.function.custom_udf.function_path +2667 resources.features.*.function.custom_udf.input_bindings +2668 resources.features.*.function.custom_udf.input_bindings[*] +2669 resources.features.*.function.custom_udf.input_bindings[*].column +2670 resources.features.*.function.custom_udf.input_bindings[*].parameter +2671 resources.features.*.function.extra_parameters +2672 resources.features.*.function.extra_parameters[*] +2673 resources.features.*.function.extra_parameters[*].key +2674 resources.features.*.function.extra_parameters[*].value +2675 resources.features.*.function.function_type +2676 resources.features.*.inputs +2677 resources.features.*.inputs[*] +2678 resources.features.*.lineage_context +2679 resources.features.*.lineage_context.job_context +2680 resources.features.*.lineage_context.job_context.job_id +2681 resources.features.*.lineage_context.job_context.job_run_id +2682 resources.features.*.lineage_context.notebook_id +2683 resources.features.*.name +2684 resources.features.*.output_type_text +2685 resources.features.*.schema_name +2686 resources.features.*.source +2687 resources.features.*.source.delta_table_source +2688 resources.features.*.source.delta_table_source.dataframe_schema +2689 resources.features.*.source.delta_table_source.entity_columns +2690 resources.features.*.source.delta_table_source.entity_columns[*] +2691 resources.features.*.source.delta_table_source.filter_condition +2692 resources.features.*.source.delta_table_source.full_name +2693 resources.features.*.source.delta_table_source.timeseries_column +2694 resources.features.*.source.delta_table_source.transformation_sql +2695 resources.features.*.source.feature_view_source +2696 resources.features.*.source.feature_view_source.feature_references +2697 resources.features.*.source.feature_view_source.feature_references[*] +2698 resources.features.*.source.feature_view_source.feature_references[*].feature +2699 resources.features.*.source.iceberg_table_source +2700 resources.features.*.source.iceberg_table_source.dataframe_schema +2701 resources.features.*.source.iceberg_table_source.filter_condition +2702 resources.features.*.source.iceberg_table_source.full_name +2703 resources.features.*.source.iceberg_table_source.transformation_sql +2704 resources.features.*.source.kafka_source +2705 resources.features.*.source.kafka_source.entity_column_identifiers +2706 resources.features.*.source.kafka_source.entity_column_identifiers[*] +2707 resources.features.*.source.kafka_source.entity_column_identifiers[*].variant_expr_path +2708 resources.features.*.source.kafka_source.filter_condition +2709 resources.features.*.source.kafka_source.name +2710 resources.features.*.source.kafka_source.timeseries_column_identifier +2711 resources.features.*.source.kafka_source.timeseries_column_identifier.variant_expr_path +2712 resources.features.*.source.lateness +2713 resources.features.*.source.lateness.settling_delay +2714 resources.features.*.source.request_source +2715 resources.features.*.source.request_source.dataframe_schema +2716 resources.features.*.source.request_source.flat_schema +2717 resources.features.*.source.request_source.flat_schema.fields +2718 resources.features.*.source.request_source.flat_schema.fields[*] +2719 resources.features.*.source.request_source.flat_schema.fields[*].data_type +2720 resources.features.*.source.request_source.flat_schema.fields[*].name +2721 resources.features.*.source.sdp_source +2722 resources.features.*.source.sdp_source.dataframe_schema +2723 resources.features.*.source.sdp_source.filter_condition +2724 resources.features.*.source.sdp_source.input_tables +2725 resources.features.*.source.sdp_source.input_tables[*] +2726 resources.features.*.source.sdp_source.output_table +2727 resources.features.*.source.sdp_source.pipeline_id +2728 resources.features.*.source.sdp_source.pipeline_state +2729 resources.features.*.source.sdp_source.transformation_sql +2730 resources.features.*.source.stream_source +2731 resources.features.*.source.stream_source.dataframe_schema +2732 resources.features.*.source.stream_source.filter_condition +2733 resources.features.*.source.stream_source.full_name +2734 resources.features.*.source.stream_source.transformation_sql +2735 resources.features.*.time_window +2736 resources.features.*.time_window.continuous +2737 resources.features.*.time_window.continuous.offset +2738 resources.features.*.time_window.continuous.window_duration +2739 resources.features.*.time_window.rolling +2740 resources.features.*.time_window.rolling.delay +2741 resources.features.*.time_window.rolling.window_duration +2742 resources.features.*.time_window.sawtooth +2743 resources.features.*.time_window.sawtooth.delay +2744 resources.features.*.time_window.sawtooth.window_duration +2745 resources.features.*.time_window.sliding +2746 resources.features.*.time_window.sliding.delay +2747 resources.features.*.time_window.sliding.offset +2748 resources.features.*.time_window.sliding.slide_duration +2749 resources.features.*.time_window.sliding.window_duration +2750 resources.features.*.time_window.start_time +2751 resources.features.*.time_window.tumbling +2752 resources.features.*.time_window.tumbling.delay +2753 resources.features.*.time_window.tumbling.offset +2754 resources.features.*.time_window.tumbling.window_duration +2755 resources.features.*.timeseries_column +2756 resources.features.*.timeseries_column.name diff --git a/acceptance/bundle/bitmap/full/output/genie_spaces.txt b/acceptance/bundle/bitmap/full/output/genie_spaces.txt new file mode 100644 index 00000000000..f70e5726fac --- /dev/null +++ b/acceptance/bundle/bitmap/full/output/genie_spaces.txt @@ -0,0 +1,25 @@ +3980 resources.genie_spaces +3981 resources.genie_spaces.* +3982 resources.genie_spaces.*.url +3983 resources.genie_spaces.*.modified_status +3984 resources.genie_spaces.*.lifecycle +3985 resources.genie_spaces.*.lifecycle.prevent_destroy +3986 resources.genie_spaces.*.lifecycle.prevent_destroy=true +3987 resources.genie_spaces.*.lifecycle.prevent_destroy=false +3988 resources.genie_spaces.*.lifecycle.depends_on +3989 resources.genie_spaces.*.lifecycle.depends_on[*] +3990 resources.genie_spaces.*.lifecycle.depends_on[*].resource +3991 resources.genie_spaces.*.id +3992 resources.genie_spaces.*.description +3993 resources.genie_spaces.*.etag +3994 resources.genie_spaces.*.title +3995 resources.genie_spaces.*.warehouse_id +3996 resources.genie_spaces.*.parent_path +3997 resources.genie_spaces.*.serialized_space +3998 resources.genie_spaces.*.permissions +3999 resources.genie_spaces.*.permissions[*] +4000 resources.genie_spaces.*.permissions[*].level +4001 resources.genie_spaces.*.permissions[*].user_name +4002 resources.genie_spaces.*.permissions[*].service_principal_name +4003 resources.genie_spaces.*.permissions[*].group_name +4004 resources.genie_spaces.*.file_path diff --git a/acceptance/bundle/bitmap/full/output/instance_pools.txt b/acceptance/bundle/bitmap/full/output/instance_pools.txt new file mode 100644 index 00000000000..76def69af3b --- /dev/null +++ b/acceptance/bundle/bitmap/full/output/instance_pools.txt @@ -0,0 +1,63 @@ +4938 resources.instance_pools +4939 resources.instance_pools.* +4940 resources.instance_pools.*.url +4941 resources.instance_pools.*.modified_status +4942 resources.instance_pools.*.lifecycle +4943 resources.instance_pools.*.lifecycle.prevent_destroy +4944 resources.instance_pools.*.lifecycle.prevent_destroy=true +4945 resources.instance_pools.*.lifecycle.prevent_destroy=false +4946 resources.instance_pools.*.lifecycle.depends_on +4947 resources.instance_pools.*.lifecycle.depends_on[*] +4948 resources.instance_pools.*.lifecycle.depends_on[*].resource +4949 resources.instance_pools.*.id +4950 resources.instance_pools.*.aws_attributes +4951 resources.instance_pools.*.aws_attributes.availability +4952 resources.instance_pools.*.aws_attributes.instance_profile_arn +4953 resources.instance_pools.*.aws_attributes.spot_bid_price_percent +4954 resources.instance_pools.*.aws_attributes.zone_id +4955 resources.instance_pools.*.azure_attributes +4956 resources.instance_pools.*.azure_attributes.availability +4957 resources.instance_pools.*.azure_attributes.capacity_reservation_group +4958 resources.instance_pools.*.azure_attributes.spot_bid_max_price +4959 resources.instance_pools.*.custom_tags +4960 resources.instance_pools.*.custom_tags.* +4961 resources.instance_pools.*.disk_spec +4962 resources.instance_pools.*.disk_spec.disk_count +4963 resources.instance_pools.*.disk_spec.disk_iops +4964 resources.instance_pools.*.disk_spec.disk_size +4965 resources.instance_pools.*.disk_spec.disk_throughput +4966 resources.instance_pools.*.disk_spec.disk_type +4967 resources.instance_pools.*.disk_spec.disk_type.azure_disk_volume_type +4968 resources.instance_pools.*.disk_spec.disk_type.ebs_volume_type +4969 resources.instance_pools.*.enable_elastic_disk +4970 resources.instance_pools.*.enable_elastic_disk=true +4971 resources.instance_pools.*.enable_elastic_disk=false +4972 resources.instance_pools.*.gcp_attributes +4973 resources.instance_pools.*.gcp_attributes.gcp_availability +4974 resources.instance_pools.*.gcp_attributes.local_ssd_count +4975 resources.instance_pools.*.gcp_attributes.zone_id +4976 resources.instance_pools.*.idle_instance_autotermination_minutes +4977 resources.instance_pools.*.instance_pool_name +4978 resources.instance_pools.*.max_capacity +4979 resources.instance_pools.*.min_idle_instances +4980 resources.instance_pools.*.node_type_flexibility +4981 resources.instance_pools.*.node_type_flexibility.alternate_node_type_ids +4982 resources.instance_pools.*.node_type_flexibility.alternate_node_type_ids[*] +4983 resources.instance_pools.*.node_type_flexibility.aws_context_id +4984 resources.instance_pools.*.node_type_id +4985 resources.instance_pools.*.preloaded_docker_images +4986 resources.instance_pools.*.preloaded_docker_images[*] +4987 resources.instance_pools.*.preloaded_docker_images[*].basic_auth +4988 resources.instance_pools.*.preloaded_docker_images[*].basic_auth.password +4989 resources.instance_pools.*.preloaded_docker_images[*].basic_auth.username +4990 resources.instance_pools.*.preloaded_docker_images[*].url +4991 resources.instance_pools.*.preloaded_spark_versions +4992 resources.instance_pools.*.preloaded_spark_versions[*] +4993 resources.instance_pools.*.remote_disk_throughput +4994 resources.instance_pools.*.total_initial_remote_disk_size +4995 resources.instance_pools.*.permissions +4996 resources.instance_pools.*.permissions[*] +4997 resources.instance_pools.*.permissions[*].level +4998 resources.instance_pools.*.permissions[*].user_name +4999 resources.instance_pools.*.permissions[*].service_principal_name +5000 resources.instance_pools.*.permissions[*].group_name diff --git a/acceptance/bundle/bitmap/full/output/internal_immutable_snapshots.txt b/acceptance/bundle/bitmap/full/output/internal_immutable_snapshots.txt new file mode 100644 index 00000000000..3ca96723ec1 --- /dev/null +++ b/acceptance/bundle/bitmap/full/output/internal_immutable_snapshots.txt @@ -0,0 +1,16 @@ +5074 resources.internal_immutable_snapshots +5075 resources.internal_immutable_snapshots.* +5076 resources.internal_immutable_snapshots.*.bundle_id +5077 resources.internal_immutable_snapshots.*.acl +5078 resources.internal_immutable_snapshots.*.acl[*] +5079 resources.internal_immutable_snapshots.*.acl[*].user_name +5080 resources.internal_immutable_snapshots.*.acl[*].group_name +5081 resources.internal_immutable_snapshots.*.acl[*].service_principal_name +5082 resources.internal_immutable_snapshots.*.acl[*].permission_level +5083 resources.internal_immutable_snapshots.*.can_manage +5084 resources.internal_immutable_snapshots.*.can_manage[*] +5085 resources.internal_immutable_snapshots.*.can_manage[*].user_name +5086 resources.internal_immutable_snapshots.*.can_manage[*].group_name +5087 resources.internal_immutable_snapshots.*.can_manage[*].service_principal_name +5088 resources.internal_immutable_snapshots.*.zip_path +5089 resources.internal_immutable_snapshots.*.remote_root diff --git a/acceptance/bundle/bitmap/full/output/job_runs.txt b/acceptance/bundle/bitmap/full/output/job_runs.txt new file mode 100644 index 00000000000..3e34510d182 --- /dev/null +++ b/acceptance/bundle/bitmap/full/output/job_runs.txt @@ -0,0 +1,61 @@ +1522 resources.job_runs +1523 resources.job_runs.* +1524 resources.job_runs.*.url +1525 resources.job_runs.*.modified_status +1526 resources.job_runs.*.lifecycle +1527 resources.job_runs.*.lifecycle.prevent_destroy +1528 resources.job_runs.*.lifecycle.prevent_destroy=true +1529 resources.job_runs.*.lifecycle.prevent_destroy=false +1530 resources.job_runs.*.lifecycle.depends_on +1531 resources.job_runs.*.lifecycle.depends_on[*] +1532 resources.job_runs.*.lifecycle.depends_on[*].resource +1533 resources.job_runs.*.id +1534 resources.job_runs.*.dbt_commands +1535 resources.job_runs.*.dbt_commands[*] +1536 resources.job_runs.*.duration_target +1537 resources.job_runs.*.idempotency_token +1538 resources.job_runs.*.jar_params +1539 resources.job_runs.*.jar_params[*] +1540 resources.job_runs.*.job_id +1541 resources.job_runs.*.job_parameters +1542 resources.job_runs.*.job_parameters.* +1543 resources.job_runs.*.notebook_params +1544 resources.job_runs.*.notebook_params.* +1545 resources.job_runs.*.only +1546 resources.job_runs.*.only[*] +1547 resources.job_runs.*.performance_target +1548 resources.job_runs.*.pipeline_params +1549 resources.job_runs.*.pipeline_params.full_refresh +1550 resources.job_runs.*.pipeline_params.full_refresh=true +1551 resources.job_runs.*.pipeline_params.full_refresh=false +1552 resources.job_runs.*.pipeline_params.full_refresh_selection +1553 resources.job_runs.*.pipeline_params.full_refresh_selection[*] +1554 resources.job_runs.*.pipeline_params.refresh_flow_selection +1555 resources.job_runs.*.pipeline_params.refresh_flow_selection[*] +1556 resources.job_runs.*.pipeline_params.refresh_selection +1557 resources.job_runs.*.pipeline_params.refresh_selection[*] +1558 resources.job_runs.*.pipeline_params.reset_checkpoint_selection +1559 resources.job_runs.*.pipeline_params.reset_checkpoint_selection[*] +1560 resources.job_runs.*.python_named_params +1561 resources.job_runs.*.python_named_params.* +1562 resources.job_runs.*.python_params +1563 resources.job_runs.*.python_params[*] +1564 resources.job_runs.*.queue +1565 resources.job_runs.*.queue.enabled +1566 resources.job_runs.*.queue.enabled=true +1567 resources.job_runs.*.queue.enabled=false +1568 resources.job_runs.*.spark_submit_params +1569 resources.job_runs.*.spark_submit_params[*] +1570 resources.job_runs.*.sql_params +1571 resources.job_runs.*.sql_params.* +1572 resources.job_runs.*.lifecycle.triggers +1573 resources.job_runs.*.lifecycle.triggers[*] +1574 resources.job_runs.*.lifecycle.triggers[*].on_bundle_deploy +1575 resources.job_runs.*.lifecycle.triggers[*].on_bundle_deploy=true +1576 resources.job_runs.*.lifecycle.triggers[*].on_bundle_deploy=false +1577 resources.job_runs.*.lifecycle.triggers[*].on_file_change +1578 resources.job_runs.*.lifecycle.triggers_state +1579 resources.job_runs.*.lifecycle.triggers_state.on_bundle_deploy +1580 resources.job_runs.*.lifecycle.triggers_state.on_file_change +1581 resources.job_runs.*.lifecycle.triggers_state.on_file_change.* +1582 resources.job_runs.*.resolved_job_id diff --git a/acceptance/bundle/bitmap/full/output/jobs.txt b/acceptance/bundle/bitmap/full/output/jobs.txt new file mode 100644 index 00000000000..d0496e325a6 --- /dev/null +++ b/acceptance/bundle/bitmap/full/output/jobs.txt @@ -0,0 +1,1369 @@ + 153 resources.jobs + 154 resources.jobs.* + 155 resources.jobs.*.url + 156 resources.jobs.*.modified_status + 157 resources.jobs.*.lifecycle + 158 resources.jobs.*.lifecycle.prevent_destroy + 159 resources.jobs.*.lifecycle.prevent_destroy=true + 160 resources.jobs.*.lifecycle.prevent_destroy=false + 161 resources.jobs.*.lifecycle.depends_on + 162 resources.jobs.*.lifecycle.depends_on[*] + 163 resources.jobs.*.lifecycle.depends_on[*].resource + 164 resources.jobs.*.id + 165 resources.jobs.*.budget_policy_id + 166 resources.jobs.*.continuous + 167 resources.jobs.*.continuous.maintenance_window + 168 resources.jobs.*.continuous.maintenance_window.day_of_week + 169 resources.jobs.*.continuous.maintenance_window.start_hour + 170 resources.jobs.*.continuous.maintenance_window.timezone_id + 171 resources.jobs.*.continuous.pause_status + 172 resources.jobs.*.continuous.task_retry_mode + 173 resources.jobs.*.deployment + 174 resources.jobs.*.deployment.deployment_id + 175 resources.jobs.*.deployment.kind + 176 resources.jobs.*.deployment.metadata_file_path + 177 resources.jobs.*.deployment.version_id + 178 resources.jobs.*.description + 179 resources.jobs.*.duration_target + 180 resources.jobs.*.edit_mode + 181 resources.jobs.*.email_notifications + 182 resources.jobs.*.email_notifications.no_alert_for_skipped_runs + 183 resources.jobs.*.email_notifications.no_alert_for_skipped_runs=true + 184 resources.jobs.*.email_notifications.no_alert_for_skipped_runs=false + 185 resources.jobs.*.email_notifications.on_duration_warning_threshold_exceeded + 186 resources.jobs.*.email_notifications.on_duration_warning_threshold_exceeded[*] + 187 resources.jobs.*.email_notifications.on_failure + 188 resources.jobs.*.email_notifications.on_failure[*] + 189 resources.jobs.*.email_notifications.on_maintenance_complete + 190 resources.jobs.*.email_notifications.on_maintenance_complete[*] + 191 resources.jobs.*.email_notifications.on_maintenance_start + 192 resources.jobs.*.email_notifications.on_maintenance_start[*] + 193 resources.jobs.*.email_notifications.on_start + 194 resources.jobs.*.email_notifications.on_start[*] + 195 resources.jobs.*.email_notifications.on_streaming_backlog_exceeded + 196 resources.jobs.*.email_notifications.on_streaming_backlog_exceeded[*] + 197 resources.jobs.*.email_notifications.on_success + 198 resources.jobs.*.email_notifications.on_success[*] + 199 resources.jobs.*.environment_variables + 200 resources.jobs.*.environment_variables[*] + 201 resources.jobs.*.environment_variables[*].environment_variables_key + 202 resources.jobs.*.environment_variables[*].spec + 203 resources.jobs.*.environment_variables[*].spec.files + 204 resources.jobs.*.environment_variables[*].spec.files[*] + 205 resources.jobs.*.environment_variables[*].spec.variables + 206 resources.jobs.*.environment_variables[*].spec.variables.* + 207 resources.jobs.*.environments + 208 resources.jobs.*.environments[*] + 209 resources.jobs.*.environments[*].environment_key + 210 resources.jobs.*.environments[*].spec + 211 resources.jobs.*.environments[*].spec.base_environment + 212 resources.jobs.*.environments[*].spec.client + 213 resources.jobs.*.environments[*].spec.dependencies + 214 resources.jobs.*.environments[*].spec.dependencies[*] + 215 resources.jobs.*.environments[*].spec.environment_version + 216 resources.jobs.*.environments[*].spec.image_uri + 217 resources.jobs.*.environments[*].spec.java_dependencies + 218 resources.jobs.*.environments[*].spec.java_dependencies[*] + 219 resources.jobs.*.environments[*].spec.project_environment + 220 resources.jobs.*.format + 221 resources.jobs.*.git_source + 222 resources.jobs.*.git_source.git_branch + 223 resources.jobs.*.git_source.git_commit + 224 resources.jobs.*.git_source.git_provider + 225 resources.jobs.*.git_source.git_snapshot + 226 resources.jobs.*.git_source.git_snapshot.used_commit + 227 resources.jobs.*.git_source.git_tag + 228 resources.jobs.*.git_source.git_url + 229 resources.jobs.*.git_source.job_source + 230 resources.jobs.*.git_source.job_source.dirty_state + 231 resources.jobs.*.git_source.job_source.import_from_git_branch + 232 resources.jobs.*.git_source.job_source.job_config_path + 233 resources.jobs.*.git_source.sparse_checkout + 234 resources.jobs.*.git_source.sparse_checkout.patterns + 235 resources.jobs.*.git_source.sparse_checkout.patterns[*] + 236 resources.jobs.*.health + 237 resources.jobs.*.health.rules + 238 resources.jobs.*.health.rules[*] + 239 resources.jobs.*.health.rules[*].metric + 240 resources.jobs.*.health.rules[*].op + 241 resources.jobs.*.health.rules[*].value + 242 resources.jobs.*.job_clusters + 243 resources.jobs.*.job_clusters[*] + 244 resources.jobs.*.job_clusters[*].job_cluster_key + 245 resources.jobs.*.job_clusters[*].new_cluster + 246 resources.jobs.*.job_clusters[*].new_cluster.apply_policy_default_values + 247 resources.jobs.*.job_clusters[*].new_cluster.apply_policy_default_values=true + 248 resources.jobs.*.job_clusters[*].new_cluster.apply_policy_default_values=false + 249 resources.jobs.*.job_clusters[*].new_cluster.autoscale + 250 resources.jobs.*.job_clusters[*].new_cluster.autoscale.max_workers + 251 resources.jobs.*.job_clusters[*].new_cluster.autoscale.min_workers + 252 resources.jobs.*.job_clusters[*].new_cluster.autotermination_minutes + 253 resources.jobs.*.job_clusters[*].new_cluster.aws_attributes + 254 resources.jobs.*.job_clusters[*].new_cluster.aws_attributes.availability + 255 resources.jobs.*.job_clusters[*].new_cluster.aws_attributes.ebs_volume_count + 256 resources.jobs.*.job_clusters[*].new_cluster.aws_attributes.ebs_volume_iops + 257 resources.jobs.*.job_clusters[*].new_cluster.aws_attributes.ebs_volume_size + 258 resources.jobs.*.job_clusters[*].new_cluster.aws_attributes.ebs_volume_throughput + 259 resources.jobs.*.job_clusters[*].new_cluster.aws_attributes.ebs_volume_type + 260 resources.jobs.*.job_clusters[*].new_cluster.aws_attributes.first_on_demand + 261 resources.jobs.*.job_clusters[*].new_cluster.aws_attributes.instance_profile_arn + 262 resources.jobs.*.job_clusters[*].new_cluster.aws_attributes.spot_bid_price_percent + 263 resources.jobs.*.job_clusters[*].new_cluster.aws_attributes.zone_id + 264 resources.jobs.*.job_clusters[*].new_cluster.azure_attributes + 265 resources.jobs.*.job_clusters[*].new_cluster.azure_attributes.availability + 266 resources.jobs.*.job_clusters[*].new_cluster.azure_attributes.capacity_reservation_group + 267 resources.jobs.*.job_clusters[*].new_cluster.azure_attributes.first_on_demand + 268 resources.jobs.*.job_clusters[*].new_cluster.azure_attributes.log_analytics_info + 269 resources.jobs.*.job_clusters[*].new_cluster.azure_attributes.log_analytics_info.log_analytics_primary_key + 270 resources.jobs.*.job_clusters[*].new_cluster.azure_attributes.log_analytics_info.log_analytics_workspace_id + 271 resources.jobs.*.job_clusters[*].new_cluster.azure_attributes.spot_bid_max_price + 272 resources.jobs.*.job_clusters[*].new_cluster.cluster_log_conf + 273 resources.jobs.*.job_clusters[*].new_cluster.cluster_log_conf.dbfs + 274 resources.jobs.*.job_clusters[*].new_cluster.cluster_log_conf.dbfs.destination + 275 resources.jobs.*.job_clusters[*].new_cluster.cluster_log_conf.s3 + 276 resources.jobs.*.job_clusters[*].new_cluster.cluster_log_conf.s3.canned_acl + 277 resources.jobs.*.job_clusters[*].new_cluster.cluster_log_conf.s3.destination + 278 resources.jobs.*.job_clusters[*].new_cluster.cluster_log_conf.s3.enable_encryption + 279 resources.jobs.*.job_clusters[*].new_cluster.cluster_log_conf.s3.enable_encryption=true + 280 resources.jobs.*.job_clusters[*].new_cluster.cluster_log_conf.s3.enable_encryption=false + 281 resources.jobs.*.job_clusters[*].new_cluster.cluster_log_conf.s3.encryption_type + 282 resources.jobs.*.job_clusters[*].new_cluster.cluster_log_conf.s3.endpoint + 283 resources.jobs.*.job_clusters[*].new_cluster.cluster_log_conf.s3.kms_key + 284 resources.jobs.*.job_clusters[*].new_cluster.cluster_log_conf.s3.region + 285 resources.jobs.*.job_clusters[*].new_cluster.cluster_log_conf.volumes + 286 resources.jobs.*.job_clusters[*].new_cluster.cluster_log_conf.volumes.destination + 287 resources.jobs.*.job_clusters[*].new_cluster.cluster_name + 288 resources.jobs.*.job_clusters[*].new_cluster.custom_tags + 289 resources.jobs.*.job_clusters[*].new_cluster.custom_tags.* + 290 resources.jobs.*.job_clusters[*].new_cluster.data_security_mode + 291 resources.jobs.*.job_clusters[*].new_cluster.dependency_mode + 292 resources.jobs.*.job_clusters[*].new_cluster.docker_image + 293 resources.jobs.*.job_clusters[*].new_cluster.docker_image.basic_auth + 294 resources.jobs.*.job_clusters[*].new_cluster.docker_image.basic_auth.password + 295 resources.jobs.*.job_clusters[*].new_cluster.docker_image.basic_auth.username + 296 resources.jobs.*.job_clusters[*].new_cluster.docker_image.url + 297 resources.jobs.*.job_clusters[*].new_cluster.driver_instance_pool_id + 298 resources.jobs.*.job_clusters[*].new_cluster.driver_node_type_flexibility + 299 resources.jobs.*.job_clusters[*].new_cluster.driver_node_type_flexibility.alternate_node_type_ids + 300 resources.jobs.*.job_clusters[*].new_cluster.driver_node_type_flexibility.alternate_node_type_ids[*] + 301 resources.jobs.*.job_clusters[*].new_cluster.driver_node_type_flexibility.aws_context_id + 302 resources.jobs.*.job_clusters[*].new_cluster.driver_node_type_id + 303 resources.jobs.*.job_clusters[*].new_cluster.enable_elastic_disk + 304 resources.jobs.*.job_clusters[*].new_cluster.enable_elastic_disk=true + 305 resources.jobs.*.job_clusters[*].new_cluster.enable_elastic_disk=false + 306 resources.jobs.*.job_clusters[*].new_cluster.enable_local_disk_encryption + 307 resources.jobs.*.job_clusters[*].new_cluster.enable_local_disk_encryption=true + 308 resources.jobs.*.job_clusters[*].new_cluster.enable_local_disk_encryption=false + 309 resources.jobs.*.job_clusters[*].new_cluster.gcp_attributes + 310 resources.jobs.*.job_clusters[*].new_cluster.gcp_attributes.availability + 311 resources.jobs.*.job_clusters[*].new_cluster.gcp_attributes.boot_disk_size + 312 resources.jobs.*.job_clusters[*].new_cluster.gcp_attributes.confidential_compute_type + 313 resources.jobs.*.job_clusters[*].new_cluster.gcp_attributes.first_on_demand + 314 resources.jobs.*.job_clusters[*].new_cluster.gcp_attributes.google_service_account + 315 resources.jobs.*.job_clusters[*].new_cluster.gcp_attributes.local_ssd_count + 316 resources.jobs.*.job_clusters[*].new_cluster.gcp_attributes.use_preemptible_executors + 317 resources.jobs.*.job_clusters[*].new_cluster.gcp_attributes.use_preemptible_executors=true + 318 resources.jobs.*.job_clusters[*].new_cluster.gcp_attributes.use_preemptible_executors=false + 319 resources.jobs.*.job_clusters[*].new_cluster.gcp_attributes.zone_id + 320 resources.jobs.*.job_clusters[*].new_cluster.init_scripts + 321 resources.jobs.*.job_clusters[*].new_cluster.init_scripts[*] + 322 resources.jobs.*.job_clusters[*].new_cluster.init_scripts[*].abfss + 323 resources.jobs.*.job_clusters[*].new_cluster.init_scripts[*].abfss.destination + 324 resources.jobs.*.job_clusters[*].new_cluster.init_scripts[*].dbfs + 325 resources.jobs.*.job_clusters[*].new_cluster.init_scripts[*].dbfs.destination + 326 resources.jobs.*.job_clusters[*].new_cluster.init_scripts[*].file + 327 resources.jobs.*.job_clusters[*].new_cluster.init_scripts[*].file.destination + 328 resources.jobs.*.job_clusters[*].new_cluster.init_scripts[*].gcs + 329 resources.jobs.*.job_clusters[*].new_cluster.init_scripts[*].gcs.destination + 330 resources.jobs.*.job_clusters[*].new_cluster.init_scripts[*].s3 + 331 resources.jobs.*.job_clusters[*].new_cluster.init_scripts[*].s3.canned_acl + 332 resources.jobs.*.job_clusters[*].new_cluster.init_scripts[*].s3.destination + 333 resources.jobs.*.job_clusters[*].new_cluster.init_scripts[*].s3.enable_encryption + 334 resources.jobs.*.job_clusters[*].new_cluster.init_scripts[*].s3.enable_encryption=true + 335 resources.jobs.*.job_clusters[*].new_cluster.init_scripts[*].s3.enable_encryption=false + 336 resources.jobs.*.job_clusters[*].new_cluster.init_scripts[*].s3.encryption_type + 337 resources.jobs.*.job_clusters[*].new_cluster.init_scripts[*].s3.endpoint + 338 resources.jobs.*.job_clusters[*].new_cluster.init_scripts[*].s3.kms_key + 339 resources.jobs.*.job_clusters[*].new_cluster.init_scripts[*].s3.region + 340 resources.jobs.*.job_clusters[*].new_cluster.init_scripts[*].volumes + 341 resources.jobs.*.job_clusters[*].new_cluster.init_scripts[*].volumes.destination + 342 resources.jobs.*.job_clusters[*].new_cluster.init_scripts[*].workspace + 343 resources.jobs.*.job_clusters[*].new_cluster.init_scripts[*].workspace.destination + 344 resources.jobs.*.job_clusters[*].new_cluster.instance_pool_id + 345 resources.jobs.*.job_clusters[*].new_cluster.is_single_node + 346 resources.jobs.*.job_clusters[*].new_cluster.is_single_node=true + 347 resources.jobs.*.job_clusters[*].new_cluster.is_single_node=false + 348 resources.jobs.*.job_clusters[*].new_cluster.kind + 349 resources.jobs.*.job_clusters[*].new_cluster.node_type_id + 350 resources.jobs.*.job_clusters[*].new_cluster.num_workers + 351 resources.jobs.*.job_clusters[*].new_cluster.policy_id + 352 resources.jobs.*.job_clusters[*].new_cluster.remote_disk_throughput + 353 resources.jobs.*.job_clusters[*].new_cluster.runtime_engine + 354 resources.jobs.*.job_clusters[*].new_cluster.single_user_name + 355 resources.jobs.*.job_clusters[*].new_cluster.spark_conf + 356 resources.jobs.*.job_clusters[*].new_cluster.spark_conf.* + 357 resources.jobs.*.job_clusters[*].new_cluster.spark_env_vars + 358 resources.jobs.*.job_clusters[*].new_cluster.spark_env_vars.* + 359 resources.jobs.*.job_clusters[*].new_cluster.spark_version + 360 resources.jobs.*.job_clusters[*].new_cluster.ssh_public_keys + 361 resources.jobs.*.job_clusters[*].new_cluster.ssh_public_keys[*] + 362 resources.jobs.*.job_clusters[*].new_cluster.total_initial_remote_disk_size + 363 resources.jobs.*.job_clusters[*].new_cluster.use_ml_runtime + 364 resources.jobs.*.job_clusters[*].new_cluster.use_ml_runtime=true + 365 resources.jobs.*.job_clusters[*].new_cluster.use_ml_runtime=false + 366 resources.jobs.*.job_clusters[*].new_cluster.worker_node_type_flexibility + 367 resources.jobs.*.job_clusters[*].new_cluster.worker_node_type_flexibility.alternate_node_type_ids + 368 resources.jobs.*.job_clusters[*].new_cluster.worker_node_type_flexibility.alternate_node_type_ids[*] + 369 resources.jobs.*.job_clusters[*].new_cluster.worker_node_type_flexibility.aws_context_id + 370 resources.jobs.*.job_clusters[*].new_cluster.workload_type + 371 resources.jobs.*.job_clusters[*].new_cluster.workload_type.clients + 372 resources.jobs.*.job_clusters[*].new_cluster.workload_type.clients.jobs + 373 resources.jobs.*.job_clusters[*].new_cluster.workload_type.clients.jobs=true + 374 resources.jobs.*.job_clusters[*].new_cluster.workload_type.clients.jobs=false + 375 resources.jobs.*.job_clusters[*].new_cluster.workload_type.clients.notebooks + 376 resources.jobs.*.job_clusters[*].new_cluster.workload_type.clients.notebooks=true + 377 resources.jobs.*.job_clusters[*].new_cluster.workload_type.clients.notebooks=false + 378 resources.jobs.*.job_clusters[*].serverless_compute + 379 resources.jobs.*.job_clusters[*].serverless_compute.policy_id + 380 resources.jobs.*.job_clusters[*].serverless_compute_id + 381 resources.jobs.*.max_concurrent_runs + 382 resources.jobs.*.name + 383 resources.jobs.*.notification_settings + 384 resources.jobs.*.notification_settings.no_alert_for_canceled_runs + 385 resources.jobs.*.notification_settings.no_alert_for_canceled_runs=true + 386 resources.jobs.*.notification_settings.no_alert_for_canceled_runs=false + 387 resources.jobs.*.notification_settings.no_alert_for_skipped_runs + 388 resources.jobs.*.notification_settings.no_alert_for_skipped_runs=true + 389 resources.jobs.*.notification_settings.no_alert_for_skipped_runs=false + 390 resources.jobs.*.parameters + 391 resources.jobs.*.parameters[*] + 392 resources.jobs.*.parameters[*].default + 393 resources.jobs.*.parameters[*].name + 394 resources.jobs.*.parent_path + 395 resources.jobs.*.performance_target + 396 resources.jobs.*.queue + 397 resources.jobs.*.queue.enabled + 398 resources.jobs.*.queue.enabled=true + 399 resources.jobs.*.queue.enabled=false + 400 resources.jobs.*.run_as + 401 resources.jobs.*.run_as.group_name + 402 resources.jobs.*.run_as.service_principal_name + 403 resources.jobs.*.run_as.user_name + 404 resources.jobs.*.schedule + 405 resources.jobs.*.schedule.pause_status + 406 resources.jobs.*.schedule.quartz_cron_expression + 407 resources.jobs.*.schedule.sql_condition + 408 resources.jobs.*.schedule.sql_condition.sql_query_id + 409 resources.jobs.*.schedule.sql_condition.trigger_mode + 410 resources.jobs.*.schedule.sql_condition.warehouse_id + 411 resources.jobs.*.schedule.timezone_id + 412 resources.jobs.*.tags + 413 resources.jobs.*.tags.* + 414 resources.jobs.*.tasks + 415 resources.jobs.*.tasks[*] + 416 resources.jobs.*.tasks[*].ai_function_task + 417 resources.jobs.*.tasks[*].ai_function_task.ai_classify + 418 resources.jobs.*.tasks[*].ai_function_task.ai_classify.enable_confidence_scores + 419 resources.jobs.*.tasks[*].ai_function_task.ai_classify.enable_confidence_scores=true + 420 resources.jobs.*.tasks[*].ai_function_task.ai_classify.enable_confidence_scores=false + 421 resources.jobs.*.tasks[*].ai_function_task.ai_classify.enable_rationales + 422 resources.jobs.*.tasks[*].ai_function_task.ai_classify.enable_rationales=true + 423 resources.jobs.*.tasks[*].ai_function_task.ai_classify.enable_rationales=false + 424 resources.jobs.*.tasks[*].ai_function_task.ai_classify.inline_labels + 425 resources.jobs.*.tasks[*].ai_function_task.ai_classify.inline_labels.labels + 426 resources.jobs.*.tasks[*].ai_function_task.ai_classify.inline_labels.labels[*] + 427 resources.jobs.*.tasks[*].ai_function_task.ai_classify.inline_labels.labels[*].description + 428 resources.jobs.*.tasks[*].ai_function_task.ai_classify.inline_labels.labels[*].name + 429 resources.jobs.*.tasks[*].ai_function_task.ai_classify.instructions + 430 resources.jobs.*.tasks[*].ai_function_task.ai_classify.multilabel + 431 resources.jobs.*.tasks[*].ai_function_task.ai_classify.multilabel=true + 432 resources.jobs.*.tasks[*].ai_function_task.ai_classify.multilabel=false + 433 resources.jobs.*.tasks[*].ai_function_task.ai_classify.version + 434 resources.jobs.*.tasks[*].ai_function_task.ai_extract + 435 resources.jobs.*.tasks[*].ai_function_task.ai_extract.enable_citations + 436 resources.jobs.*.tasks[*].ai_function_task.ai_extract.enable_citations=true + 437 resources.jobs.*.tasks[*].ai_function_task.ai_extract.enable_citations=false + 438 resources.jobs.*.tasks[*].ai_function_task.ai_extract.enable_confidence_scores + 439 resources.jobs.*.tasks[*].ai_function_task.ai_extract.enable_confidence_scores=true + 440 resources.jobs.*.tasks[*].ai_function_task.ai_extract.enable_confidence_scores=false + 441 resources.jobs.*.tasks[*].ai_function_task.ai_extract.instructions + 442 resources.jobs.*.tasks[*].ai_function_task.ai_extract.mode + 443 resources.jobs.*.tasks[*].ai_function_task.ai_extract.schema + 444 resources.jobs.*.tasks[*].ai_function_task.ai_extract.version + 445 resources.jobs.*.tasks[*].ai_function_task.ai_parse_document + 446 resources.jobs.*.tasks[*].ai_function_task.ai_parse_document.description_element_types + 447 resources.jobs.*.tasks[*].ai_function_task.ai_parse_document.description_element_types[*] + 448 resources.jobs.*.tasks[*].ai_function_task.ai_parse_document.image_output_path + 449 resources.jobs.*.tasks[*].ai_function_task.ai_parse_document.page_range + 450 resources.jobs.*.tasks[*].ai_function_task.ai_parse_document.version + 451 resources.jobs.*.tasks[*].ai_function_task.delta_table + 452 resources.jobs.*.tasks[*].ai_function_task.delta_table.input_column + 453 resources.jobs.*.tasks[*].ai_function_task.delta_table.merge_key_columns + 454 resources.jobs.*.tasks[*].ai_function_task.delta_table.merge_key_columns[*] + 455 resources.jobs.*.tasks[*].ai_function_task.delta_table.table_name + 456 resources.jobs.*.tasks[*].ai_function_task.output + 457 resources.jobs.*.tasks[*].ai_function_task.output.catalog + 458 resources.jobs.*.tasks[*].ai_function_task.output.schema + 459 resources.jobs.*.tasks[*].ai_function_task.output.table + 460 resources.jobs.*.tasks[*].ai_function_task.volume + 461 resources.jobs.*.tasks[*].ai_function_task.volume.volume_path + 462 resources.jobs.*.tasks[*].ai_runtime_task + 463 resources.jobs.*.tasks[*].ai_runtime_task.code_source_path + 464 resources.jobs.*.tasks[*].ai_runtime_task.deployments + 465 resources.jobs.*.tasks[*].ai_runtime_task.deployments[*] + 466 resources.jobs.*.tasks[*].ai_runtime_task.deployments[*].command_path + 467 resources.jobs.*.tasks[*].ai_runtime_task.deployments[*].compute + 468 resources.jobs.*.tasks[*].ai_runtime_task.deployments[*].compute.accelerator_count + 469 resources.jobs.*.tasks[*].ai_runtime_task.deployments[*].compute.accelerator_type + 470 resources.jobs.*.tasks[*].ai_runtime_task.deployments[*].compute.provisioned_capacity_id + 471 resources.jobs.*.tasks[*].ai_runtime_task.deployments[*].name + 472 resources.jobs.*.tasks[*].ai_runtime_task.docker_image_url + 473 resources.jobs.*.tasks[*].ai_runtime_task.experiment + 474 resources.jobs.*.tasks[*].ai_runtime_task.mlflow_artifact_location + 475 resources.jobs.*.tasks[*].ai_runtime_task.mlflow_experiment_directory + 476 resources.jobs.*.tasks[*].ai_runtime_task.mlflow_run + 477 resources.jobs.*.tasks[*].ai_runtime_task.priority_class + 478 resources.jobs.*.tasks[*].ai_runtime_task.unity_catalog_image_path + 479 resources.jobs.*.tasks[*].alert_task + 480 resources.jobs.*.tasks[*].alert_task.alert_id + 481 resources.jobs.*.tasks[*].alert_task.parameters + 482 resources.jobs.*.tasks[*].alert_task.parameters.* + 483 resources.jobs.*.tasks[*].alert_task.subscribers + 484 resources.jobs.*.tasks[*].alert_task.subscribers[*] + 485 resources.jobs.*.tasks[*].alert_task.subscribers[*].destination_id + 486 resources.jobs.*.tasks[*].alert_task.subscribers[*].user_name + 487 resources.jobs.*.tasks[*].alert_task.warehouse_id + 488 resources.jobs.*.tasks[*].alert_task.workspace_path + 489 resources.jobs.*.tasks[*].clean_rooms_notebook_task + 490 resources.jobs.*.tasks[*].clean_rooms_notebook_task.clean_room_name + 491 resources.jobs.*.tasks[*].clean_rooms_notebook_task.etag + 492 resources.jobs.*.tasks[*].clean_rooms_notebook_task.notebook_base_parameters + 493 resources.jobs.*.tasks[*].clean_rooms_notebook_task.notebook_base_parameters.* + 494 resources.jobs.*.tasks[*].clean_rooms_notebook_task.notebook_name + 495 resources.jobs.*.tasks[*].compute + 496 resources.jobs.*.tasks[*].compute.hardware_accelerator + 497 resources.jobs.*.tasks[*].condition_task + 498 resources.jobs.*.tasks[*].condition_task.left + 499 resources.jobs.*.tasks[*].condition_task.op + 500 resources.jobs.*.tasks[*].condition_task.right + 501 resources.jobs.*.tasks[*].dashboard_task + 502 resources.jobs.*.tasks[*].dashboard_task.dashboard_id + 503 resources.jobs.*.tasks[*].dashboard_task.filters + 504 resources.jobs.*.tasks[*].dashboard_task.filters.* + 505 resources.jobs.*.tasks[*].dashboard_task.subscription + 506 resources.jobs.*.tasks[*].dashboard_task.subscription.attachment + 507 resources.jobs.*.tasks[*].dashboard_task.subscription.attachment.data_attachment + 508 resources.jobs.*.tasks[*].dashboard_task.subscription.attachment.data_attachment.file_format + 509 resources.jobs.*.tasks[*].dashboard_task.subscription.attachment.data_attachment.include_applied_filters + 510 resources.jobs.*.tasks[*].dashboard_task.subscription.attachment.data_attachment.include_applied_filters=true + 511 resources.jobs.*.tasks[*].dashboard_task.subscription.attachment.data_attachment.include_applied_filters=false + 512 resources.jobs.*.tasks[*].dashboard_task.subscription.attachment.data_attachment.widgets + 513 resources.jobs.*.tasks[*].dashboard_task.subscription.attachment.data_attachment.widgets[*] + 514 resources.jobs.*.tasks[*].dashboard_task.subscription.attachment.data_attachment.widgets[*].page_id + 515 resources.jobs.*.tasks[*].dashboard_task.subscription.attachment.data_attachment.widgets[*].widget_id + 516 resources.jobs.*.tasks[*].dashboard_task.subscription.attachment.page_attachment + 517 resources.jobs.*.tasks[*].dashboard_task.subscription.attachment.page_attachment.page_ids + 518 resources.jobs.*.tasks[*].dashboard_task.subscription.attachment.page_attachment.page_ids[*] + 519 resources.jobs.*.tasks[*].dashboard_task.subscription.attachment.page_attachment.page_selection + 520 resources.jobs.*.tasks[*].dashboard_task.subscription.custom_subject + 521 resources.jobs.*.tasks[*].dashboard_task.subscription.paused + 522 resources.jobs.*.tasks[*].dashboard_task.subscription.paused=true + 523 resources.jobs.*.tasks[*].dashboard_task.subscription.paused=false + 524 resources.jobs.*.tasks[*].dashboard_task.subscription.subscribers + 525 resources.jobs.*.tasks[*].dashboard_task.subscription.subscribers[*] + 526 resources.jobs.*.tasks[*].dashboard_task.subscription.subscribers[*].destination_id + 527 resources.jobs.*.tasks[*].dashboard_task.subscription.subscribers[*].user_name + 528 resources.jobs.*.tasks[*].dashboard_task.warehouse_id + 529 resources.jobs.*.tasks[*].dbt_cloud_task + 530 resources.jobs.*.tasks[*].dbt_cloud_task.connection_resource_name + 531 resources.jobs.*.tasks[*].dbt_cloud_task.dbt_cloud_job_id + 532 resources.jobs.*.tasks[*].dbt_platform_task + 533 resources.jobs.*.tasks[*].dbt_platform_task.connection_resource_name + 534 resources.jobs.*.tasks[*].dbt_platform_task.dbt_platform_job_id + 535 resources.jobs.*.tasks[*].dbt_task + 536 resources.jobs.*.tasks[*].dbt_task.catalog + 537 resources.jobs.*.tasks[*].dbt_task.commands + 538 resources.jobs.*.tasks[*].dbt_task.commands[*] + 539 resources.jobs.*.tasks[*].dbt_task.profiles_directory + 540 resources.jobs.*.tasks[*].dbt_task.project_directory + 541 resources.jobs.*.tasks[*].dbt_task.schema + 542 resources.jobs.*.tasks[*].dbt_task.source + 543 resources.jobs.*.tasks[*].dbt_task.warehouse_id + 544 resources.jobs.*.tasks[*].depends_on + 545 resources.jobs.*.tasks[*].depends_on[*] + 546 resources.jobs.*.tasks[*].depends_on[*].outcome + 547 resources.jobs.*.tasks[*].depends_on[*].task_key + 548 resources.jobs.*.tasks[*].description + 549 resources.jobs.*.tasks[*].disable_auto_optimization + 550 resources.jobs.*.tasks[*].disable_auto_optimization=true + 551 resources.jobs.*.tasks[*].disable_auto_optimization=false + 552 resources.jobs.*.tasks[*].disabled + 553 resources.jobs.*.tasks[*].disabled=true + 554 resources.jobs.*.tasks[*].disabled=false + 555 resources.jobs.*.tasks[*].email_notifications + 556 resources.jobs.*.tasks[*].email_notifications.no_alert_for_skipped_runs + 557 resources.jobs.*.tasks[*].email_notifications.no_alert_for_skipped_runs=true + 558 resources.jobs.*.tasks[*].email_notifications.no_alert_for_skipped_runs=false + 559 resources.jobs.*.tasks[*].email_notifications.on_duration_warning_threshold_exceeded + 560 resources.jobs.*.tasks[*].email_notifications.on_duration_warning_threshold_exceeded[*] + 561 resources.jobs.*.tasks[*].email_notifications.on_failure + 562 resources.jobs.*.tasks[*].email_notifications.on_failure[*] + 563 resources.jobs.*.tasks[*].email_notifications.on_maintenance_complete + 564 resources.jobs.*.tasks[*].email_notifications.on_maintenance_complete[*] + 565 resources.jobs.*.tasks[*].email_notifications.on_maintenance_start + 566 resources.jobs.*.tasks[*].email_notifications.on_maintenance_start[*] + 567 resources.jobs.*.tasks[*].email_notifications.on_start + 568 resources.jobs.*.tasks[*].email_notifications.on_start[*] + 569 resources.jobs.*.tasks[*].email_notifications.on_streaming_backlog_exceeded + 570 resources.jobs.*.tasks[*].email_notifications.on_streaming_backlog_exceeded[*] + 571 resources.jobs.*.tasks[*].email_notifications.on_success + 572 resources.jobs.*.tasks[*].email_notifications.on_success[*] + 573 resources.jobs.*.tasks[*].environment_key + 574 resources.jobs.*.tasks[*].environment_variables_key + 575 resources.jobs.*.tasks[*].existing_cluster_id + 576 resources.jobs.*.tasks[*].for_each_task + 577 resources.jobs.*.tasks[*].for_each_task.concurrency + 578 resources.jobs.*.tasks[*].for_each_task.inputs + 579 resources.jobs.*.tasks[*].for_each_task.task + 580 resources.jobs.*.tasks[*].for_each_task.task.ai_function_task + 581 resources.jobs.*.tasks[*].for_each_task.task.ai_function_task.ai_classify + 582 resources.jobs.*.tasks[*].for_each_task.task.ai_function_task.ai_classify.enable_confidence_scores + 583 resources.jobs.*.tasks[*].for_each_task.task.ai_function_task.ai_classify.enable_confidence_scores=true + 584 resources.jobs.*.tasks[*].for_each_task.task.ai_function_task.ai_classify.enable_confidence_scores=false + 585 resources.jobs.*.tasks[*].for_each_task.task.ai_function_task.ai_classify.enable_rationales + 586 resources.jobs.*.tasks[*].for_each_task.task.ai_function_task.ai_classify.enable_rationales=true + 587 resources.jobs.*.tasks[*].for_each_task.task.ai_function_task.ai_classify.enable_rationales=false + 588 resources.jobs.*.tasks[*].for_each_task.task.ai_function_task.ai_classify.inline_labels + 589 resources.jobs.*.tasks[*].for_each_task.task.ai_function_task.ai_classify.inline_labels.labels + 590 resources.jobs.*.tasks[*].for_each_task.task.ai_function_task.ai_classify.inline_labels.labels[*] + 591 resources.jobs.*.tasks[*].for_each_task.task.ai_function_task.ai_classify.inline_labels.labels[*].description + 592 resources.jobs.*.tasks[*].for_each_task.task.ai_function_task.ai_classify.inline_labels.labels[*].name + 593 resources.jobs.*.tasks[*].for_each_task.task.ai_function_task.ai_classify.instructions + 594 resources.jobs.*.tasks[*].for_each_task.task.ai_function_task.ai_classify.multilabel + 595 resources.jobs.*.tasks[*].for_each_task.task.ai_function_task.ai_classify.multilabel=true + 596 resources.jobs.*.tasks[*].for_each_task.task.ai_function_task.ai_classify.multilabel=false + 597 resources.jobs.*.tasks[*].for_each_task.task.ai_function_task.ai_classify.version + 598 resources.jobs.*.tasks[*].for_each_task.task.ai_function_task.ai_extract + 599 resources.jobs.*.tasks[*].for_each_task.task.ai_function_task.ai_extract.enable_citations + 600 resources.jobs.*.tasks[*].for_each_task.task.ai_function_task.ai_extract.enable_citations=true + 601 resources.jobs.*.tasks[*].for_each_task.task.ai_function_task.ai_extract.enable_citations=false + 602 resources.jobs.*.tasks[*].for_each_task.task.ai_function_task.ai_extract.enable_confidence_scores + 603 resources.jobs.*.tasks[*].for_each_task.task.ai_function_task.ai_extract.enable_confidence_scores=true + 604 resources.jobs.*.tasks[*].for_each_task.task.ai_function_task.ai_extract.enable_confidence_scores=false + 605 resources.jobs.*.tasks[*].for_each_task.task.ai_function_task.ai_extract.instructions + 606 resources.jobs.*.tasks[*].for_each_task.task.ai_function_task.ai_extract.mode + 607 resources.jobs.*.tasks[*].for_each_task.task.ai_function_task.ai_extract.schema + 608 resources.jobs.*.tasks[*].for_each_task.task.ai_function_task.ai_extract.version + 609 resources.jobs.*.tasks[*].for_each_task.task.ai_function_task.ai_parse_document + 610 resources.jobs.*.tasks[*].for_each_task.task.ai_function_task.ai_parse_document.description_element_types + 611 resources.jobs.*.tasks[*].for_each_task.task.ai_function_task.ai_parse_document.description_element_types[*] + 612 resources.jobs.*.tasks[*].for_each_task.task.ai_function_task.ai_parse_document.image_output_path + 613 resources.jobs.*.tasks[*].for_each_task.task.ai_function_task.ai_parse_document.page_range + 614 resources.jobs.*.tasks[*].for_each_task.task.ai_function_task.ai_parse_document.version + 615 resources.jobs.*.tasks[*].for_each_task.task.ai_function_task.delta_table + 616 resources.jobs.*.tasks[*].for_each_task.task.ai_function_task.delta_table.input_column + 617 resources.jobs.*.tasks[*].for_each_task.task.ai_function_task.delta_table.merge_key_columns + 618 resources.jobs.*.tasks[*].for_each_task.task.ai_function_task.delta_table.merge_key_columns[*] + 619 resources.jobs.*.tasks[*].for_each_task.task.ai_function_task.delta_table.table_name + 620 resources.jobs.*.tasks[*].for_each_task.task.ai_function_task.output + 621 resources.jobs.*.tasks[*].for_each_task.task.ai_function_task.output.catalog + 622 resources.jobs.*.tasks[*].for_each_task.task.ai_function_task.output.schema + 623 resources.jobs.*.tasks[*].for_each_task.task.ai_function_task.output.table + 624 resources.jobs.*.tasks[*].for_each_task.task.ai_function_task.volume + 625 resources.jobs.*.tasks[*].for_each_task.task.ai_function_task.volume.volume_path + 626 resources.jobs.*.tasks[*].for_each_task.task.ai_runtime_task + 627 resources.jobs.*.tasks[*].for_each_task.task.ai_runtime_task.code_source_path + 628 resources.jobs.*.tasks[*].for_each_task.task.ai_runtime_task.deployments + 629 resources.jobs.*.tasks[*].for_each_task.task.ai_runtime_task.deployments[*] + 630 resources.jobs.*.tasks[*].for_each_task.task.ai_runtime_task.deployments[*].command_path + 631 resources.jobs.*.tasks[*].for_each_task.task.ai_runtime_task.deployments[*].compute + 632 resources.jobs.*.tasks[*].for_each_task.task.ai_runtime_task.deployments[*].compute.accelerator_count + 633 resources.jobs.*.tasks[*].for_each_task.task.ai_runtime_task.deployments[*].compute.accelerator_type + 634 resources.jobs.*.tasks[*].for_each_task.task.ai_runtime_task.deployments[*].compute.provisioned_capacity_id + 635 resources.jobs.*.tasks[*].for_each_task.task.ai_runtime_task.deployments[*].name + 636 resources.jobs.*.tasks[*].for_each_task.task.ai_runtime_task.docker_image_url + 637 resources.jobs.*.tasks[*].for_each_task.task.ai_runtime_task.experiment + 638 resources.jobs.*.tasks[*].for_each_task.task.ai_runtime_task.mlflow_artifact_location + 639 resources.jobs.*.tasks[*].for_each_task.task.ai_runtime_task.mlflow_experiment_directory + 640 resources.jobs.*.tasks[*].for_each_task.task.ai_runtime_task.mlflow_run + 641 resources.jobs.*.tasks[*].for_each_task.task.ai_runtime_task.priority_class + 642 resources.jobs.*.tasks[*].for_each_task.task.ai_runtime_task.unity_catalog_image_path + 643 resources.jobs.*.tasks[*].for_each_task.task.alert_task + 644 resources.jobs.*.tasks[*].for_each_task.task.alert_task.alert_id + 645 resources.jobs.*.tasks[*].for_each_task.task.alert_task.parameters + 646 resources.jobs.*.tasks[*].for_each_task.task.alert_task.parameters.* + 647 resources.jobs.*.tasks[*].for_each_task.task.alert_task.subscribers + 648 resources.jobs.*.tasks[*].for_each_task.task.alert_task.subscribers[*] + 649 resources.jobs.*.tasks[*].for_each_task.task.alert_task.subscribers[*].destination_id + 650 resources.jobs.*.tasks[*].for_each_task.task.alert_task.subscribers[*].user_name + 651 resources.jobs.*.tasks[*].for_each_task.task.alert_task.warehouse_id + 652 resources.jobs.*.tasks[*].for_each_task.task.alert_task.workspace_path + 653 resources.jobs.*.tasks[*].for_each_task.task.clean_rooms_notebook_task + 654 resources.jobs.*.tasks[*].for_each_task.task.clean_rooms_notebook_task.clean_room_name + 655 resources.jobs.*.tasks[*].for_each_task.task.clean_rooms_notebook_task.etag + 656 resources.jobs.*.tasks[*].for_each_task.task.clean_rooms_notebook_task.notebook_base_parameters + 657 resources.jobs.*.tasks[*].for_each_task.task.clean_rooms_notebook_task.notebook_base_parameters.* + 658 resources.jobs.*.tasks[*].for_each_task.task.clean_rooms_notebook_task.notebook_name + 659 resources.jobs.*.tasks[*].for_each_task.task.compute + 660 resources.jobs.*.tasks[*].for_each_task.task.compute.hardware_accelerator + 661 resources.jobs.*.tasks[*].for_each_task.task.condition_task + 662 resources.jobs.*.tasks[*].for_each_task.task.condition_task.left + 663 resources.jobs.*.tasks[*].for_each_task.task.condition_task.op + 664 resources.jobs.*.tasks[*].for_each_task.task.condition_task.right + 665 resources.jobs.*.tasks[*].for_each_task.task.dashboard_task + 666 resources.jobs.*.tasks[*].for_each_task.task.dashboard_task.dashboard_id + 667 resources.jobs.*.tasks[*].for_each_task.task.dashboard_task.filters + 668 resources.jobs.*.tasks[*].for_each_task.task.dashboard_task.filters.* + 669 resources.jobs.*.tasks[*].for_each_task.task.dashboard_task.subscription + 670 resources.jobs.*.tasks[*].for_each_task.task.dashboard_task.subscription.attachment + 671 resources.jobs.*.tasks[*].for_each_task.task.dashboard_task.subscription.attachment.data_attachment + 672 resources.jobs.*.tasks[*].for_each_task.task.dashboard_task.subscription.attachment.data_attachment.file_format + 673 resources.jobs.*.tasks[*].for_each_task.task.dashboard_task.subscription.attachment.data_attachment.include_applied_filters + 674 resources.jobs.*.tasks[*].for_each_task.task.dashboard_task.subscription.attachment.data_attachment.include_applied_filters=true + 675 resources.jobs.*.tasks[*].for_each_task.task.dashboard_task.subscription.attachment.data_attachment.include_applied_filters=false + 676 resources.jobs.*.tasks[*].for_each_task.task.dashboard_task.subscription.attachment.data_attachment.widgets + 677 resources.jobs.*.tasks[*].for_each_task.task.dashboard_task.subscription.attachment.data_attachment.widgets[*] + 678 resources.jobs.*.tasks[*].for_each_task.task.dashboard_task.subscription.attachment.data_attachment.widgets[*].page_id + 679 resources.jobs.*.tasks[*].for_each_task.task.dashboard_task.subscription.attachment.data_attachment.widgets[*].widget_id + 680 resources.jobs.*.tasks[*].for_each_task.task.dashboard_task.subscription.attachment.page_attachment + 681 resources.jobs.*.tasks[*].for_each_task.task.dashboard_task.subscription.attachment.page_attachment.page_ids + 682 resources.jobs.*.tasks[*].for_each_task.task.dashboard_task.subscription.attachment.page_attachment.page_ids[*] + 683 resources.jobs.*.tasks[*].for_each_task.task.dashboard_task.subscription.attachment.page_attachment.page_selection + 684 resources.jobs.*.tasks[*].for_each_task.task.dashboard_task.subscription.custom_subject + 685 resources.jobs.*.tasks[*].for_each_task.task.dashboard_task.subscription.paused + 686 resources.jobs.*.tasks[*].for_each_task.task.dashboard_task.subscription.paused=true + 687 resources.jobs.*.tasks[*].for_each_task.task.dashboard_task.subscription.paused=false + 688 resources.jobs.*.tasks[*].for_each_task.task.dashboard_task.subscription.subscribers + 689 resources.jobs.*.tasks[*].for_each_task.task.dashboard_task.subscription.subscribers[*] + 690 resources.jobs.*.tasks[*].for_each_task.task.dashboard_task.subscription.subscribers[*].destination_id + 691 resources.jobs.*.tasks[*].for_each_task.task.dashboard_task.subscription.subscribers[*].user_name + 692 resources.jobs.*.tasks[*].for_each_task.task.dashboard_task.warehouse_id + 693 resources.jobs.*.tasks[*].for_each_task.task.dbt_cloud_task + 694 resources.jobs.*.tasks[*].for_each_task.task.dbt_cloud_task.connection_resource_name + 695 resources.jobs.*.tasks[*].for_each_task.task.dbt_cloud_task.dbt_cloud_job_id + 696 resources.jobs.*.tasks[*].for_each_task.task.dbt_platform_task + 697 resources.jobs.*.tasks[*].for_each_task.task.dbt_platform_task.connection_resource_name + 698 resources.jobs.*.tasks[*].for_each_task.task.dbt_platform_task.dbt_platform_job_id + 699 resources.jobs.*.tasks[*].for_each_task.task.dbt_task + 700 resources.jobs.*.tasks[*].for_each_task.task.dbt_task.catalog + 701 resources.jobs.*.tasks[*].for_each_task.task.dbt_task.commands + 702 resources.jobs.*.tasks[*].for_each_task.task.dbt_task.commands[*] + 703 resources.jobs.*.tasks[*].for_each_task.task.dbt_task.profiles_directory + 704 resources.jobs.*.tasks[*].for_each_task.task.dbt_task.project_directory + 705 resources.jobs.*.tasks[*].for_each_task.task.dbt_task.schema + 706 resources.jobs.*.tasks[*].for_each_task.task.dbt_task.source + 707 resources.jobs.*.tasks[*].for_each_task.task.dbt_task.warehouse_id + 708 resources.jobs.*.tasks[*].for_each_task.task.depends_on + 709 resources.jobs.*.tasks[*].for_each_task.task.depends_on[*] + 710 resources.jobs.*.tasks[*].for_each_task.task.depends_on[*].outcome + 711 resources.jobs.*.tasks[*].for_each_task.task.depends_on[*].task_key + 712 resources.jobs.*.tasks[*].for_each_task.task.description + 713 resources.jobs.*.tasks[*].for_each_task.task.disable_auto_optimization + 714 resources.jobs.*.tasks[*].for_each_task.task.disable_auto_optimization=true + 715 resources.jobs.*.tasks[*].for_each_task.task.disable_auto_optimization=false + 716 resources.jobs.*.tasks[*].for_each_task.task.disabled + 717 resources.jobs.*.tasks[*].for_each_task.task.disabled=true + 718 resources.jobs.*.tasks[*].for_each_task.task.disabled=false + 719 resources.jobs.*.tasks[*].for_each_task.task.email_notifications + 720 resources.jobs.*.tasks[*].for_each_task.task.email_notifications.no_alert_for_skipped_runs + 721 resources.jobs.*.tasks[*].for_each_task.task.email_notifications.no_alert_for_skipped_runs=true + 722 resources.jobs.*.tasks[*].for_each_task.task.email_notifications.no_alert_for_skipped_runs=false + 723 resources.jobs.*.tasks[*].for_each_task.task.email_notifications.on_duration_warning_threshold_exceeded + 724 resources.jobs.*.tasks[*].for_each_task.task.email_notifications.on_duration_warning_threshold_exceeded[*] + 725 resources.jobs.*.tasks[*].for_each_task.task.email_notifications.on_failure + 726 resources.jobs.*.tasks[*].for_each_task.task.email_notifications.on_failure[*] + 727 resources.jobs.*.tasks[*].for_each_task.task.email_notifications.on_maintenance_complete + 728 resources.jobs.*.tasks[*].for_each_task.task.email_notifications.on_maintenance_complete[*] + 729 resources.jobs.*.tasks[*].for_each_task.task.email_notifications.on_maintenance_start + 730 resources.jobs.*.tasks[*].for_each_task.task.email_notifications.on_maintenance_start[*] + 731 resources.jobs.*.tasks[*].for_each_task.task.email_notifications.on_start + 732 resources.jobs.*.tasks[*].for_each_task.task.email_notifications.on_start[*] + 733 resources.jobs.*.tasks[*].for_each_task.task.email_notifications.on_streaming_backlog_exceeded + 734 resources.jobs.*.tasks[*].for_each_task.task.email_notifications.on_streaming_backlog_exceeded[*] + 735 resources.jobs.*.tasks[*].for_each_task.task.email_notifications.on_success + 736 resources.jobs.*.tasks[*].for_each_task.task.email_notifications.on_success[*] + 737 resources.jobs.*.tasks[*].for_each_task.task.environment_key + 738 resources.jobs.*.tasks[*].for_each_task.task.environment_variables_key + 739 resources.jobs.*.tasks[*].for_each_task.task.existing_cluster_id + 740 resources.jobs.*.tasks[*].for_each_task.task.for_each_task + 741 resources.jobs.*.tasks[*].for_each_task.task.for_each_task.concurrency + 742 resources.jobs.*.tasks[*].for_each_task.task.for_each_task.inputs + 743 resources.jobs.*.tasks[*].for_each_task.task.for_each_task.task + 744 resources.jobs.*.tasks[*].for_each_task.task.gen_ai_compute_task + 745 resources.jobs.*.tasks[*].for_each_task.task.gen_ai_compute_task.command + 746 resources.jobs.*.tasks[*].for_each_task.task.gen_ai_compute_task.compute + 747 resources.jobs.*.tasks[*].for_each_task.task.gen_ai_compute_task.compute.gpu_node_pool_id + 748 resources.jobs.*.tasks[*].for_each_task.task.gen_ai_compute_task.compute.gpu_type + 749 resources.jobs.*.tasks[*].for_each_task.task.gen_ai_compute_task.compute.num_gpus + 750 resources.jobs.*.tasks[*].for_each_task.task.gen_ai_compute_task.dl_runtime_image + 751 resources.jobs.*.tasks[*].for_each_task.task.gen_ai_compute_task.mlflow_experiment_name + 752 resources.jobs.*.tasks[*].for_each_task.task.gen_ai_compute_task.source + 753 resources.jobs.*.tasks[*].for_each_task.task.gen_ai_compute_task.training_script_path + 754 resources.jobs.*.tasks[*].for_each_task.task.gen_ai_compute_task.yaml_parameters + 755 resources.jobs.*.tasks[*].for_each_task.task.gen_ai_compute_task.yaml_parameters_file_path + 756 resources.jobs.*.tasks[*].for_each_task.task.health + 757 resources.jobs.*.tasks[*].for_each_task.task.health.rules + 758 resources.jobs.*.tasks[*].for_each_task.task.health.rules[*] + 759 resources.jobs.*.tasks[*].for_each_task.task.health.rules[*].metric + 760 resources.jobs.*.tasks[*].for_each_task.task.health.rules[*].op + 761 resources.jobs.*.tasks[*].for_each_task.task.health.rules[*].value + 762 resources.jobs.*.tasks[*].for_each_task.task.job_cluster_key + 763 resources.jobs.*.tasks[*].for_each_task.task.libraries + 764 resources.jobs.*.tasks[*].for_each_task.task.libraries[*] + 765 resources.jobs.*.tasks[*].for_each_task.task.libraries[*].cran + 766 resources.jobs.*.tasks[*].for_each_task.task.libraries[*].cran.package + 767 resources.jobs.*.tasks[*].for_each_task.task.libraries[*].cran.repo + 768 resources.jobs.*.tasks[*].for_each_task.task.libraries[*].egg + 769 resources.jobs.*.tasks[*].for_each_task.task.libraries[*].jar + 770 resources.jobs.*.tasks[*].for_each_task.task.libraries[*].maven + 771 resources.jobs.*.tasks[*].for_each_task.task.libraries[*].maven.coordinates + 772 resources.jobs.*.tasks[*].for_each_task.task.libraries[*].maven.exclusions + 773 resources.jobs.*.tasks[*].for_each_task.task.libraries[*].maven.exclusions[*] + 774 resources.jobs.*.tasks[*].for_each_task.task.libraries[*].maven.repo + 775 resources.jobs.*.tasks[*].for_each_task.task.libraries[*].pypi + 776 resources.jobs.*.tasks[*].for_each_task.task.libraries[*].pypi.package + 777 resources.jobs.*.tasks[*].for_each_task.task.libraries[*].pypi.repo + 778 resources.jobs.*.tasks[*].for_each_task.task.libraries[*].requirements + 779 resources.jobs.*.tasks[*].for_each_task.task.libraries[*].whl + 780 resources.jobs.*.tasks[*].for_each_task.task.max_retries + 781 resources.jobs.*.tasks[*].for_each_task.task.min_retry_interval_millis + 782 resources.jobs.*.tasks[*].for_each_task.task.new_cluster + 783 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.apply_policy_default_values + 784 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.apply_policy_default_values=true + 785 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.apply_policy_default_values=false + 786 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.autoscale + 787 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.autoscale.max_workers + 788 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.autoscale.min_workers + 789 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.autotermination_minutes + 790 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.aws_attributes + 791 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.aws_attributes.availability + 792 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.aws_attributes.ebs_volume_count + 793 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.aws_attributes.ebs_volume_iops + 794 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.aws_attributes.ebs_volume_size + 795 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.aws_attributes.ebs_volume_throughput + 796 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.aws_attributes.ebs_volume_type + 797 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.aws_attributes.first_on_demand + 798 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.aws_attributes.instance_profile_arn + 799 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.aws_attributes.spot_bid_price_percent + 800 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.aws_attributes.zone_id + 801 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.azure_attributes + 802 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.azure_attributes.availability + 803 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.azure_attributes.capacity_reservation_group + 804 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.azure_attributes.first_on_demand + 805 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.azure_attributes.log_analytics_info + 806 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.azure_attributes.log_analytics_info.log_analytics_primary_key + 807 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.azure_attributes.log_analytics_info.log_analytics_workspace_id + 808 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.azure_attributes.spot_bid_max_price + 809 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.cluster_log_conf + 810 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.cluster_log_conf.dbfs + 811 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.cluster_log_conf.dbfs.destination + 812 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.cluster_log_conf.s3 + 813 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.cluster_log_conf.s3.canned_acl + 814 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.cluster_log_conf.s3.destination + 815 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.cluster_log_conf.s3.enable_encryption + 816 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.cluster_log_conf.s3.enable_encryption=true + 817 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.cluster_log_conf.s3.enable_encryption=false + 818 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.cluster_log_conf.s3.encryption_type + 819 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.cluster_log_conf.s3.endpoint + 820 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.cluster_log_conf.s3.kms_key + 821 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.cluster_log_conf.s3.region + 822 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.cluster_log_conf.volumes + 823 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.cluster_log_conf.volumes.destination + 824 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.cluster_name + 825 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.custom_tags + 826 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.custom_tags.* + 827 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.data_security_mode + 828 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.dependency_mode + 829 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.docker_image + 830 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.docker_image.basic_auth + 831 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.docker_image.basic_auth.password + 832 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.docker_image.basic_auth.username + 833 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.docker_image.url + 834 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.driver_instance_pool_id + 835 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.driver_node_type_flexibility + 836 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.driver_node_type_flexibility.alternate_node_type_ids + 837 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.driver_node_type_flexibility.alternate_node_type_ids[*] + 838 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.driver_node_type_flexibility.aws_context_id + 839 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.driver_node_type_id + 840 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.enable_elastic_disk + 841 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.enable_elastic_disk=true + 842 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.enable_elastic_disk=false + 843 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.enable_local_disk_encryption + 844 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.enable_local_disk_encryption=true + 845 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.enable_local_disk_encryption=false + 846 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.gcp_attributes + 847 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.gcp_attributes.availability + 848 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.gcp_attributes.boot_disk_size + 849 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.gcp_attributes.confidential_compute_type + 850 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.gcp_attributes.first_on_demand + 851 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.gcp_attributes.google_service_account + 852 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.gcp_attributes.local_ssd_count + 853 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.gcp_attributes.use_preemptible_executors + 854 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.gcp_attributes.use_preemptible_executors=true + 855 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.gcp_attributes.use_preemptible_executors=false + 856 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.gcp_attributes.zone_id + 857 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.init_scripts + 858 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.init_scripts[*] + 859 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.init_scripts[*].abfss + 860 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.init_scripts[*].abfss.destination + 861 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.init_scripts[*].dbfs + 862 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.init_scripts[*].dbfs.destination + 863 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.init_scripts[*].file + 864 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.init_scripts[*].file.destination + 865 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.init_scripts[*].gcs + 866 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.init_scripts[*].gcs.destination + 867 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.init_scripts[*].s3 + 868 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.init_scripts[*].s3.canned_acl + 869 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.init_scripts[*].s3.destination + 870 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.init_scripts[*].s3.enable_encryption + 871 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.init_scripts[*].s3.enable_encryption=true + 872 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.init_scripts[*].s3.enable_encryption=false + 873 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.init_scripts[*].s3.encryption_type + 874 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.init_scripts[*].s3.endpoint + 875 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.init_scripts[*].s3.kms_key + 876 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.init_scripts[*].s3.region + 877 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.init_scripts[*].volumes + 878 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.init_scripts[*].volumes.destination + 879 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.init_scripts[*].workspace + 880 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.init_scripts[*].workspace.destination + 881 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.instance_pool_id + 882 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.is_single_node + 883 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.is_single_node=true + 884 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.is_single_node=false + 885 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.kind + 886 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.node_type_id + 887 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.num_workers + 888 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.policy_id + 889 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.remote_disk_throughput + 890 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.runtime_engine + 891 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.single_user_name + 892 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.spark_conf + 893 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.spark_conf.* + 894 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.spark_env_vars + 895 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.spark_env_vars.* + 896 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.spark_version + 897 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.ssh_public_keys + 898 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.ssh_public_keys[*] + 899 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.total_initial_remote_disk_size + 900 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.use_ml_runtime + 901 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.use_ml_runtime=true + 902 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.use_ml_runtime=false + 903 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.worker_node_type_flexibility + 904 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.worker_node_type_flexibility.alternate_node_type_ids + 905 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.worker_node_type_flexibility.alternate_node_type_ids[*] + 906 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.worker_node_type_flexibility.aws_context_id + 907 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.workload_type + 908 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.workload_type.clients + 909 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.workload_type.clients.jobs + 910 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.workload_type.clients.jobs=true + 911 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.workload_type.clients.jobs=false + 912 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.workload_type.clients.notebooks + 913 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.workload_type.clients.notebooks=true + 914 resources.jobs.*.tasks[*].for_each_task.task.new_cluster.workload_type.clients.notebooks=false + 915 resources.jobs.*.tasks[*].for_each_task.task.notebook_task + 916 resources.jobs.*.tasks[*].for_each_task.task.notebook_task.base_parameters + 917 resources.jobs.*.tasks[*].for_each_task.task.notebook_task.base_parameters.* + 918 resources.jobs.*.tasks[*].for_each_task.task.notebook_task.notebook_path + 919 resources.jobs.*.tasks[*].for_each_task.task.notebook_task.source + 920 resources.jobs.*.tasks[*].for_each_task.task.notebook_task.warehouse_id + 921 resources.jobs.*.tasks[*].for_each_task.task.notification_settings + 922 resources.jobs.*.tasks[*].for_each_task.task.notification_settings.alert_on_last_attempt + 923 resources.jobs.*.tasks[*].for_each_task.task.notification_settings.alert_on_last_attempt=true + 924 resources.jobs.*.tasks[*].for_each_task.task.notification_settings.alert_on_last_attempt=false + 925 resources.jobs.*.tasks[*].for_each_task.task.notification_settings.no_alert_for_canceled_runs + 926 resources.jobs.*.tasks[*].for_each_task.task.notification_settings.no_alert_for_canceled_runs=true + 927 resources.jobs.*.tasks[*].for_each_task.task.notification_settings.no_alert_for_canceled_runs=false + 928 resources.jobs.*.tasks[*].for_each_task.task.notification_settings.no_alert_for_skipped_runs + 929 resources.jobs.*.tasks[*].for_each_task.task.notification_settings.no_alert_for_skipped_runs=true + 930 resources.jobs.*.tasks[*].for_each_task.task.notification_settings.no_alert_for_skipped_runs=false + 931 resources.jobs.*.tasks[*].for_each_task.task.pipeline_task + 932 resources.jobs.*.tasks[*].for_each_task.task.pipeline_task.full_refresh + 933 resources.jobs.*.tasks[*].for_each_task.task.pipeline_task.full_refresh=true + 934 resources.jobs.*.tasks[*].for_each_task.task.pipeline_task.full_refresh=false + 935 resources.jobs.*.tasks[*].for_each_task.task.pipeline_task.full_refresh_selection + 936 resources.jobs.*.tasks[*].for_each_task.task.pipeline_task.full_refresh_selection[*] + 937 resources.jobs.*.tasks[*].for_each_task.task.pipeline_task.parameters + 938 resources.jobs.*.tasks[*].for_each_task.task.pipeline_task.parameters.* + 939 resources.jobs.*.tasks[*].for_each_task.task.pipeline_task.pipeline_id + 940 resources.jobs.*.tasks[*].for_each_task.task.pipeline_task.refresh_flow_selection + 941 resources.jobs.*.tasks[*].for_each_task.task.pipeline_task.refresh_flow_selection[*] + 942 resources.jobs.*.tasks[*].for_each_task.task.pipeline_task.refresh_selection + 943 resources.jobs.*.tasks[*].for_each_task.task.pipeline_task.refresh_selection[*] + 944 resources.jobs.*.tasks[*].for_each_task.task.pipeline_task.reset_checkpoint_selection + 945 resources.jobs.*.tasks[*].for_each_task.task.pipeline_task.reset_checkpoint_selection[*] + 946 resources.jobs.*.tasks[*].for_each_task.task.power_bi_task + 947 resources.jobs.*.tasks[*].for_each_task.task.power_bi_task.connection_resource_name + 948 resources.jobs.*.tasks[*].for_each_task.task.power_bi_task.power_bi_model + 949 resources.jobs.*.tasks[*].for_each_task.task.power_bi_task.power_bi_model.authentication_method + 950 resources.jobs.*.tasks[*].for_each_task.task.power_bi_task.power_bi_model.model_name + 951 resources.jobs.*.tasks[*].for_each_task.task.power_bi_task.power_bi_model.overwrite_existing + 952 resources.jobs.*.tasks[*].for_each_task.task.power_bi_task.power_bi_model.overwrite_existing=true + 953 resources.jobs.*.tasks[*].for_each_task.task.power_bi_task.power_bi_model.overwrite_existing=false + 954 resources.jobs.*.tasks[*].for_each_task.task.power_bi_task.power_bi_model.storage_mode + 955 resources.jobs.*.tasks[*].for_each_task.task.power_bi_task.power_bi_model.workspace_name + 956 resources.jobs.*.tasks[*].for_each_task.task.power_bi_task.refresh_after_update + 957 resources.jobs.*.tasks[*].for_each_task.task.power_bi_task.refresh_after_update=true + 958 resources.jobs.*.tasks[*].for_each_task.task.power_bi_task.refresh_after_update=false + 959 resources.jobs.*.tasks[*].for_each_task.task.power_bi_task.tables + 960 resources.jobs.*.tasks[*].for_each_task.task.power_bi_task.tables[*] + 961 resources.jobs.*.tasks[*].for_each_task.task.power_bi_task.tables[*].catalog + 962 resources.jobs.*.tasks[*].for_each_task.task.power_bi_task.tables[*].name + 963 resources.jobs.*.tasks[*].for_each_task.task.power_bi_task.tables[*].schema + 964 resources.jobs.*.tasks[*].for_each_task.task.power_bi_task.tables[*].storage_mode + 965 resources.jobs.*.tasks[*].for_each_task.task.power_bi_task.warehouse_id + 966 resources.jobs.*.tasks[*].for_each_task.task.python_operator_task + 967 resources.jobs.*.tasks[*].for_each_task.task.python_operator_task.main + 968 resources.jobs.*.tasks[*].for_each_task.task.python_operator_task.parameters + 969 resources.jobs.*.tasks[*].for_each_task.task.python_operator_task.parameters[*] + 970 resources.jobs.*.tasks[*].for_each_task.task.python_operator_task.parameters[*].name + 971 resources.jobs.*.tasks[*].for_each_task.task.python_operator_task.parameters[*].value + 972 resources.jobs.*.tasks[*].for_each_task.task.python_wheel_task + 973 resources.jobs.*.tasks[*].for_each_task.task.python_wheel_task.entry_point + 974 resources.jobs.*.tasks[*].for_each_task.task.python_wheel_task.named_parameters + 975 resources.jobs.*.tasks[*].for_each_task.task.python_wheel_task.named_parameters.* + 976 resources.jobs.*.tasks[*].for_each_task.task.python_wheel_task.package_name + 977 resources.jobs.*.tasks[*].for_each_task.task.python_wheel_task.parameters + 978 resources.jobs.*.tasks[*].for_each_task.task.python_wheel_task.parameters[*] + 979 resources.jobs.*.tasks[*].for_each_task.task.retry_on_timeout + 980 resources.jobs.*.tasks[*].for_each_task.task.retry_on_timeout=true + 981 resources.jobs.*.tasks[*].for_each_task.task.retry_on_timeout=false + 982 resources.jobs.*.tasks[*].for_each_task.task.run_if + 983 resources.jobs.*.tasks[*].for_each_task.task.run_job_task + 984 resources.jobs.*.tasks[*].for_each_task.task.run_job_task.dbt_commands + 985 resources.jobs.*.tasks[*].for_each_task.task.run_job_task.dbt_commands[*] + 986 resources.jobs.*.tasks[*].for_each_task.task.run_job_task.jar_params + 987 resources.jobs.*.tasks[*].for_each_task.task.run_job_task.jar_params[*] + 988 resources.jobs.*.tasks[*].for_each_task.task.run_job_task.job_id + 989 resources.jobs.*.tasks[*].for_each_task.task.run_job_task.job_parameters + 990 resources.jobs.*.tasks[*].for_each_task.task.run_job_task.job_parameters.* + 991 resources.jobs.*.tasks[*].for_each_task.task.run_job_task.notebook_params + 992 resources.jobs.*.tasks[*].for_each_task.task.run_job_task.notebook_params.* + 993 resources.jobs.*.tasks[*].for_each_task.task.run_job_task.pipeline_params + 994 resources.jobs.*.tasks[*].for_each_task.task.run_job_task.pipeline_params.full_refresh + 995 resources.jobs.*.tasks[*].for_each_task.task.run_job_task.pipeline_params.full_refresh=true + 996 resources.jobs.*.tasks[*].for_each_task.task.run_job_task.pipeline_params.full_refresh=false + 997 resources.jobs.*.tasks[*].for_each_task.task.run_job_task.pipeline_params.full_refresh_selection + 998 resources.jobs.*.tasks[*].for_each_task.task.run_job_task.pipeline_params.full_refresh_selection[*] + 999 resources.jobs.*.tasks[*].for_each_task.task.run_job_task.pipeline_params.refresh_flow_selection +1000 resources.jobs.*.tasks[*].for_each_task.task.run_job_task.pipeline_params.refresh_flow_selection[*] +1001 resources.jobs.*.tasks[*].for_each_task.task.run_job_task.pipeline_params.refresh_selection +1002 resources.jobs.*.tasks[*].for_each_task.task.run_job_task.pipeline_params.refresh_selection[*] +1003 resources.jobs.*.tasks[*].for_each_task.task.run_job_task.pipeline_params.reset_checkpoint_selection +1004 resources.jobs.*.tasks[*].for_each_task.task.run_job_task.pipeline_params.reset_checkpoint_selection[*] +1005 resources.jobs.*.tasks[*].for_each_task.task.run_job_task.python_named_params +1006 resources.jobs.*.tasks[*].for_each_task.task.run_job_task.python_named_params.* +1007 resources.jobs.*.tasks[*].for_each_task.task.run_job_task.python_params +1008 resources.jobs.*.tasks[*].for_each_task.task.run_job_task.python_params[*] +1009 resources.jobs.*.tasks[*].for_each_task.task.run_job_task.spark_submit_params +1010 resources.jobs.*.tasks[*].for_each_task.task.run_job_task.spark_submit_params[*] +1011 resources.jobs.*.tasks[*].for_each_task.task.run_job_task.sql_params +1012 resources.jobs.*.tasks[*].for_each_task.task.run_job_task.sql_params.* +1013 resources.jobs.*.tasks[*].for_each_task.task.spark_jar_task +1014 resources.jobs.*.tasks[*].for_each_task.task.spark_jar_task.jar_uri +1015 resources.jobs.*.tasks[*].for_each_task.task.spark_jar_task.main_class_name +1016 resources.jobs.*.tasks[*].for_each_task.task.spark_jar_task.parameters +1017 resources.jobs.*.tasks[*].for_each_task.task.spark_jar_task.parameters[*] +1018 resources.jobs.*.tasks[*].for_each_task.task.spark_jar_task.run_as_repl +1019 resources.jobs.*.tasks[*].for_each_task.task.spark_jar_task.run_as_repl=true +1020 resources.jobs.*.tasks[*].for_each_task.task.spark_jar_task.run_as_repl=false +1021 resources.jobs.*.tasks[*].for_each_task.task.spark_python_task +1022 resources.jobs.*.tasks[*].for_each_task.task.spark_python_task.parameters +1023 resources.jobs.*.tasks[*].for_each_task.task.spark_python_task.parameters[*] +1024 resources.jobs.*.tasks[*].for_each_task.task.spark_python_task.python_file +1025 resources.jobs.*.tasks[*].for_each_task.task.spark_python_task.source +1026 resources.jobs.*.tasks[*].for_each_task.task.spark_submit_task +1027 resources.jobs.*.tasks[*].for_each_task.task.spark_submit_task.parameters +1028 resources.jobs.*.tasks[*].for_each_task.task.spark_submit_task.parameters[*] +1029 resources.jobs.*.tasks[*].for_each_task.task.sql_task +1030 resources.jobs.*.tasks[*].for_each_task.task.sql_task.alert +1031 resources.jobs.*.tasks[*].for_each_task.task.sql_task.alert.alert_id +1032 resources.jobs.*.tasks[*].for_each_task.task.sql_task.alert.pause_subscriptions +1033 resources.jobs.*.tasks[*].for_each_task.task.sql_task.alert.pause_subscriptions=true +1034 resources.jobs.*.tasks[*].for_each_task.task.sql_task.alert.pause_subscriptions=false +1035 resources.jobs.*.tasks[*].for_each_task.task.sql_task.alert.subscriptions +1036 resources.jobs.*.tasks[*].for_each_task.task.sql_task.alert.subscriptions[*] +1037 resources.jobs.*.tasks[*].for_each_task.task.sql_task.alert.subscriptions[*].destination_id +1038 resources.jobs.*.tasks[*].for_each_task.task.sql_task.alert.subscriptions[*].user_name +1039 resources.jobs.*.tasks[*].for_each_task.task.sql_task.dashboard +1040 resources.jobs.*.tasks[*].for_each_task.task.sql_task.dashboard.custom_subject +1041 resources.jobs.*.tasks[*].for_each_task.task.sql_task.dashboard.dashboard_id +1042 resources.jobs.*.tasks[*].for_each_task.task.sql_task.dashboard.pause_subscriptions +1043 resources.jobs.*.tasks[*].for_each_task.task.sql_task.dashboard.pause_subscriptions=true +1044 resources.jobs.*.tasks[*].for_each_task.task.sql_task.dashboard.pause_subscriptions=false +1045 resources.jobs.*.tasks[*].for_each_task.task.sql_task.dashboard.subscriptions +1046 resources.jobs.*.tasks[*].for_each_task.task.sql_task.dashboard.subscriptions[*] +1047 resources.jobs.*.tasks[*].for_each_task.task.sql_task.dashboard.subscriptions[*].destination_id +1048 resources.jobs.*.tasks[*].for_each_task.task.sql_task.dashboard.subscriptions[*].user_name +1049 resources.jobs.*.tasks[*].for_each_task.task.sql_task.file +1050 resources.jobs.*.tasks[*].for_each_task.task.sql_task.file.path +1051 resources.jobs.*.tasks[*].for_each_task.task.sql_task.file.source +1052 resources.jobs.*.tasks[*].for_each_task.task.sql_task.parameters +1053 resources.jobs.*.tasks[*].for_each_task.task.sql_task.parameters.* +1054 resources.jobs.*.tasks[*].for_each_task.task.sql_task.query +1055 resources.jobs.*.tasks[*].for_each_task.task.sql_task.query.query_id +1056 resources.jobs.*.tasks[*].for_each_task.task.sql_task.warehouse_id +1057 resources.jobs.*.tasks[*].for_each_task.task.task_key +1058 resources.jobs.*.tasks[*].for_each_task.task.timeout_seconds +1059 resources.jobs.*.tasks[*].for_each_task.task.webhook_notifications +1060 resources.jobs.*.tasks[*].for_each_task.task.webhook_notifications.on_duration_warning_threshold_exceeded +1061 resources.jobs.*.tasks[*].for_each_task.task.webhook_notifications.on_duration_warning_threshold_exceeded[*] +1062 resources.jobs.*.tasks[*].for_each_task.task.webhook_notifications.on_duration_warning_threshold_exceeded[*].id +1063 resources.jobs.*.tasks[*].for_each_task.task.webhook_notifications.on_failure +1064 resources.jobs.*.tasks[*].for_each_task.task.webhook_notifications.on_failure[*] +1065 resources.jobs.*.tasks[*].for_each_task.task.webhook_notifications.on_failure[*].id +1066 resources.jobs.*.tasks[*].for_each_task.task.webhook_notifications.on_maintenance_complete +1067 resources.jobs.*.tasks[*].for_each_task.task.webhook_notifications.on_maintenance_complete[*] +1068 resources.jobs.*.tasks[*].for_each_task.task.webhook_notifications.on_maintenance_complete[*].id +1069 resources.jobs.*.tasks[*].for_each_task.task.webhook_notifications.on_maintenance_start +1070 resources.jobs.*.tasks[*].for_each_task.task.webhook_notifications.on_maintenance_start[*] +1071 resources.jobs.*.tasks[*].for_each_task.task.webhook_notifications.on_maintenance_start[*].id +1072 resources.jobs.*.tasks[*].for_each_task.task.webhook_notifications.on_start +1073 resources.jobs.*.tasks[*].for_each_task.task.webhook_notifications.on_start[*] +1074 resources.jobs.*.tasks[*].for_each_task.task.webhook_notifications.on_start[*].id +1075 resources.jobs.*.tasks[*].for_each_task.task.webhook_notifications.on_streaming_backlog_exceeded +1076 resources.jobs.*.tasks[*].for_each_task.task.webhook_notifications.on_streaming_backlog_exceeded[*] +1077 resources.jobs.*.tasks[*].for_each_task.task.webhook_notifications.on_streaming_backlog_exceeded[*].id +1078 resources.jobs.*.tasks[*].for_each_task.task.webhook_notifications.on_success +1079 resources.jobs.*.tasks[*].for_each_task.task.webhook_notifications.on_success[*] +1080 resources.jobs.*.tasks[*].for_each_task.task.webhook_notifications.on_success[*].id +1081 resources.jobs.*.tasks[*].gen_ai_compute_task +1082 resources.jobs.*.tasks[*].gen_ai_compute_task.command +1083 resources.jobs.*.tasks[*].gen_ai_compute_task.compute +1084 resources.jobs.*.tasks[*].gen_ai_compute_task.compute.gpu_node_pool_id +1085 resources.jobs.*.tasks[*].gen_ai_compute_task.compute.gpu_type +1086 resources.jobs.*.tasks[*].gen_ai_compute_task.compute.num_gpus +1087 resources.jobs.*.tasks[*].gen_ai_compute_task.dl_runtime_image +1088 resources.jobs.*.tasks[*].gen_ai_compute_task.mlflow_experiment_name +1089 resources.jobs.*.tasks[*].gen_ai_compute_task.source +1090 resources.jobs.*.tasks[*].gen_ai_compute_task.training_script_path +1091 resources.jobs.*.tasks[*].gen_ai_compute_task.yaml_parameters +1092 resources.jobs.*.tasks[*].gen_ai_compute_task.yaml_parameters_file_path +1093 resources.jobs.*.tasks[*].health +1094 resources.jobs.*.tasks[*].health.rules +1095 resources.jobs.*.tasks[*].health.rules[*] +1096 resources.jobs.*.tasks[*].health.rules[*].metric +1097 resources.jobs.*.tasks[*].health.rules[*].op +1098 resources.jobs.*.tasks[*].health.rules[*].value +1099 resources.jobs.*.tasks[*].job_cluster_key +1100 resources.jobs.*.tasks[*].libraries +1101 resources.jobs.*.tasks[*].libraries[*] +1102 resources.jobs.*.tasks[*].libraries[*].cran +1103 resources.jobs.*.tasks[*].libraries[*].cran.package +1104 resources.jobs.*.tasks[*].libraries[*].cran.repo +1105 resources.jobs.*.tasks[*].libraries[*].egg +1106 resources.jobs.*.tasks[*].libraries[*].jar +1107 resources.jobs.*.tasks[*].libraries[*].maven +1108 resources.jobs.*.tasks[*].libraries[*].maven.coordinates +1109 resources.jobs.*.tasks[*].libraries[*].maven.exclusions +1110 resources.jobs.*.tasks[*].libraries[*].maven.exclusions[*] +1111 resources.jobs.*.tasks[*].libraries[*].maven.repo +1112 resources.jobs.*.tasks[*].libraries[*].pypi +1113 resources.jobs.*.tasks[*].libraries[*].pypi.package +1114 resources.jobs.*.tasks[*].libraries[*].pypi.repo +1115 resources.jobs.*.tasks[*].libraries[*].requirements +1116 resources.jobs.*.tasks[*].libraries[*].whl +1117 resources.jobs.*.tasks[*].max_retries +1118 resources.jobs.*.tasks[*].min_retry_interval_millis +1119 resources.jobs.*.tasks[*].new_cluster +1120 resources.jobs.*.tasks[*].new_cluster.apply_policy_default_values +1121 resources.jobs.*.tasks[*].new_cluster.apply_policy_default_values=true +1122 resources.jobs.*.tasks[*].new_cluster.apply_policy_default_values=false +1123 resources.jobs.*.tasks[*].new_cluster.autoscale +1124 resources.jobs.*.tasks[*].new_cluster.autoscale.max_workers +1125 resources.jobs.*.tasks[*].new_cluster.autoscale.min_workers +1126 resources.jobs.*.tasks[*].new_cluster.autotermination_minutes +1127 resources.jobs.*.tasks[*].new_cluster.aws_attributes +1128 resources.jobs.*.tasks[*].new_cluster.aws_attributes.availability +1129 resources.jobs.*.tasks[*].new_cluster.aws_attributes.ebs_volume_count +1130 resources.jobs.*.tasks[*].new_cluster.aws_attributes.ebs_volume_iops +1131 resources.jobs.*.tasks[*].new_cluster.aws_attributes.ebs_volume_size +1132 resources.jobs.*.tasks[*].new_cluster.aws_attributes.ebs_volume_throughput +1133 resources.jobs.*.tasks[*].new_cluster.aws_attributes.ebs_volume_type +1134 resources.jobs.*.tasks[*].new_cluster.aws_attributes.first_on_demand +1135 resources.jobs.*.tasks[*].new_cluster.aws_attributes.instance_profile_arn +1136 resources.jobs.*.tasks[*].new_cluster.aws_attributes.spot_bid_price_percent +1137 resources.jobs.*.tasks[*].new_cluster.aws_attributes.zone_id +1138 resources.jobs.*.tasks[*].new_cluster.azure_attributes +1139 resources.jobs.*.tasks[*].new_cluster.azure_attributes.availability +1140 resources.jobs.*.tasks[*].new_cluster.azure_attributes.capacity_reservation_group +1141 resources.jobs.*.tasks[*].new_cluster.azure_attributes.first_on_demand +1142 resources.jobs.*.tasks[*].new_cluster.azure_attributes.log_analytics_info +1143 resources.jobs.*.tasks[*].new_cluster.azure_attributes.log_analytics_info.log_analytics_primary_key +1144 resources.jobs.*.tasks[*].new_cluster.azure_attributes.log_analytics_info.log_analytics_workspace_id +1145 resources.jobs.*.tasks[*].new_cluster.azure_attributes.spot_bid_max_price +1146 resources.jobs.*.tasks[*].new_cluster.cluster_log_conf +1147 resources.jobs.*.tasks[*].new_cluster.cluster_log_conf.dbfs +1148 resources.jobs.*.tasks[*].new_cluster.cluster_log_conf.dbfs.destination +1149 resources.jobs.*.tasks[*].new_cluster.cluster_log_conf.s3 +1150 resources.jobs.*.tasks[*].new_cluster.cluster_log_conf.s3.canned_acl +1151 resources.jobs.*.tasks[*].new_cluster.cluster_log_conf.s3.destination +1152 resources.jobs.*.tasks[*].new_cluster.cluster_log_conf.s3.enable_encryption +1153 resources.jobs.*.tasks[*].new_cluster.cluster_log_conf.s3.enable_encryption=true +1154 resources.jobs.*.tasks[*].new_cluster.cluster_log_conf.s3.enable_encryption=false +1155 resources.jobs.*.tasks[*].new_cluster.cluster_log_conf.s3.encryption_type +1156 resources.jobs.*.tasks[*].new_cluster.cluster_log_conf.s3.endpoint +1157 resources.jobs.*.tasks[*].new_cluster.cluster_log_conf.s3.kms_key +1158 resources.jobs.*.tasks[*].new_cluster.cluster_log_conf.s3.region +1159 resources.jobs.*.tasks[*].new_cluster.cluster_log_conf.volumes +1160 resources.jobs.*.tasks[*].new_cluster.cluster_log_conf.volumes.destination +1161 resources.jobs.*.tasks[*].new_cluster.cluster_name +1162 resources.jobs.*.tasks[*].new_cluster.custom_tags +1163 resources.jobs.*.tasks[*].new_cluster.custom_tags.* +1164 resources.jobs.*.tasks[*].new_cluster.data_security_mode +1165 resources.jobs.*.tasks[*].new_cluster.dependency_mode +1166 resources.jobs.*.tasks[*].new_cluster.docker_image +1167 resources.jobs.*.tasks[*].new_cluster.docker_image.basic_auth +1168 resources.jobs.*.tasks[*].new_cluster.docker_image.basic_auth.password +1169 resources.jobs.*.tasks[*].new_cluster.docker_image.basic_auth.username +1170 resources.jobs.*.tasks[*].new_cluster.docker_image.url +1171 resources.jobs.*.tasks[*].new_cluster.driver_instance_pool_id +1172 resources.jobs.*.tasks[*].new_cluster.driver_node_type_flexibility +1173 resources.jobs.*.tasks[*].new_cluster.driver_node_type_flexibility.alternate_node_type_ids +1174 resources.jobs.*.tasks[*].new_cluster.driver_node_type_flexibility.alternate_node_type_ids[*] +1175 resources.jobs.*.tasks[*].new_cluster.driver_node_type_flexibility.aws_context_id +1176 resources.jobs.*.tasks[*].new_cluster.driver_node_type_id +1177 resources.jobs.*.tasks[*].new_cluster.enable_elastic_disk +1178 resources.jobs.*.tasks[*].new_cluster.enable_elastic_disk=true +1179 resources.jobs.*.tasks[*].new_cluster.enable_elastic_disk=false +1180 resources.jobs.*.tasks[*].new_cluster.enable_local_disk_encryption +1181 resources.jobs.*.tasks[*].new_cluster.enable_local_disk_encryption=true +1182 resources.jobs.*.tasks[*].new_cluster.enable_local_disk_encryption=false +1183 resources.jobs.*.tasks[*].new_cluster.gcp_attributes +1184 resources.jobs.*.tasks[*].new_cluster.gcp_attributes.availability +1185 resources.jobs.*.tasks[*].new_cluster.gcp_attributes.boot_disk_size +1186 resources.jobs.*.tasks[*].new_cluster.gcp_attributes.confidential_compute_type +1187 resources.jobs.*.tasks[*].new_cluster.gcp_attributes.first_on_demand +1188 resources.jobs.*.tasks[*].new_cluster.gcp_attributes.google_service_account +1189 resources.jobs.*.tasks[*].new_cluster.gcp_attributes.local_ssd_count +1190 resources.jobs.*.tasks[*].new_cluster.gcp_attributes.use_preemptible_executors +1191 resources.jobs.*.tasks[*].new_cluster.gcp_attributes.use_preemptible_executors=true +1192 resources.jobs.*.tasks[*].new_cluster.gcp_attributes.use_preemptible_executors=false +1193 resources.jobs.*.tasks[*].new_cluster.gcp_attributes.zone_id +1194 resources.jobs.*.tasks[*].new_cluster.init_scripts +1195 resources.jobs.*.tasks[*].new_cluster.init_scripts[*] +1196 resources.jobs.*.tasks[*].new_cluster.init_scripts[*].abfss +1197 resources.jobs.*.tasks[*].new_cluster.init_scripts[*].abfss.destination +1198 resources.jobs.*.tasks[*].new_cluster.init_scripts[*].dbfs +1199 resources.jobs.*.tasks[*].new_cluster.init_scripts[*].dbfs.destination +1200 resources.jobs.*.tasks[*].new_cluster.init_scripts[*].file +1201 resources.jobs.*.tasks[*].new_cluster.init_scripts[*].file.destination +1202 resources.jobs.*.tasks[*].new_cluster.init_scripts[*].gcs +1203 resources.jobs.*.tasks[*].new_cluster.init_scripts[*].gcs.destination +1204 resources.jobs.*.tasks[*].new_cluster.init_scripts[*].s3 +1205 resources.jobs.*.tasks[*].new_cluster.init_scripts[*].s3.canned_acl +1206 resources.jobs.*.tasks[*].new_cluster.init_scripts[*].s3.destination +1207 resources.jobs.*.tasks[*].new_cluster.init_scripts[*].s3.enable_encryption +1208 resources.jobs.*.tasks[*].new_cluster.init_scripts[*].s3.enable_encryption=true +1209 resources.jobs.*.tasks[*].new_cluster.init_scripts[*].s3.enable_encryption=false +1210 resources.jobs.*.tasks[*].new_cluster.init_scripts[*].s3.encryption_type +1211 resources.jobs.*.tasks[*].new_cluster.init_scripts[*].s3.endpoint +1212 resources.jobs.*.tasks[*].new_cluster.init_scripts[*].s3.kms_key +1213 resources.jobs.*.tasks[*].new_cluster.init_scripts[*].s3.region +1214 resources.jobs.*.tasks[*].new_cluster.init_scripts[*].volumes +1215 resources.jobs.*.tasks[*].new_cluster.init_scripts[*].volumes.destination +1216 resources.jobs.*.tasks[*].new_cluster.init_scripts[*].workspace +1217 resources.jobs.*.tasks[*].new_cluster.init_scripts[*].workspace.destination +1218 resources.jobs.*.tasks[*].new_cluster.instance_pool_id +1219 resources.jobs.*.tasks[*].new_cluster.is_single_node +1220 resources.jobs.*.tasks[*].new_cluster.is_single_node=true +1221 resources.jobs.*.tasks[*].new_cluster.is_single_node=false +1222 resources.jobs.*.tasks[*].new_cluster.kind +1223 resources.jobs.*.tasks[*].new_cluster.node_type_id +1224 resources.jobs.*.tasks[*].new_cluster.num_workers +1225 resources.jobs.*.tasks[*].new_cluster.policy_id +1226 resources.jobs.*.tasks[*].new_cluster.remote_disk_throughput +1227 resources.jobs.*.tasks[*].new_cluster.runtime_engine +1228 resources.jobs.*.tasks[*].new_cluster.single_user_name +1229 resources.jobs.*.tasks[*].new_cluster.spark_conf +1230 resources.jobs.*.tasks[*].new_cluster.spark_conf.* +1231 resources.jobs.*.tasks[*].new_cluster.spark_env_vars +1232 resources.jobs.*.tasks[*].new_cluster.spark_env_vars.* +1233 resources.jobs.*.tasks[*].new_cluster.spark_version +1234 resources.jobs.*.tasks[*].new_cluster.ssh_public_keys +1235 resources.jobs.*.tasks[*].new_cluster.ssh_public_keys[*] +1236 resources.jobs.*.tasks[*].new_cluster.total_initial_remote_disk_size +1237 resources.jobs.*.tasks[*].new_cluster.use_ml_runtime +1238 resources.jobs.*.tasks[*].new_cluster.use_ml_runtime=true +1239 resources.jobs.*.tasks[*].new_cluster.use_ml_runtime=false +1240 resources.jobs.*.tasks[*].new_cluster.worker_node_type_flexibility +1241 resources.jobs.*.tasks[*].new_cluster.worker_node_type_flexibility.alternate_node_type_ids +1242 resources.jobs.*.tasks[*].new_cluster.worker_node_type_flexibility.alternate_node_type_ids[*] +1243 resources.jobs.*.tasks[*].new_cluster.worker_node_type_flexibility.aws_context_id +1244 resources.jobs.*.tasks[*].new_cluster.workload_type +1245 resources.jobs.*.tasks[*].new_cluster.workload_type.clients +1246 resources.jobs.*.tasks[*].new_cluster.workload_type.clients.jobs +1247 resources.jobs.*.tasks[*].new_cluster.workload_type.clients.jobs=true +1248 resources.jobs.*.tasks[*].new_cluster.workload_type.clients.jobs=false +1249 resources.jobs.*.tasks[*].new_cluster.workload_type.clients.notebooks +1250 resources.jobs.*.tasks[*].new_cluster.workload_type.clients.notebooks=true +1251 resources.jobs.*.tasks[*].new_cluster.workload_type.clients.notebooks=false +1252 resources.jobs.*.tasks[*].notebook_task +1253 resources.jobs.*.tasks[*].notebook_task.base_parameters +1254 resources.jobs.*.tasks[*].notebook_task.base_parameters.* +1255 resources.jobs.*.tasks[*].notebook_task.notebook_path +1256 resources.jobs.*.tasks[*].notebook_task.source +1257 resources.jobs.*.tasks[*].notebook_task.warehouse_id +1258 resources.jobs.*.tasks[*].notification_settings +1259 resources.jobs.*.tasks[*].notification_settings.alert_on_last_attempt +1260 resources.jobs.*.tasks[*].notification_settings.alert_on_last_attempt=true +1261 resources.jobs.*.tasks[*].notification_settings.alert_on_last_attempt=false +1262 resources.jobs.*.tasks[*].notification_settings.no_alert_for_canceled_runs +1263 resources.jobs.*.tasks[*].notification_settings.no_alert_for_canceled_runs=true +1264 resources.jobs.*.tasks[*].notification_settings.no_alert_for_canceled_runs=false +1265 resources.jobs.*.tasks[*].notification_settings.no_alert_for_skipped_runs +1266 resources.jobs.*.tasks[*].notification_settings.no_alert_for_skipped_runs=true +1267 resources.jobs.*.tasks[*].notification_settings.no_alert_for_skipped_runs=false +1268 resources.jobs.*.tasks[*].pipeline_task +1269 resources.jobs.*.tasks[*].pipeline_task.full_refresh +1270 resources.jobs.*.tasks[*].pipeline_task.full_refresh=true +1271 resources.jobs.*.tasks[*].pipeline_task.full_refresh=false +1272 resources.jobs.*.tasks[*].pipeline_task.full_refresh_selection +1273 resources.jobs.*.tasks[*].pipeline_task.full_refresh_selection[*] +1274 resources.jobs.*.tasks[*].pipeline_task.parameters +1275 resources.jobs.*.tasks[*].pipeline_task.parameters.* +1276 resources.jobs.*.tasks[*].pipeline_task.pipeline_id +1277 resources.jobs.*.tasks[*].pipeline_task.refresh_flow_selection +1278 resources.jobs.*.tasks[*].pipeline_task.refresh_flow_selection[*] +1279 resources.jobs.*.tasks[*].pipeline_task.refresh_selection +1280 resources.jobs.*.tasks[*].pipeline_task.refresh_selection[*] +1281 resources.jobs.*.tasks[*].pipeline_task.reset_checkpoint_selection +1282 resources.jobs.*.tasks[*].pipeline_task.reset_checkpoint_selection[*] +1283 resources.jobs.*.tasks[*].power_bi_task +1284 resources.jobs.*.tasks[*].power_bi_task.connection_resource_name +1285 resources.jobs.*.tasks[*].power_bi_task.power_bi_model +1286 resources.jobs.*.tasks[*].power_bi_task.power_bi_model.authentication_method +1287 resources.jobs.*.tasks[*].power_bi_task.power_bi_model.model_name +1288 resources.jobs.*.tasks[*].power_bi_task.power_bi_model.overwrite_existing +1289 resources.jobs.*.tasks[*].power_bi_task.power_bi_model.overwrite_existing=true +1290 resources.jobs.*.tasks[*].power_bi_task.power_bi_model.overwrite_existing=false +1291 resources.jobs.*.tasks[*].power_bi_task.power_bi_model.storage_mode +1292 resources.jobs.*.tasks[*].power_bi_task.power_bi_model.workspace_name +1293 resources.jobs.*.tasks[*].power_bi_task.refresh_after_update +1294 resources.jobs.*.tasks[*].power_bi_task.refresh_after_update=true +1295 resources.jobs.*.tasks[*].power_bi_task.refresh_after_update=false +1296 resources.jobs.*.tasks[*].power_bi_task.tables +1297 resources.jobs.*.tasks[*].power_bi_task.tables[*] +1298 resources.jobs.*.tasks[*].power_bi_task.tables[*].catalog +1299 resources.jobs.*.tasks[*].power_bi_task.tables[*].name +1300 resources.jobs.*.tasks[*].power_bi_task.tables[*].schema +1301 resources.jobs.*.tasks[*].power_bi_task.tables[*].storage_mode +1302 resources.jobs.*.tasks[*].power_bi_task.warehouse_id +1303 resources.jobs.*.tasks[*].python_operator_task +1304 resources.jobs.*.tasks[*].python_operator_task.main +1305 resources.jobs.*.tasks[*].python_operator_task.parameters +1306 resources.jobs.*.tasks[*].python_operator_task.parameters[*] +1307 resources.jobs.*.tasks[*].python_operator_task.parameters[*].name +1308 resources.jobs.*.tasks[*].python_operator_task.parameters[*].value +1309 resources.jobs.*.tasks[*].python_wheel_task +1310 resources.jobs.*.tasks[*].python_wheel_task.entry_point +1311 resources.jobs.*.tasks[*].python_wheel_task.named_parameters +1312 resources.jobs.*.tasks[*].python_wheel_task.named_parameters.* +1313 resources.jobs.*.tasks[*].python_wheel_task.package_name +1314 resources.jobs.*.tasks[*].python_wheel_task.parameters +1315 resources.jobs.*.tasks[*].python_wheel_task.parameters[*] +1316 resources.jobs.*.tasks[*].retry_on_timeout +1317 resources.jobs.*.tasks[*].retry_on_timeout=true +1318 resources.jobs.*.tasks[*].retry_on_timeout=false +1319 resources.jobs.*.tasks[*].run_if +1320 resources.jobs.*.tasks[*].run_job_task +1321 resources.jobs.*.tasks[*].run_job_task.dbt_commands +1322 resources.jobs.*.tasks[*].run_job_task.dbt_commands[*] +1323 resources.jobs.*.tasks[*].run_job_task.jar_params +1324 resources.jobs.*.tasks[*].run_job_task.jar_params[*] +1325 resources.jobs.*.tasks[*].run_job_task.job_id +1326 resources.jobs.*.tasks[*].run_job_task.job_parameters +1327 resources.jobs.*.tasks[*].run_job_task.job_parameters.* +1328 resources.jobs.*.tasks[*].run_job_task.notebook_params +1329 resources.jobs.*.tasks[*].run_job_task.notebook_params.* +1330 resources.jobs.*.tasks[*].run_job_task.pipeline_params +1331 resources.jobs.*.tasks[*].run_job_task.pipeline_params.full_refresh +1332 resources.jobs.*.tasks[*].run_job_task.pipeline_params.full_refresh=true +1333 resources.jobs.*.tasks[*].run_job_task.pipeline_params.full_refresh=false +1334 resources.jobs.*.tasks[*].run_job_task.pipeline_params.full_refresh_selection +1335 resources.jobs.*.tasks[*].run_job_task.pipeline_params.full_refresh_selection[*] +1336 resources.jobs.*.tasks[*].run_job_task.pipeline_params.refresh_flow_selection +1337 resources.jobs.*.tasks[*].run_job_task.pipeline_params.refresh_flow_selection[*] +1338 resources.jobs.*.tasks[*].run_job_task.pipeline_params.refresh_selection +1339 resources.jobs.*.tasks[*].run_job_task.pipeline_params.refresh_selection[*] +1340 resources.jobs.*.tasks[*].run_job_task.pipeline_params.reset_checkpoint_selection +1341 resources.jobs.*.tasks[*].run_job_task.pipeline_params.reset_checkpoint_selection[*] +1342 resources.jobs.*.tasks[*].run_job_task.python_named_params +1343 resources.jobs.*.tasks[*].run_job_task.python_named_params.* +1344 resources.jobs.*.tasks[*].run_job_task.python_params +1345 resources.jobs.*.tasks[*].run_job_task.python_params[*] +1346 resources.jobs.*.tasks[*].run_job_task.spark_submit_params +1347 resources.jobs.*.tasks[*].run_job_task.spark_submit_params[*] +1348 resources.jobs.*.tasks[*].run_job_task.sql_params +1349 resources.jobs.*.tasks[*].run_job_task.sql_params.* +1350 resources.jobs.*.tasks[*].spark_jar_task +1351 resources.jobs.*.tasks[*].spark_jar_task.jar_uri +1352 resources.jobs.*.tasks[*].spark_jar_task.main_class_name +1353 resources.jobs.*.tasks[*].spark_jar_task.parameters +1354 resources.jobs.*.tasks[*].spark_jar_task.parameters[*] +1355 resources.jobs.*.tasks[*].spark_jar_task.run_as_repl +1356 resources.jobs.*.tasks[*].spark_jar_task.run_as_repl=true +1357 resources.jobs.*.tasks[*].spark_jar_task.run_as_repl=false +1358 resources.jobs.*.tasks[*].spark_python_task +1359 resources.jobs.*.tasks[*].spark_python_task.parameters +1360 resources.jobs.*.tasks[*].spark_python_task.parameters[*] +1361 resources.jobs.*.tasks[*].spark_python_task.python_file +1362 resources.jobs.*.tasks[*].spark_python_task.source +1363 resources.jobs.*.tasks[*].spark_submit_task +1364 resources.jobs.*.tasks[*].spark_submit_task.parameters +1365 resources.jobs.*.tasks[*].spark_submit_task.parameters[*] +1366 resources.jobs.*.tasks[*].sql_task +1367 resources.jobs.*.tasks[*].sql_task.alert +1368 resources.jobs.*.tasks[*].sql_task.alert.alert_id +1369 resources.jobs.*.tasks[*].sql_task.alert.pause_subscriptions +1370 resources.jobs.*.tasks[*].sql_task.alert.pause_subscriptions=true +1371 resources.jobs.*.tasks[*].sql_task.alert.pause_subscriptions=false +1372 resources.jobs.*.tasks[*].sql_task.alert.subscriptions +1373 resources.jobs.*.tasks[*].sql_task.alert.subscriptions[*] +1374 resources.jobs.*.tasks[*].sql_task.alert.subscriptions[*].destination_id +1375 resources.jobs.*.tasks[*].sql_task.alert.subscriptions[*].user_name +1376 resources.jobs.*.tasks[*].sql_task.dashboard +1377 resources.jobs.*.tasks[*].sql_task.dashboard.custom_subject +1378 resources.jobs.*.tasks[*].sql_task.dashboard.dashboard_id +1379 resources.jobs.*.tasks[*].sql_task.dashboard.pause_subscriptions +1380 resources.jobs.*.tasks[*].sql_task.dashboard.pause_subscriptions=true +1381 resources.jobs.*.tasks[*].sql_task.dashboard.pause_subscriptions=false +1382 resources.jobs.*.tasks[*].sql_task.dashboard.subscriptions +1383 resources.jobs.*.tasks[*].sql_task.dashboard.subscriptions[*] +1384 resources.jobs.*.tasks[*].sql_task.dashboard.subscriptions[*].destination_id +1385 resources.jobs.*.tasks[*].sql_task.dashboard.subscriptions[*].user_name +1386 resources.jobs.*.tasks[*].sql_task.file +1387 resources.jobs.*.tasks[*].sql_task.file.path +1388 resources.jobs.*.tasks[*].sql_task.file.source +1389 resources.jobs.*.tasks[*].sql_task.parameters +1390 resources.jobs.*.tasks[*].sql_task.parameters.* +1391 resources.jobs.*.tasks[*].sql_task.query +1392 resources.jobs.*.tasks[*].sql_task.query.query_id +1393 resources.jobs.*.tasks[*].sql_task.warehouse_id +1394 resources.jobs.*.tasks[*].task_key +1395 resources.jobs.*.tasks[*].timeout_seconds +1396 resources.jobs.*.tasks[*].webhook_notifications +1397 resources.jobs.*.tasks[*].webhook_notifications.on_duration_warning_threshold_exceeded +1398 resources.jobs.*.tasks[*].webhook_notifications.on_duration_warning_threshold_exceeded[*] +1399 resources.jobs.*.tasks[*].webhook_notifications.on_duration_warning_threshold_exceeded[*].id +1400 resources.jobs.*.tasks[*].webhook_notifications.on_failure +1401 resources.jobs.*.tasks[*].webhook_notifications.on_failure[*] +1402 resources.jobs.*.tasks[*].webhook_notifications.on_failure[*].id +1403 resources.jobs.*.tasks[*].webhook_notifications.on_maintenance_complete +1404 resources.jobs.*.tasks[*].webhook_notifications.on_maintenance_complete[*] +1405 resources.jobs.*.tasks[*].webhook_notifications.on_maintenance_complete[*].id +1406 resources.jobs.*.tasks[*].webhook_notifications.on_maintenance_start +1407 resources.jobs.*.tasks[*].webhook_notifications.on_maintenance_start[*] +1408 resources.jobs.*.tasks[*].webhook_notifications.on_maintenance_start[*].id +1409 resources.jobs.*.tasks[*].webhook_notifications.on_start +1410 resources.jobs.*.tasks[*].webhook_notifications.on_start[*] +1411 resources.jobs.*.tasks[*].webhook_notifications.on_start[*].id +1412 resources.jobs.*.tasks[*].webhook_notifications.on_streaming_backlog_exceeded +1413 resources.jobs.*.tasks[*].webhook_notifications.on_streaming_backlog_exceeded[*] +1414 resources.jobs.*.tasks[*].webhook_notifications.on_streaming_backlog_exceeded[*].id +1415 resources.jobs.*.tasks[*].webhook_notifications.on_success +1416 resources.jobs.*.tasks[*].webhook_notifications.on_success[*] +1417 resources.jobs.*.tasks[*].webhook_notifications.on_success[*].id +1418 resources.jobs.*.timeout_seconds +1419 resources.jobs.*.trigger +1420 resources.jobs.*.trigger.file_arrival +1421 resources.jobs.*.trigger.file_arrival.min_time_between_triggers_seconds +1422 resources.jobs.*.trigger.file_arrival.url +1423 resources.jobs.*.trigger.file_arrival.wait_after_last_change_seconds +1424 resources.jobs.*.trigger.job_completion +1425 resources.jobs.*.trigger.job_completion.if_status +1426 resources.jobs.*.trigger.job_completion.jobs +1427 resources.jobs.*.trigger.job_completion.jobs[*] +1428 resources.jobs.*.trigger.job_completion.jobs[*].job_id +1429 resources.jobs.*.trigger.job_completion.jobs[*].workspace_id +1430 resources.jobs.*.trigger.model +1431 resources.jobs.*.trigger.model.aliases +1432 resources.jobs.*.trigger.model.aliases[*] +1433 resources.jobs.*.trigger.model.condition +1434 resources.jobs.*.trigger.model.min_time_between_triggers_seconds +1435 resources.jobs.*.trigger.model.securable_name +1436 resources.jobs.*.trigger.model.wait_after_last_change_seconds +1437 resources.jobs.*.trigger.pause_status +1438 resources.jobs.*.trigger.periodic +1439 resources.jobs.*.trigger.periodic.interval +1440 resources.jobs.*.trigger.periodic.unit +1441 resources.jobs.*.trigger.sql_condition +1442 resources.jobs.*.trigger.sql_condition.sql_query_id +1443 resources.jobs.*.trigger.sql_condition.trigger_mode +1444 resources.jobs.*.trigger.sql_condition.warehouse_id +1445 resources.jobs.*.trigger.table_update +1446 resources.jobs.*.trigger.table_update.condition +1447 resources.jobs.*.trigger.table_update.min_time_between_triggers_seconds +1448 resources.jobs.*.trigger.table_update.table_names +1449 resources.jobs.*.trigger.table_update.table_names[*] +1450 resources.jobs.*.trigger.table_update.wait_after_last_change_seconds +1451 resources.jobs.*.triggers +1452 resources.jobs.*.triggers[*] +1453 resources.jobs.*.triggers[*].continuous +1454 resources.jobs.*.triggers[*].continuous.maintenance_window +1455 resources.jobs.*.triggers[*].continuous.maintenance_window.day_of_week +1456 resources.jobs.*.triggers[*].continuous.maintenance_window.start_hour +1457 resources.jobs.*.triggers[*].continuous.maintenance_window.timezone_id +1458 resources.jobs.*.triggers[*].continuous.task_retry_mode +1459 resources.jobs.*.triggers[*].file_arrival +1460 resources.jobs.*.triggers[*].file_arrival.min_time_between_triggers_seconds +1461 resources.jobs.*.triggers[*].file_arrival.url +1462 resources.jobs.*.triggers[*].file_arrival.wait_after_last_change_seconds +1463 resources.jobs.*.triggers[*].job_completion +1464 resources.jobs.*.triggers[*].job_completion.if_status +1465 resources.jobs.*.triggers[*].job_completion.jobs +1466 resources.jobs.*.triggers[*].job_completion.jobs[*] +1467 resources.jobs.*.triggers[*].job_completion.jobs[*].job_id +1468 resources.jobs.*.triggers[*].job_completion.jobs[*].workspace_id +1469 resources.jobs.*.triggers[*].model +1470 resources.jobs.*.triggers[*].model.aliases +1471 resources.jobs.*.triggers[*].model.aliases[*] +1472 resources.jobs.*.triggers[*].model.condition +1473 resources.jobs.*.triggers[*].model.min_time_between_triggers_seconds +1474 resources.jobs.*.triggers[*].model.securable_name +1475 resources.jobs.*.triggers[*].model.wait_after_last_change_seconds +1476 resources.jobs.*.triggers[*].pause_status +1477 resources.jobs.*.triggers[*].periodic +1478 resources.jobs.*.triggers[*].periodic.interval +1479 resources.jobs.*.triggers[*].periodic.unit +1480 resources.jobs.*.triggers[*].schedule +1481 resources.jobs.*.triggers[*].schedule.quartz_cron_expression +1482 resources.jobs.*.triggers[*].schedule.timezone_id +1483 resources.jobs.*.triggers[*].sql_condition +1484 resources.jobs.*.triggers[*].sql_condition.sql_query_id +1485 resources.jobs.*.triggers[*].sql_condition.trigger_mode +1486 resources.jobs.*.triggers[*].sql_condition.warehouse_id +1487 resources.jobs.*.triggers[*].table_update +1488 resources.jobs.*.triggers[*].table_update.condition +1489 resources.jobs.*.triggers[*].table_update.min_time_between_triggers_seconds +1490 resources.jobs.*.triggers[*].table_update.table_names +1491 resources.jobs.*.triggers[*].table_update.table_names[*] +1492 resources.jobs.*.triggers[*].table_update.wait_after_last_change_seconds +1493 resources.jobs.*.usage_policy_id +1494 resources.jobs.*.webhook_notifications +1495 resources.jobs.*.webhook_notifications.on_duration_warning_threshold_exceeded +1496 resources.jobs.*.webhook_notifications.on_duration_warning_threshold_exceeded[*] +1497 resources.jobs.*.webhook_notifications.on_duration_warning_threshold_exceeded[*].id +1498 resources.jobs.*.webhook_notifications.on_failure +1499 resources.jobs.*.webhook_notifications.on_failure[*] +1500 resources.jobs.*.webhook_notifications.on_failure[*].id +1501 resources.jobs.*.webhook_notifications.on_maintenance_complete +1502 resources.jobs.*.webhook_notifications.on_maintenance_complete[*] +1503 resources.jobs.*.webhook_notifications.on_maintenance_complete[*].id +1504 resources.jobs.*.webhook_notifications.on_maintenance_start +1505 resources.jobs.*.webhook_notifications.on_maintenance_start[*] +1506 resources.jobs.*.webhook_notifications.on_maintenance_start[*].id +1507 resources.jobs.*.webhook_notifications.on_start +1508 resources.jobs.*.webhook_notifications.on_start[*] +1509 resources.jobs.*.webhook_notifications.on_start[*].id +1510 resources.jobs.*.webhook_notifications.on_streaming_backlog_exceeded +1511 resources.jobs.*.webhook_notifications.on_streaming_backlog_exceeded[*] +1512 resources.jobs.*.webhook_notifications.on_streaming_backlog_exceeded[*].id +1513 resources.jobs.*.webhook_notifications.on_success +1514 resources.jobs.*.webhook_notifications.on_success[*] +1515 resources.jobs.*.webhook_notifications.on_success[*].id +1516 resources.jobs.*.permissions +1517 resources.jobs.*.permissions[*] +1518 resources.jobs.*.permissions[*].level +1519 resources.jobs.*.permissions[*].user_name +1520 resources.jobs.*.permissions[*].service_principal_name +1521 resources.jobs.*.permissions[*].group_name diff --git a/acceptance/bundle/bitmap/full/output/materialization_sets.txt b/acceptance/bundle/bitmap/full/output/materialization_sets.txt new file mode 100644 index 00000000000..f437d201f80 --- /dev/null +++ b/acceptance/bundle/bitmap/full/output/materialization_sets.txt @@ -0,0 +1,87 @@ +2757 resources.materialization_sets +2758 resources.materialization_sets.* +2759 resources.materialization_sets.*.url +2760 resources.materialization_sets.*.modified_status +2761 resources.materialization_sets.*.lifecycle +2762 resources.materialization_sets.*.lifecycle.prevent_destroy +2763 resources.materialization_sets.*.lifecycle.prevent_destroy=true +2764 resources.materialization_sets.*.lifecycle.prevent_destroy=false +2765 resources.materialization_sets.*.lifecycle.depends_on +2766 resources.materialization_sets.*.lifecycle.depends_on[*] +2767 resources.materialization_sets.*.lifecycle.depends_on[*].resource +2768 resources.materialization_sets.*.id +2769 resources.materialization_sets.*.materialization_set_id +2770 resources.materialization_sets.*.materialized_features +2771 resources.materialization_sets.*.materialized_features[*] +2772 resources.materialization_sets.*.materialized_features[*].budget_policy_id +2773 resources.materialization_sets.*.materialized_features[*].cron_schedule +2774 resources.materialization_sets.*.materialized_features[*].cron_schedule_trigger +2775 resources.materialization_sets.*.materialized_features[*].cron_schedule_trigger.cron_expression +2776 resources.materialization_sets.*.materialized_features[*].cron_schedule_trigger.mode +2777 resources.materialization_sets.*.materialized_features[*].cron_schedule_trigger.paused +2778 resources.materialization_sets.*.materialized_features[*].cron_schedule_trigger.paused=true +2779 resources.materialization_sets.*.materialized_features[*].cron_schedule_trigger.paused=false +2780 resources.materialization_sets.*.materialized_features[*].cron_schedule_trigger.timezone_id +2781 resources.materialization_sets.*.materialized_features[*].external_trigger +2782 resources.materialization_sets.*.materialized_features[*].feature_name +2783 resources.materialization_sets.*.materialized_features[*].is_online +2784 resources.materialization_sets.*.materialized_features[*].is_online=true +2785 resources.materialization_sets.*.materialized_features[*].is_online=false +2786 resources.materialization_sets.*.materialized_features[*].job_id +2787 resources.materialization_sets.*.materialized_features[*].last_materialization_time +2788 resources.materialization_sets.*.materialized_features[*].latest_backfill_operation +2789 resources.materialization_sets.*.materialized_features[*].materialization_set_id +2790 resources.materialization_sets.*.materialized_features[*].materialized_feature_id +2791 resources.materialization_sets.*.materialized_features[*].notifications +2792 resources.materialization_sets.*.materialized_features[*].notifications.on_failure +2793 resources.materialization_sets.*.materialized_features[*].notifications.on_failure.destination_ids +2794 resources.materialization_sets.*.materialized_features[*].notifications.on_failure.destination_ids[*] +2795 resources.materialization_sets.*.materialized_features[*].notifications.on_failure.email_addresses +2796 resources.materialization_sets.*.materialized_features[*].notifications.on_failure.email_addresses[*] +2797 resources.materialization_sets.*.materialized_features[*].notifications.on_failure.final_attempt_only +2798 resources.materialization_sets.*.materialized_features[*].notifications.on_failure.final_attempt_only=true +2799 resources.materialization_sets.*.materialized_features[*].notifications.on_failure.final_attempt_only=false +2800 resources.materialization_sets.*.materialized_features[*].notifications.on_success +2801 resources.materialization_sets.*.materialized_features[*].notifications.on_success.destination_ids +2802 resources.materialization_sets.*.materialized_features[*].notifications.on_success.destination_ids[*] +2803 resources.materialization_sets.*.materialized_features[*].notifications.on_success.email_addresses +2804 resources.materialization_sets.*.materialized_features[*].notifications.on_success.email_addresses[*] +2805 resources.materialization_sets.*.materialized_features[*].offline_store_config +2806 resources.materialization_sets.*.materialized_features[*].offline_store_config.catalog_name +2807 resources.materialization_sets.*.materialized_features[*].offline_store_config.schema_name +2808 resources.materialization_sets.*.materialized_features[*].offline_store_config.table_name_prefix +2809 resources.materialization_sets.*.materialized_features[*].online_store_config +2810 resources.materialization_sets.*.materialized_features[*].online_store_config.catalog_name +2811 resources.materialization_sets.*.materialized_features[*].online_store_config.online_store_name +2812 resources.materialization_sets.*.materialized_features[*].online_store_config.schema_name +2813 resources.materialization_sets.*.materialized_features[*].online_store_config.table_name_prefix +2814 resources.materialization_sets.*.materialized_features[*].pipeline_id +2815 resources.materialization_sets.*.materialized_features[*].pipeline_schedule_state +2816 resources.materialization_sets.*.materialized_features[*].streaming_mode +2817 resources.materialization_sets.*.materialized_features[*].streaming_mode.freshness_target +2818 resources.materialization_sets.*.materialized_features[*].streaming_mode.mode +2819 resources.materialization_sets.*.materialized_features[*].streaming_mode.shuffle_partitions +2820 resources.materialization_sets.*.materialized_features[*].table_name +2821 resources.materialization_sets.*.materialized_features[*].table_trigger +2822 resources.materialization_sets.*.materialized_features[*].table_trigger.condition +2823 resources.materialization_sets.*.materialized_features[*].table_trigger.min_time_between_triggers +2824 resources.materialization_sets.*.materialized_features[*].table_trigger.paused +2825 resources.materialization_sets.*.materialized_features[*].table_trigger.paused=true +2826 resources.materialization_sets.*.materialized_features[*].table_trigger.paused=false +2827 resources.materialization_sets.*.materialized_features[*].table_trigger.wait_after_last_change +2828 resources.materialization_sets.*.materialized_features[*].tags +2829 resources.materialization_sets.*.materialized_features[*].tags.* +2830 resources.materialization_sets.*.notifications +2831 resources.materialization_sets.*.notifications.on_failure +2832 resources.materialization_sets.*.notifications.on_failure.destination_ids +2833 resources.materialization_sets.*.notifications.on_failure.destination_ids[*] +2834 resources.materialization_sets.*.notifications.on_failure.email_addresses +2835 resources.materialization_sets.*.notifications.on_failure.email_addresses[*] +2836 resources.materialization_sets.*.notifications.on_failure.final_attempt_only +2837 resources.materialization_sets.*.notifications.on_failure.final_attempt_only=true +2838 resources.materialization_sets.*.notifications.on_failure.final_attempt_only=false +2839 resources.materialization_sets.*.notifications.on_success +2840 resources.materialization_sets.*.notifications.on_success.destination_ids +2841 resources.materialization_sets.*.notifications.on_success.destination_ids[*] +2842 resources.materialization_sets.*.notifications.on_success.email_addresses +2843 resources.materialization_sets.*.notifications.on_success.email_addresses[*] diff --git a/acceptance/bundle/bitmap/full/output/mcp_services.txt b/acceptance/bundle/bitmap/full/output/mcp_services.txt new file mode 100644 index 00000000000..ecfcab2c387 --- /dev/null +++ b/acceptance/bundle/bitmap/full/output/mcp_services.txt @@ -0,0 +1,37 @@ +3232 resources.mcp_services +3233 resources.mcp_services.* +3234 resources.mcp_services.*.url +3235 resources.mcp_services.*.modified_status +3236 resources.mcp_services.*.lifecycle +3237 resources.mcp_services.*.lifecycle.prevent_destroy +3238 resources.mcp_services.*.lifecycle.prevent_destroy=true +3239 resources.mcp_services.*.lifecycle.prevent_destroy=false +3240 resources.mcp_services.*.lifecycle.depends_on +3241 resources.mcp_services.*.lifecycle.depends_on[*] +3242 resources.mcp_services.*.lifecycle.depends_on[*].resource +3243 resources.mcp_services.*.id +3244 resources.mcp_services.*.parent +3245 resources.mcp_services.*.mcp_service_id +3246 resources.mcp_services.*.comment +3247 resources.mcp_services.*.config +3248 resources.mcp_services.*.config.include_tool_selectors +3249 resources.mcp_services.*.config.include_tool_selectors[*] +3250 resources.mcp_services.*.config.rate_limits +3251 resources.mcp_services.*.config.rate_limits[*] +3252 resources.mcp_services.*.config.rate_limits[*].key +3253 resources.mcp_services.*.config.rate_limits[*].principal +3254 resources.mcp_services.*.config.rate_limits[*].renewal_period +3255 resources.mcp_services.*.config.rate_limits[*].requests +3256 resources.mcp_services.*.config.rate_limits[*].tokens +3257 resources.mcp_services.*.config.source_connection +3258 resources.mcp_services.*.config.source_connection.is_deleted +3259 resources.mcp_services.*.config.source_connection.is_deleted=true +3260 resources.mcp_services.*.config.source_connection.is_deleted=false +3261 resources.mcp_services.*.config.source_connection.name +3262 resources.mcp_services.*.config.source_connection.options +3263 resources.mcp_services.*.config.source_connection.options.* +3264 resources.mcp_services.*.grants +3265 resources.mcp_services.*.grants[*] +3266 resources.mcp_services.*.grants[*].principal +3267 resources.mcp_services.*.grants[*].privileges +3268 resources.mcp_services.*.grants[*].privileges[*] diff --git a/acceptance/bundle/bitmap/full/output/model_provider_services.txt b/acceptance/bundle/bitmap/full/output/model_provider_services.txt new file mode 100644 index 00000000000..7b6fca48f79 --- /dev/null +++ b/acceptance/bundle/bitmap/full/output/model_provider_services.txt @@ -0,0 +1,148 @@ +3269 resources.model_provider_services +3270 resources.model_provider_services.* +3271 resources.model_provider_services.*.url +3272 resources.model_provider_services.*.modified_status +3273 resources.model_provider_services.*.lifecycle +3274 resources.model_provider_services.*.lifecycle.prevent_destroy +3275 resources.model_provider_services.*.lifecycle.prevent_destroy=true +3276 resources.model_provider_services.*.lifecycle.prevent_destroy=false +3277 resources.model_provider_services.*.lifecycle.depends_on +3278 resources.model_provider_services.*.lifecycle.depends_on[*] +3279 resources.model_provider_services.*.lifecycle.depends_on[*].resource +3280 resources.model_provider_services.*.id +3281 resources.model_provider_services.*.parent +3282 resources.model_provider_services.*.model_provider_service_id +3283 resources.model_provider_services.*.comment +3284 resources.model_provider_services.*.config +3285 resources.model_provider_services.*.config.allow_all_targets +3286 resources.model_provider_services.*.config.allow_all_targets=true +3287 resources.model_provider_services.*.config.allow_all_targets=false +3288 resources.model_provider_services.*.config.amazon_bedrock +3289 resources.model_provider_services.*.config.amazon_bedrock.direct +3290 resources.model_provider_services.*.config.amazon_bedrock.direct.aws_access_key +3291 resources.model_provider_services.*.config.amazon_bedrock.direct.aws_access_key.access_key_id +3292 resources.model_provider_services.*.config.amazon_bedrock.direct.aws_access_key.secret_access_key +3293 resources.model_provider_services.*.config.amazon_bedrock.direct.aws_access_key.secret_access_key.plaintext +3294 resources.model_provider_services.*.config.amazon_bedrock.direct.aws_access_key.secret_access_key.secret_reference +3295 resources.model_provider_services.*.config.amazon_bedrock.direct.aws_access_key.secret_access_key.secret_reference.name +3296 resources.model_provider_services.*.config.amazon_bedrock.direct.region +3297 resources.model_provider_services.*.config.amazon_bedrock.direct.service_credential +3298 resources.model_provider_services.*.config.amazon_bedrock.direct.service_credential.name +3299 resources.model_provider_services.*.config.anthropic +3300 resources.model_provider_services.*.config.anthropic.direct +3301 resources.model_provider_services.*.config.anthropic.direct.api_key +3302 resources.model_provider_services.*.config.anthropic.direct.api_key.plaintext +3303 resources.model_provider_services.*.config.anthropic.direct.api_key.secret_reference +3304 resources.model_provider_services.*.config.anthropic.direct.api_key.secret_reference.name +3305 resources.model_provider_services.*.config.anthropic.relayed +3306 resources.model_provider_services.*.config.azure_openai +3307 resources.model_provider_services.*.config.azure_openai.direct +3308 resources.model_provider_services.*.config.azure_openai.direct.api_key +3309 resources.model_provider_services.*.config.azure_openai.direct.api_key.plaintext +3310 resources.model_provider_services.*.config.azure_openai.direct.api_key.secret_reference +3311 resources.model_provider_services.*.config.azure_openai.direct.api_key.secret_reference.name +3312 resources.model_provider_services.*.config.azure_openai.direct.base_url +3313 resources.model_provider_services.*.config.azure_openai.direct.entra_service_principal +3314 resources.model_provider_services.*.config.azure_openai.direct.entra_service_principal.client_id +3315 resources.model_provider_services.*.config.azure_openai.direct.entra_service_principal.client_secret +3316 resources.model_provider_services.*.config.azure_openai.direct.entra_service_principal.client_secret.plaintext +3317 resources.model_provider_services.*.config.azure_openai.direct.entra_service_principal.client_secret.secret_reference +3318 resources.model_provider_services.*.config.azure_openai.direct.entra_service_principal.client_secret.secret_reference.name +3319 resources.model_provider_services.*.config.azure_openai.direct.entra_service_principal.tenant_id +3320 resources.model_provider_services.*.config.azure_openai.direct.service_credential +3321 resources.model_provider_services.*.config.azure_openai.direct.service_credential.name +3322 resources.model_provider_services.*.config.bedrock_mantle +3323 resources.model_provider_services.*.config.bedrock_mantle.direct +3324 resources.model_provider_services.*.config.bedrock_mantle.direct.aws_access_key +3325 resources.model_provider_services.*.config.bedrock_mantle.direct.aws_access_key.access_key_id +3326 resources.model_provider_services.*.config.bedrock_mantle.direct.aws_access_key.secret_access_key +3327 resources.model_provider_services.*.config.bedrock_mantle.direct.aws_access_key.secret_access_key.plaintext +3328 resources.model_provider_services.*.config.bedrock_mantle.direct.aws_access_key.secret_access_key.secret_reference +3329 resources.model_provider_services.*.config.bedrock_mantle.direct.aws_access_key.secret_access_key.secret_reference.name +3330 resources.model_provider_services.*.config.bedrock_mantle.direct.region +3331 resources.model_provider_services.*.config.bedrock_mantle.direct.service_credential +3332 resources.model_provider_services.*.config.bedrock_mantle.direct.service_credential.name +3333 resources.model_provider_services.*.config.custom +3334 resources.model_provider_services.*.config.custom.direct +3335 resources.model_provider_services.*.config.custom.direct.api_key +3336 resources.model_provider_services.*.config.custom.direct.api_key.plaintext +3337 resources.model_provider_services.*.config.custom.direct.api_key.secret_reference +3338 resources.model_provider_services.*.config.custom.direct.api_key.secret_reference.name +3339 resources.model_provider_services.*.config.custom.direct.base_url +3340 resources.model_provider_services.*.config.custom.direct.header_auth +3341 resources.model_provider_services.*.config.custom.direct.header_auth.api_key_name +3342 resources.model_provider_services.*.config.custom.direct.header_auth.api_key_value +3343 resources.model_provider_services.*.config.custom.direct.header_auth.api_key_value.plaintext +3344 resources.model_provider_services.*.config.custom.direct.header_auth.api_key_value.secret_reference +3345 resources.model_provider_services.*.config.custom.direct.header_auth.api_key_value.secret_reference.name +3346 resources.model_provider_services.*.config.forward_headers +3347 resources.model_provider_services.*.config.forward_headers=true +3348 resources.model_provider_services.*.config.forward_headers=false +3349 resources.model_provider_services.*.config.forward_query_parameters +3350 resources.model_provider_services.*.config.forward_query_parameters=true +3351 resources.model_provider_services.*.config.forward_query_parameters=false +3352 resources.model_provider_services.*.config.forward_unmanaged_paths +3353 resources.model_provider_services.*.config.forward_unmanaged_paths=true +3354 resources.model_provider_services.*.config.forward_unmanaged_paths=false +3355 resources.model_provider_services.*.config.gemini_enterprise +3356 resources.model_provider_services.*.config.gemini_enterprise.direct +3357 resources.model_provider_services.*.config.gemini_enterprise.direct.api_key +3358 resources.model_provider_services.*.config.gemini_enterprise.direct.api_key.plaintext +3359 resources.model_provider_services.*.config.gemini_enterprise.direct.api_key.secret_reference +3360 resources.model_provider_services.*.config.gemini_enterprise.direct.api_key.secret_reference.name +3361 resources.model_provider_services.*.config.gemini_enterprise.direct.project_id +3362 resources.model_provider_services.*.config.gemini_enterprise.direct.region +3363 resources.model_provider_services.*.config.gemini_enterprise.direct.service_credential +3364 resources.model_provider_services.*.config.gemini_enterprise.direct.service_credential.name +3365 resources.model_provider_services.*.config.inference_table +3366 resources.model_provider_services.*.config.inference_table.is_deleted +3367 resources.model_provider_services.*.config.inference_table.is_deleted=true +3368 resources.model_provider_services.*.config.inference_table.is_deleted=false +3369 resources.model_provider_services.*.config.inference_table.parent +3370 resources.model_provider_services.*.config.inference_table.table +3371 resources.model_provider_services.*.config.inference_table.table_name_prefix +3372 resources.model_provider_services.*.config.microsoft_foundry +3373 resources.model_provider_services.*.config.microsoft_foundry.direct +3374 resources.model_provider_services.*.config.microsoft_foundry.direct.api_key +3375 resources.model_provider_services.*.config.microsoft_foundry.direct.api_key.plaintext +3376 resources.model_provider_services.*.config.microsoft_foundry.direct.api_key.secret_reference +3377 resources.model_provider_services.*.config.microsoft_foundry.direct.api_key.secret_reference.name +3378 resources.model_provider_services.*.config.microsoft_foundry.direct.base_url +3379 resources.model_provider_services.*.config.microsoft_foundry.direct.entra_service_principal +3380 resources.model_provider_services.*.config.microsoft_foundry.direct.entra_service_principal.client_id +3381 resources.model_provider_services.*.config.microsoft_foundry.direct.entra_service_principal.client_secret +3382 resources.model_provider_services.*.config.microsoft_foundry.direct.entra_service_principal.client_secret.plaintext +3383 resources.model_provider_services.*.config.microsoft_foundry.direct.entra_service_principal.client_secret.secret_reference +3384 resources.model_provider_services.*.config.microsoft_foundry.direct.entra_service_principal.client_secret.secret_reference.name +3385 resources.model_provider_services.*.config.microsoft_foundry.direct.entra_service_principal.tenant_id +3386 resources.model_provider_services.*.config.microsoft_foundry.direct.service_credential +3387 resources.model_provider_services.*.config.microsoft_foundry.direct.service_credential.name +3388 resources.model_provider_services.*.config.openai +3389 resources.model_provider_services.*.config.openai.direct +3390 resources.model_provider_services.*.config.openai.direct.api_key +3391 resources.model_provider_services.*.config.openai.direct.api_key.plaintext +3392 resources.model_provider_services.*.config.openai.direct.api_key.secret_reference +3393 resources.model_provider_services.*.config.openai.direct.api_key.secret_reference.name +3394 resources.model_provider_services.*.config.openai.direct.base_url +3395 resources.model_provider_services.*.config.openai.direct.organization +3396 resources.model_provider_services.*.config.pricing +3397 resources.model_provider_services.*.config.pricing.default_discount_basis_points +3398 resources.model_provider_services.*.config.provider_type +3399 resources.model_provider_services.*.config.rate_limits +3400 resources.model_provider_services.*.config.rate_limits[*] +3401 resources.model_provider_services.*.config.rate_limits[*].key +3402 resources.model_provider_services.*.config.rate_limits[*].principal +3403 resources.model_provider_services.*.config.rate_limits[*].renewal_period +3404 resources.model_provider_services.*.config.rate_limits[*].requests +3405 resources.model_provider_services.*.config.rate_limits[*].tokens +3406 resources.model_provider_services.*.config.targets +3407 resources.model_provider_services.*.config.targets[*] +3408 resources.model_provider_services.*.config.targets[*].endpoint_route +3409 resources.model_provider_services.*.config.targets[*].model +3410 resources.model_provider_services.*.config.targets[*].native_api_types +3411 resources.model_provider_services.*.config.targets[*].native_api_types[*] +3412 resources.model_provider_services.*.grants +3413 resources.model_provider_services.*.grants[*] +3414 resources.model_provider_services.*.grants[*].principal +3415 resources.model_provider_services.*.grants[*].privileges +3416 resources.model_provider_services.*.grants[*].privileges[*] diff --git a/acceptance/bundle/bitmap/full/output/model_services.txt b/acceptance/bundle/bitmap/full/output/model_services.txt new file mode 100644 index 00000000000..15fda76ff8d --- /dev/null +++ b/acceptance/bundle/bitmap/full/output/model_services.txt @@ -0,0 +1,77 @@ +3155 resources.model_services +3156 resources.model_services.* +3157 resources.model_services.*.url +3158 resources.model_services.*.modified_status +3159 resources.model_services.*.lifecycle +3160 resources.model_services.*.lifecycle.prevent_destroy +3161 resources.model_services.*.lifecycle.prevent_destroy=true +3162 resources.model_services.*.lifecycle.prevent_destroy=false +3163 resources.model_services.*.lifecycle.depends_on +3164 resources.model_services.*.lifecycle.depends_on[*] +3165 resources.model_services.*.lifecycle.depends_on[*].resource +3166 resources.model_services.*.parent +3167 resources.model_services.*.model_service_id +3168 resources.model_services.*.comment +3169 resources.model_services.*.config +3170 resources.model_services.*.config.inference_table +3171 resources.model_services.*.config.inference_table.is_deleted +3172 resources.model_services.*.config.inference_table.is_deleted=true +3173 resources.model_services.*.config.inference_table.is_deleted=false +3174 resources.model_services.*.config.inference_table.parent +3175 resources.model_services.*.config.inference_table.table +3176 resources.model_services.*.config.inference_table.table_name_prefix +3177 resources.model_services.*.config.rate_limits +3178 resources.model_services.*.config.rate_limits[*] +3179 resources.model_services.*.config.rate_limits[*].key +3180 resources.model_services.*.config.rate_limits[*].principal +3181 resources.model_services.*.config.rate_limits[*].renewal_period +3182 resources.model_services.*.config.rate_limits[*].requests +3183 resources.model_services.*.config.rate_limits[*].tokens +3184 resources.model_services.*.config.routing +3185 resources.model_services.*.config.routing.destinations +3186 resources.model_services.*.config.routing.destinations[*] +3187 resources.model_services.*.config.routing.destinations[*].destination_type +3188 resources.model_services.*.config.routing.destinations[*].external_model_config +3189 resources.model_services.*.config.routing.destinations[*].external_model_config.model_provider_service +3190 resources.model_services.*.config.routing.destinations[*].external_model_config.target +3191 resources.model_services.*.config.routing.destinations[*].external_model_config.target.endpoint_route +3192 resources.model_services.*.config.routing.destinations[*].external_model_config.target.model +3193 resources.model_services.*.config.routing.destinations[*].external_model_config.target.native_api_types +3194 resources.model_services.*.config.routing.destinations[*].external_model_config.target.native_api_types[*] +3195 resources.model_services.*.config.routing.destinations[*].is_deleted +3196 resources.model_services.*.config.routing.destinations[*].is_deleted=true +3197 resources.model_services.*.config.routing.destinations[*].is_deleted=false +3198 resources.model_services.*.config.routing.destinations[*].name +3199 resources.model_services.*.config.routing.destinations[*].pay_per_token_config +3200 resources.model_services.*.config.routing.destinations[*].pay_per_token_config.model +3201 resources.model_services.*.config.routing.destinations[*].provisioned_throughput_config +3202 resources.model_services.*.config.routing.destinations[*].provisioned_throughput_config.model +3203 resources.model_services.*.config.routing.destinations[*].provisioned_throughput_config.model_serving_endpoint +3204 resources.model_services.*.config.routing.destinations[*].traffic_percentage +3205 resources.model_services.*.config.routing.fallback +3206 resources.model_services.*.config.routing.fallback.destinations +3207 resources.model_services.*.config.routing.fallback.destinations[*] +3208 resources.model_services.*.config.routing.fallback.destinations[*].destination_type +3209 resources.model_services.*.config.routing.fallback.destinations[*].external_model_config +3210 resources.model_services.*.config.routing.fallback.destinations[*].external_model_config.model_provider_service +3211 resources.model_services.*.config.routing.fallback.destinations[*].external_model_config.target +3212 resources.model_services.*.config.routing.fallback.destinations[*].external_model_config.target.endpoint_route +3213 resources.model_services.*.config.routing.fallback.destinations[*].external_model_config.target.model +3214 resources.model_services.*.config.routing.fallback.destinations[*].external_model_config.target.native_api_types +3215 resources.model_services.*.config.routing.fallback.destinations[*].external_model_config.target.native_api_types[*] +3216 resources.model_services.*.config.routing.fallback.destinations[*].is_deleted +3217 resources.model_services.*.config.routing.fallback.destinations[*].is_deleted=true +3218 resources.model_services.*.config.routing.fallback.destinations[*].is_deleted=false +3219 resources.model_services.*.config.routing.fallback.destinations[*].name +3220 resources.model_services.*.config.routing.fallback.destinations[*].pay_per_token_config +3221 resources.model_services.*.config.routing.fallback.destinations[*].pay_per_token_config.model +3222 resources.model_services.*.config.routing.fallback.destinations[*].provisioned_throughput_config +3223 resources.model_services.*.config.routing.fallback.destinations[*].provisioned_throughput_config.model +3224 resources.model_services.*.config.routing.fallback.destinations[*].provisioned_throughput_config.model_serving_endpoint +3225 resources.model_services.*.config.routing.fallback.destinations[*].traffic_percentage +3226 resources.model_services.*.id +3227 resources.model_services.*.grants +3228 resources.model_services.*.grants[*] +3229 resources.model_services.*.grants[*].principal +3230 resources.model_services.*.grants[*].privileges +3231 resources.model_services.*.grants[*].privileges[*] diff --git a/acceptance/bundle/bitmap/full/output/model_serving_endpoints.txt b/acceptance/bundle/bitmap/full/output/model_serving_endpoints.txt new file mode 100644 index 00000000000..4fdf897aace --- /dev/null +++ b/acceptance/bundle/bitmap/full/output/model_serving_endpoints.txt @@ -0,0 +1,206 @@ +2949 resources.model_serving_endpoints +2950 resources.model_serving_endpoints.* +2951 resources.model_serving_endpoints.*.url +2952 resources.model_serving_endpoints.*.modified_status +2953 resources.model_serving_endpoints.*.lifecycle +2954 resources.model_serving_endpoints.*.lifecycle.prevent_destroy +2955 resources.model_serving_endpoints.*.lifecycle.prevent_destroy=true +2956 resources.model_serving_endpoints.*.lifecycle.prevent_destroy=false +2957 resources.model_serving_endpoints.*.lifecycle.depends_on +2958 resources.model_serving_endpoints.*.lifecycle.depends_on[*] +2959 resources.model_serving_endpoints.*.lifecycle.depends_on[*].resource +2960 resources.model_serving_endpoints.*.id +2961 resources.model_serving_endpoints.*.ai_gateway +2962 resources.model_serving_endpoints.*.ai_gateway.fallback_config +2963 resources.model_serving_endpoints.*.ai_gateway.fallback_config.enabled +2964 resources.model_serving_endpoints.*.ai_gateway.fallback_config.enabled=true +2965 resources.model_serving_endpoints.*.ai_gateway.fallback_config.enabled=false +2966 resources.model_serving_endpoints.*.ai_gateway.guardrails +2967 resources.model_serving_endpoints.*.ai_gateway.guardrails.input +2968 resources.model_serving_endpoints.*.ai_gateway.guardrails.input.invalid_keywords +2969 resources.model_serving_endpoints.*.ai_gateway.guardrails.input.invalid_keywords[*] +2970 resources.model_serving_endpoints.*.ai_gateway.guardrails.input.pii +2971 resources.model_serving_endpoints.*.ai_gateway.guardrails.input.pii.behavior +2972 resources.model_serving_endpoints.*.ai_gateway.guardrails.input.safety +2973 resources.model_serving_endpoints.*.ai_gateway.guardrails.input.safety=true +2974 resources.model_serving_endpoints.*.ai_gateway.guardrails.input.safety=false +2975 resources.model_serving_endpoints.*.ai_gateway.guardrails.input.valid_topics +2976 resources.model_serving_endpoints.*.ai_gateway.guardrails.input.valid_topics[*] +2977 resources.model_serving_endpoints.*.ai_gateway.guardrails.output +2978 resources.model_serving_endpoints.*.ai_gateway.guardrails.output.invalid_keywords +2979 resources.model_serving_endpoints.*.ai_gateway.guardrails.output.invalid_keywords[*] +2980 resources.model_serving_endpoints.*.ai_gateway.guardrails.output.pii +2981 resources.model_serving_endpoints.*.ai_gateway.guardrails.output.pii.behavior +2982 resources.model_serving_endpoints.*.ai_gateway.guardrails.output.safety +2983 resources.model_serving_endpoints.*.ai_gateway.guardrails.output.safety=true +2984 resources.model_serving_endpoints.*.ai_gateway.guardrails.output.safety=false +2985 resources.model_serving_endpoints.*.ai_gateway.guardrails.output.valid_topics +2986 resources.model_serving_endpoints.*.ai_gateway.guardrails.output.valid_topics[*] +2987 resources.model_serving_endpoints.*.ai_gateway.inference_table_config +2988 resources.model_serving_endpoints.*.ai_gateway.inference_table_config.catalog_name +2989 resources.model_serving_endpoints.*.ai_gateway.inference_table_config.enabled +2990 resources.model_serving_endpoints.*.ai_gateway.inference_table_config.enabled=true +2991 resources.model_serving_endpoints.*.ai_gateway.inference_table_config.enabled=false +2992 resources.model_serving_endpoints.*.ai_gateway.inference_table_config.schema_name +2993 resources.model_serving_endpoints.*.ai_gateway.inference_table_config.table_name_prefix +2994 resources.model_serving_endpoints.*.ai_gateway.rate_limits +2995 resources.model_serving_endpoints.*.ai_gateway.rate_limits[*] +2996 resources.model_serving_endpoints.*.ai_gateway.rate_limits[*].calls +2997 resources.model_serving_endpoints.*.ai_gateway.rate_limits[*].key +2998 resources.model_serving_endpoints.*.ai_gateway.rate_limits[*].principal +2999 resources.model_serving_endpoints.*.ai_gateway.rate_limits[*].renewal_period +3000 resources.model_serving_endpoints.*.ai_gateway.rate_limits[*].tokens +3001 resources.model_serving_endpoints.*.ai_gateway.usage_tracking_config +3002 resources.model_serving_endpoints.*.ai_gateway.usage_tracking_config.enabled +3003 resources.model_serving_endpoints.*.ai_gateway.usage_tracking_config.enabled=true +3004 resources.model_serving_endpoints.*.ai_gateway.usage_tracking_config.enabled=false +3005 resources.model_serving_endpoints.*.budget_policy_id +3006 resources.model_serving_endpoints.*.config +3007 resources.model_serving_endpoints.*.config.auto_capture_config +3008 resources.model_serving_endpoints.*.config.auto_capture_config.catalog_name +3009 resources.model_serving_endpoints.*.config.auto_capture_config.enabled +3010 resources.model_serving_endpoints.*.config.auto_capture_config.enabled=true +3011 resources.model_serving_endpoints.*.config.auto_capture_config.enabled=false +3012 resources.model_serving_endpoints.*.config.auto_capture_config.schema_name +3013 resources.model_serving_endpoints.*.config.auto_capture_config.table_name_prefix +3014 resources.model_serving_endpoints.*.config.served_entities +3015 resources.model_serving_endpoints.*.config.served_entities[*] +3016 resources.model_serving_endpoints.*.config.served_entities[*].burst_scaling_enabled +3017 resources.model_serving_endpoints.*.config.served_entities[*].burst_scaling_enabled=true +3018 resources.model_serving_endpoints.*.config.served_entities[*].burst_scaling_enabled=false +3019 resources.model_serving_endpoints.*.config.served_entities[*].entity_name +3020 resources.model_serving_endpoints.*.config.served_entities[*].entity_version +3021 resources.model_serving_endpoints.*.config.served_entities[*].environment_vars +3022 resources.model_serving_endpoints.*.config.served_entities[*].environment_vars.* +3023 resources.model_serving_endpoints.*.config.served_entities[*].external_model +3024 resources.model_serving_endpoints.*.config.served_entities[*].external_model.ai21labs_config +3025 resources.model_serving_endpoints.*.config.served_entities[*].external_model.ai21labs_config.ai21labs_api_key +3026 resources.model_serving_endpoints.*.config.served_entities[*].external_model.ai21labs_config.ai21labs_api_key_plaintext +3027 resources.model_serving_endpoints.*.config.served_entities[*].external_model.amazon_bedrock_config +3028 resources.model_serving_endpoints.*.config.served_entities[*].external_model.amazon_bedrock_config.aws_access_key_id +3029 resources.model_serving_endpoints.*.config.served_entities[*].external_model.amazon_bedrock_config.aws_access_key_id_plaintext +3030 resources.model_serving_endpoints.*.config.served_entities[*].external_model.amazon_bedrock_config.aws_region +3031 resources.model_serving_endpoints.*.config.served_entities[*].external_model.amazon_bedrock_config.aws_secret_access_key +3032 resources.model_serving_endpoints.*.config.served_entities[*].external_model.amazon_bedrock_config.aws_secret_access_key_plaintext +3033 resources.model_serving_endpoints.*.config.served_entities[*].external_model.amazon_bedrock_config.bedrock_provider +3034 resources.model_serving_endpoints.*.config.served_entities[*].external_model.amazon_bedrock_config.instance_profile_arn +3035 resources.model_serving_endpoints.*.config.served_entities[*].external_model.amazon_bedrock_config.uc_service_credential_name +3036 resources.model_serving_endpoints.*.config.served_entities[*].external_model.anthropic_config +3037 resources.model_serving_endpoints.*.config.served_entities[*].external_model.anthropic_config.anthropic_api_key +3038 resources.model_serving_endpoints.*.config.served_entities[*].external_model.anthropic_config.anthropic_api_key_plaintext +3039 resources.model_serving_endpoints.*.config.served_entities[*].external_model.cohere_config +3040 resources.model_serving_endpoints.*.config.served_entities[*].external_model.cohere_config.cohere_api_base +3041 resources.model_serving_endpoints.*.config.served_entities[*].external_model.cohere_config.cohere_api_key +3042 resources.model_serving_endpoints.*.config.served_entities[*].external_model.cohere_config.cohere_api_key_plaintext +3043 resources.model_serving_endpoints.*.config.served_entities[*].external_model.custom_provider_config +3044 resources.model_serving_endpoints.*.config.served_entities[*].external_model.custom_provider_config.api_key_auth +3045 resources.model_serving_endpoints.*.config.served_entities[*].external_model.custom_provider_config.api_key_auth.key +3046 resources.model_serving_endpoints.*.config.served_entities[*].external_model.custom_provider_config.api_key_auth.value +3047 resources.model_serving_endpoints.*.config.served_entities[*].external_model.custom_provider_config.api_key_auth.value_plaintext +3048 resources.model_serving_endpoints.*.config.served_entities[*].external_model.custom_provider_config.bearer_token_auth +3049 resources.model_serving_endpoints.*.config.served_entities[*].external_model.custom_provider_config.bearer_token_auth.token +3050 resources.model_serving_endpoints.*.config.served_entities[*].external_model.custom_provider_config.bearer_token_auth.token_plaintext +3051 resources.model_serving_endpoints.*.config.served_entities[*].external_model.custom_provider_config.custom_provider_url +3052 resources.model_serving_endpoints.*.config.served_entities[*].external_model.databricks_model_serving_config +3053 resources.model_serving_endpoints.*.config.served_entities[*].external_model.databricks_model_serving_config.databricks_api_token +3054 resources.model_serving_endpoints.*.config.served_entities[*].external_model.databricks_model_serving_config.databricks_api_token_plaintext +3055 resources.model_serving_endpoints.*.config.served_entities[*].external_model.databricks_model_serving_config.databricks_workspace_url +3056 resources.model_serving_endpoints.*.config.served_entities[*].external_model.google_cloud_vertex_ai_config +3057 resources.model_serving_endpoints.*.config.served_entities[*].external_model.google_cloud_vertex_ai_config.private_key +3058 resources.model_serving_endpoints.*.config.served_entities[*].external_model.google_cloud_vertex_ai_config.private_key_plaintext +3059 resources.model_serving_endpoints.*.config.served_entities[*].external_model.google_cloud_vertex_ai_config.project_id +3060 resources.model_serving_endpoints.*.config.served_entities[*].external_model.google_cloud_vertex_ai_config.region +3061 resources.model_serving_endpoints.*.config.served_entities[*].external_model.name +3062 resources.model_serving_endpoints.*.config.served_entities[*].external_model.openai_config +3063 resources.model_serving_endpoints.*.config.served_entities[*].external_model.openai_config.microsoft_entra_client_id +3064 resources.model_serving_endpoints.*.config.served_entities[*].external_model.openai_config.microsoft_entra_client_secret +3065 resources.model_serving_endpoints.*.config.served_entities[*].external_model.openai_config.microsoft_entra_client_secret_plaintext +3066 resources.model_serving_endpoints.*.config.served_entities[*].external_model.openai_config.microsoft_entra_tenant_id +3067 resources.model_serving_endpoints.*.config.served_entities[*].external_model.openai_config.openai_api_base +3068 resources.model_serving_endpoints.*.config.served_entities[*].external_model.openai_config.openai_api_key +3069 resources.model_serving_endpoints.*.config.served_entities[*].external_model.openai_config.openai_api_key_plaintext +3070 resources.model_serving_endpoints.*.config.served_entities[*].external_model.openai_config.openai_api_type +3071 resources.model_serving_endpoints.*.config.served_entities[*].external_model.openai_config.openai_api_version +3072 resources.model_serving_endpoints.*.config.served_entities[*].external_model.openai_config.openai_deployment_name +3073 resources.model_serving_endpoints.*.config.served_entities[*].external_model.openai_config.openai_organization +3074 resources.model_serving_endpoints.*.config.served_entities[*].external_model.palm_config +3075 resources.model_serving_endpoints.*.config.served_entities[*].external_model.palm_config.palm_api_key +3076 resources.model_serving_endpoints.*.config.served_entities[*].external_model.palm_config.palm_api_key_plaintext +3077 resources.model_serving_endpoints.*.config.served_entities[*].external_model.provider +3078 resources.model_serving_endpoints.*.config.served_entities[*].external_model.task +3079 resources.model_serving_endpoints.*.config.served_entities[*].instance_profile_arn +3080 resources.model_serving_endpoints.*.config.served_entities[*].max_provisioned_concurrency +3081 resources.model_serving_endpoints.*.config.served_entities[*].max_provisioned_throughput +3082 resources.model_serving_endpoints.*.config.served_entities[*].min_provisioned_concurrency +3083 resources.model_serving_endpoints.*.config.served_entities[*].min_provisioned_throughput +3084 resources.model_serving_endpoints.*.config.served_entities[*].name +3085 resources.model_serving_endpoints.*.config.served_entities[*].provisioned_model_units +3086 resources.model_serving_endpoints.*.config.served_entities[*].scale_to_zero_enabled +3087 resources.model_serving_endpoints.*.config.served_entities[*].scale_to_zero_enabled=true +3088 resources.model_serving_endpoints.*.config.served_entities[*].scale_to_zero_enabled=false +3089 resources.model_serving_endpoints.*.config.served_entities[*].workload_size +3090 resources.model_serving_endpoints.*.config.served_entities[*].workload_type +3091 resources.model_serving_endpoints.*.config.served_models +3092 resources.model_serving_endpoints.*.config.served_models[*] +3093 resources.model_serving_endpoints.*.config.served_models[*].burst_scaling_enabled +3094 resources.model_serving_endpoints.*.config.served_models[*].burst_scaling_enabled=true +3095 resources.model_serving_endpoints.*.config.served_models[*].burst_scaling_enabled=false +3096 resources.model_serving_endpoints.*.config.served_models[*].environment_vars +3097 resources.model_serving_endpoints.*.config.served_models[*].environment_vars.* +3098 resources.model_serving_endpoints.*.config.served_models[*].instance_profile_arn +3099 resources.model_serving_endpoints.*.config.served_models[*].max_provisioned_concurrency +3100 resources.model_serving_endpoints.*.config.served_models[*].max_provisioned_throughput +3101 resources.model_serving_endpoints.*.config.served_models[*].min_provisioned_concurrency +3102 resources.model_serving_endpoints.*.config.served_models[*].min_provisioned_throughput +3103 resources.model_serving_endpoints.*.config.served_models[*].model_name +3104 resources.model_serving_endpoints.*.config.served_models[*].model_version +3105 resources.model_serving_endpoints.*.config.served_models[*].name +3106 resources.model_serving_endpoints.*.config.served_models[*].provisioned_model_units +3107 resources.model_serving_endpoints.*.config.served_models[*].scale_to_zero_enabled +3108 resources.model_serving_endpoints.*.config.served_models[*].scale_to_zero_enabled=true +3109 resources.model_serving_endpoints.*.config.served_models[*].scale_to_zero_enabled=false +3110 resources.model_serving_endpoints.*.config.served_models[*].workload_size +3111 resources.model_serving_endpoints.*.config.served_models[*].workload_type +3112 resources.model_serving_endpoints.*.config.traffic_config +3113 resources.model_serving_endpoints.*.config.traffic_config.routes +3114 resources.model_serving_endpoints.*.config.traffic_config.routes[*] +3115 resources.model_serving_endpoints.*.config.traffic_config.routes[*].served_entity_name +3116 resources.model_serving_endpoints.*.config.traffic_config.routes[*].served_model_name +3117 resources.model_serving_endpoints.*.config.traffic_config.routes[*].traffic_percentage +3118 resources.model_serving_endpoints.*.description +3119 resources.model_serving_endpoints.*.email_notifications +3120 resources.model_serving_endpoints.*.email_notifications.on_update_failure +3121 resources.model_serving_endpoints.*.email_notifications.on_update_failure[*] +3122 resources.model_serving_endpoints.*.email_notifications.on_update_success +3123 resources.model_serving_endpoints.*.email_notifications.on_update_success[*] +3124 resources.model_serving_endpoints.*.name +3125 resources.model_serving_endpoints.*.rate_limits +3126 resources.model_serving_endpoints.*.rate_limits[*] +3127 resources.model_serving_endpoints.*.rate_limits[*].calls +3128 resources.model_serving_endpoints.*.rate_limits[*].key +3129 resources.model_serving_endpoints.*.rate_limits[*].renewal_period +3130 resources.model_serving_endpoints.*.route_optimized +3131 resources.model_serving_endpoints.*.route_optimized=true +3132 resources.model_serving_endpoints.*.route_optimized=false +3133 resources.model_serving_endpoints.*.tags +3134 resources.model_serving_endpoints.*.tags[*] +3135 resources.model_serving_endpoints.*.tags[*].key +3136 resources.model_serving_endpoints.*.tags[*].value +3137 resources.model_serving_endpoints.*.telemetry_config +3138 resources.model_serving_endpoints.*.telemetry_config.enabled_telemetry_features +3139 resources.model_serving_endpoints.*.telemetry_config.enabled_telemetry_features[*] +3140 resources.model_serving_endpoints.*.telemetry_config.inference_table_config +3141 resources.model_serving_endpoints.*.telemetry_config.inference_table_config.name +3142 resources.model_serving_endpoints.*.telemetry_config.inference_table_config.sampling_fraction +3143 resources.model_serving_endpoints.*.telemetry_config.table_names +3144 resources.model_serving_endpoints.*.telemetry_config.table_names.annotations_table +3145 resources.model_serving_endpoints.*.telemetry_config.table_names.logs_table +3146 resources.model_serving_endpoints.*.telemetry_config.table_names.metrics_table +3147 resources.model_serving_endpoints.*.telemetry_config.table_names.traces_table +3148 resources.model_serving_endpoints.*.telemetry_config.telemetry_profile_id +3149 resources.model_serving_endpoints.*.permissions +3150 resources.model_serving_endpoints.*.permissions[*] +3151 resources.model_serving_endpoints.*.permissions[*].level +3152 resources.model_serving_endpoints.*.permissions[*].user_name +3153 resources.model_serving_endpoints.*.permissions[*].service_principal_name +3154 resources.model_serving_endpoints.*.permissions[*].group_name diff --git a/acceptance/bundle/bitmap/full/output/models.txt b/acceptance/bundle/bitmap/full/output/models.txt new file mode 100644 index 00000000000..e116aefa4a4 --- /dev/null +++ b/acceptance/bundle/bitmap/full/output/models.txt @@ -0,0 +1,24 @@ +2895 resources.models +2896 resources.models.* +2897 resources.models.*.url +2898 resources.models.*.modified_status +2899 resources.models.*.lifecycle +2900 resources.models.*.lifecycle.prevent_destroy +2901 resources.models.*.lifecycle.prevent_destroy=true +2902 resources.models.*.lifecycle.prevent_destroy=false +2903 resources.models.*.lifecycle.depends_on +2904 resources.models.*.lifecycle.depends_on[*] +2905 resources.models.*.lifecycle.depends_on[*].resource +2906 resources.models.*.id +2907 resources.models.*.description +2908 resources.models.*.name +2909 resources.models.*.tags +2910 resources.models.*.tags[*] +2911 resources.models.*.tags[*].key +2912 resources.models.*.tags[*].value +2913 resources.models.*.permissions +2914 resources.models.*.permissions[*] +2915 resources.models.*.permissions[*].level +2916 resources.models.*.permissions[*].user_name +2917 resources.models.*.permissions[*].service_principal_name +2918 resources.models.*.permissions[*].group_name diff --git a/acceptance/bundle/bitmap/full/output/pipelines.txt b/acceptance/bundle/bitmap/full/output/pipelines.txt new file mode 100644 index 00000000000..97f6d940052 --- /dev/null +++ b/acceptance/bundle/bitmap/full/output/pipelines.txt @@ -0,0 +1,994 @@ +1583 resources.pipelines +1584 resources.pipelines.* +1585 resources.pipelines.*.url +1586 resources.pipelines.*.modified_status +1587 resources.pipelines.*.lifecycle +1588 resources.pipelines.*.lifecycle.prevent_destroy +1589 resources.pipelines.*.lifecycle.prevent_destroy=true +1590 resources.pipelines.*.lifecycle.prevent_destroy=false +1591 resources.pipelines.*.lifecycle.depends_on +1592 resources.pipelines.*.lifecycle.depends_on[*] +1593 resources.pipelines.*.lifecycle.depends_on[*].resource +1594 resources.pipelines.*.allow_duplicate_names +1595 resources.pipelines.*.allow_duplicate_names=true +1596 resources.pipelines.*.allow_duplicate_names=false +1597 resources.pipelines.*.budget_policy_id +1598 resources.pipelines.*.catalog +1599 resources.pipelines.*.channel +1600 resources.pipelines.*.clusters +1601 resources.pipelines.*.clusters[*] +1602 resources.pipelines.*.clusters[*].apply_policy_default_values +1603 resources.pipelines.*.clusters[*].apply_policy_default_values=true +1604 resources.pipelines.*.clusters[*].apply_policy_default_values=false +1605 resources.pipelines.*.clusters[*].autoscale +1606 resources.pipelines.*.clusters[*].autoscale.max_workers +1607 resources.pipelines.*.clusters[*].autoscale.min_workers +1608 resources.pipelines.*.clusters[*].autoscale.mode +1609 resources.pipelines.*.clusters[*].aws_attributes +1610 resources.pipelines.*.clusters[*].aws_attributes.availability +1611 resources.pipelines.*.clusters[*].aws_attributes.ebs_volume_count +1612 resources.pipelines.*.clusters[*].aws_attributes.ebs_volume_iops +1613 resources.pipelines.*.clusters[*].aws_attributes.ebs_volume_size +1614 resources.pipelines.*.clusters[*].aws_attributes.ebs_volume_throughput +1615 resources.pipelines.*.clusters[*].aws_attributes.ebs_volume_type +1616 resources.pipelines.*.clusters[*].aws_attributes.first_on_demand +1617 resources.pipelines.*.clusters[*].aws_attributes.instance_profile_arn +1618 resources.pipelines.*.clusters[*].aws_attributes.spot_bid_price_percent +1619 resources.pipelines.*.clusters[*].aws_attributes.zone_id +1620 resources.pipelines.*.clusters[*].azure_attributes +1621 resources.pipelines.*.clusters[*].azure_attributes.availability +1622 resources.pipelines.*.clusters[*].azure_attributes.capacity_reservation_group +1623 resources.pipelines.*.clusters[*].azure_attributes.first_on_demand +1624 resources.pipelines.*.clusters[*].azure_attributes.log_analytics_info +1625 resources.pipelines.*.clusters[*].azure_attributes.log_analytics_info.log_analytics_primary_key +1626 resources.pipelines.*.clusters[*].azure_attributes.log_analytics_info.log_analytics_workspace_id +1627 resources.pipelines.*.clusters[*].azure_attributes.spot_bid_max_price +1628 resources.pipelines.*.clusters[*].cluster_log_conf +1629 resources.pipelines.*.clusters[*].cluster_log_conf.dbfs +1630 resources.pipelines.*.clusters[*].cluster_log_conf.dbfs.destination +1631 resources.pipelines.*.clusters[*].cluster_log_conf.s3 +1632 resources.pipelines.*.clusters[*].cluster_log_conf.s3.canned_acl +1633 resources.pipelines.*.clusters[*].cluster_log_conf.s3.destination +1634 resources.pipelines.*.clusters[*].cluster_log_conf.s3.enable_encryption +1635 resources.pipelines.*.clusters[*].cluster_log_conf.s3.enable_encryption=true +1636 resources.pipelines.*.clusters[*].cluster_log_conf.s3.enable_encryption=false +1637 resources.pipelines.*.clusters[*].cluster_log_conf.s3.encryption_type +1638 resources.pipelines.*.clusters[*].cluster_log_conf.s3.endpoint +1639 resources.pipelines.*.clusters[*].cluster_log_conf.s3.kms_key +1640 resources.pipelines.*.clusters[*].cluster_log_conf.s3.region +1641 resources.pipelines.*.clusters[*].cluster_log_conf.volumes +1642 resources.pipelines.*.clusters[*].cluster_log_conf.volumes.destination +1643 resources.pipelines.*.clusters[*].custom_tags +1644 resources.pipelines.*.clusters[*].custom_tags.* +1645 resources.pipelines.*.clusters[*].driver_instance_pool_id +1646 resources.pipelines.*.clusters[*].driver_node_type_id +1647 resources.pipelines.*.clusters[*].enable_local_disk_encryption +1648 resources.pipelines.*.clusters[*].enable_local_disk_encryption=true +1649 resources.pipelines.*.clusters[*].enable_local_disk_encryption=false +1650 resources.pipelines.*.clusters[*].gcp_attributes +1651 resources.pipelines.*.clusters[*].gcp_attributes.availability +1652 resources.pipelines.*.clusters[*].gcp_attributes.boot_disk_size +1653 resources.pipelines.*.clusters[*].gcp_attributes.confidential_compute_type +1654 resources.pipelines.*.clusters[*].gcp_attributes.first_on_demand +1655 resources.pipelines.*.clusters[*].gcp_attributes.google_service_account +1656 resources.pipelines.*.clusters[*].gcp_attributes.local_ssd_count +1657 resources.pipelines.*.clusters[*].gcp_attributes.use_preemptible_executors +1658 resources.pipelines.*.clusters[*].gcp_attributes.use_preemptible_executors=true +1659 resources.pipelines.*.clusters[*].gcp_attributes.use_preemptible_executors=false +1660 resources.pipelines.*.clusters[*].gcp_attributes.zone_id +1661 resources.pipelines.*.clusters[*].init_scripts +1662 resources.pipelines.*.clusters[*].init_scripts[*] +1663 resources.pipelines.*.clusters[*].init_scripts[*].abfss +1664 resources.pipelines.*.clusters[*].init_scripts[*].abfss.destination +1665 resources.pipelines.*.clusters[*].init_scripts[*].dbfs +1666 resources.pipelines.*.clusters[*].init_scripts[*].dbfs.destination +1667 resources.pipelines.*.clusters[*].init_scripts[*].file +1668 resources.pipelines.*.clusters[*].init_scripts[*].file.destination +1669 resources.pipelines.*.clusters[*].init_scripts[*].gcs +1670 resources.pipelines.*.clusters[*].init_scripts[*].gcs.destination +1671 resources.pipelines.*.clusters[*].init_scripts[*].s3 +1672 resources.pipelines.*.clusters[*].init_scripts[*].s3.canned_acl +1673 resources.pipelines.*.clusters[*].init_scripts[*].s3.destination +1674 resources.pipelines.*.clusters[*].init_scripts[*].s3.enable_encryption +1675 resources.pipelines.*.clusters[*].init_scripts[*].s3.enable_encryption=true +1676 resources.pipelines.*.clusters[*].init_scripts[*].s3.enable_encryption=false +1677 resources.pipelines.*.clusters[*].init_scripts[*].s3.encryption_type +1678 resources.pipelines.*.clusters[*].init_scripts[*].s3.endpoint +1679 resources.pipelines.*.clusters[*].init_scripts[*].s3.kms_key +1680 resources.pipelines.*.clusters[*].init_scripts[*].s3.region +1681 resources.pipelines.*.clusters[*].init_scripts[*].volumes +1682 resources.pipelines.*.clusters[*].init_scripts[*].volumes.destination +1683 resources.pipelines.*.clusters[*].init_scripts[*].workspace +1684 resources.pipelines.*.clusters[*].init_scripts[*].workspace.destination +1685 resources.pipelines.*.clusters[*].instance_pool_id +1686 resources.pipelines.*.clusters[*].label +1687 resources.pipelines.*.clusters[*].node_type_id +1688 resources.pipelines.*.clusters[*].num_workers +1689 resources.pipelines.*.clusters[*].policy_id +1690 resources.pipelines.*.clusters[*].spark_conf +1691 resources.pipelines.*.clusters[*].spark_conf.* +1692 resources.pipelines.*.clusters[*].spark_env_vars +1693 resources.pipelines.*.clusters[*].spark_env_vars.* +1694 resources.pipelines.*.clusters[*].ssh_public_keys +1695 resources.pipelines.*.clusters[*].ssh_public_keys[*] +1696 resources.pipelines.*.configuration +1697 resources.pipelines.*.configuration.* +1698 resources.pipelines.*.continuous +1699 resources.pipelines.*.continuous=true +1700 resources.pipelines.*.continuous=false +1701 resources.pipelines.*.deployment +1702 resources.pipelines.*.deployment.deployment_id +1703 resources.pipelines.*.deployment.kind +1704 resources.pipelines.*.deployment.metadata_file_path +1705 resources.pipelines.*.deployment.version_id +1706 resources.pipelines.*.development +1707 resources.pipelines.*.development=true +1708 resources.pipelines.*.development=false +1709 resources.pipelines.*.dry_run +1710 resources.pipelines.*.dry_run=true +1711 resources.pipelines.*.dry_run=false +1712 resources.pipelines.*.edition +1713 resources.pipelines.*.environment +1714 resources.pipelines.*.environment.dependencies +1715 resources.pipelines.*.environment.dependencies[*] +1716 resources.pipelines.*.environment.environment_version +1717 resources.pipelines.*.event_log +1718 resources.pipelines.*.event_log.catalog +1719 resources.pipelines.*.event_log.name +1720 resources.pipelines.*.event_log.schema +1721 resources.pipelines.*.filters +1722 resources.pipelines.*.filters.exclude +1723 resources.pipelines.*.filters.exclude[*] +1724 resources.pipelines.*.filters.include +1725 resources.pipelines.*.filters.include[*] +1726 resources.pipelines.*.gateway_definition +1727 resources.pipelines.*.gateway_definition.connection_id +1728 resources.pipelines.*.gateway_definition.connection_name +1729 resources.pipelines.*.gateway_definition.connection_parameters +1730 resources.pipelines.*.gateway_definition.connection_parameters.source_catalog +1731 resources.pipelines.*.gateway_definition.gateway_storage_catalog +1732 resources.pipelines.*.gateway_definition.gateway_storage_name +1733 resources.pipelines.*.gateway_definition.gateway_storage_schema +1734 resources.pipelines.*.id +1735 resources.pipelines.*.ingestion_definition +1736 resources.pipelines.*.ingestion_definition.connection_name +1737 resources.pipelines.*.ingestion_definition.connector_type +1738 resources.pipelines.*.ingestion_definition.data_staging_options +1739 resources.pipelines.*.ingestion_definition.data_staging_options.catalog_name +1740 resources.pipelines.*.ingestion_definition.data_staging_options.schema_name +1741 resources.pipelines.*.ingestion_definition.data_staging_options.volume_name +1742 resources.pipelines.*.ingestion_definition.full_refresh_window +1743 resources.pipelines.*.ingestion_definition.full_refresh_window.days_of_week +1744 resources.pipelines.*.ingestion_definition.full_refresh_window.days_of_week[*] +1745 resources.pipelines.*.ingestion_definition.full_refresh_window.start_hour +1746 resources.pipelines.*.ingestion_definition.full_refresh_window.time_zone_id +1747 resources.pipelines.*.ingestion_definition.ingest_from_uc_foreign_catalog +1748 resources.pipelines.*.ingestion_definition.ingest_from_uc_foreign_catalog=true +1749 resources.pipelines.*.ingestion_definition.ingest_from_uc_foreign_catalog=false +1750 resources.pipelines.*.ingestion_definition.ingestion_gateway_id +1751 resources.pipelines.*.ingestion_definition.netsuite_jar_path +1752 resources.pipelines.*.ingestion_definition.objects +1753 resources.pipelines.*.ingestion_definition.objects[*] +1754 resources.pipelines.*.ingestion_definition.objects[*].report +1755 resources.pipelines.*.ingestion_definition.objects[*].report.destination_catalog +1756 resources.pipelines.*.ingestion_definition.objects[*].report.destination_schema +1757 resources.pipelines.*.ingestion_definition.objects[*].report.destination_table +1758 resources.pipelines.*.ingestion_definition.objects[*].report.source_url +1759 resources.pipelines.*.ingestion_definition.objects[*].report.table_configuration +1760 resources.pipelines.*.ingestion_definition.objects[*].report.table_configuration.auto_full_refresh_policy +1761 resources.pipelines.*.ingestion_definition.objects[*].report.table_configuration.auto_full_refresh_policy.enabled +1762 resources.pipelines.*.ingestion_definition.objects[*].report.table_configuration.auto_full_refresh_policy.enabled=true +1763 resources.pipelines.*.ingestion_definition.objects[*].report.table_configuration.auto_full_refresh_policy.enabled=false +1764 resources.pipelines.*.ingestion_definition.objects[*].report.table_configuration.auto_full_refresh_policy.min_interval_hours +1765 resources.pipelines.*.ingestion_definition.objects[*].report.table_configuration.clustering_columns +1766 resources.pipelines.*.ingestion_definition.objects[*].report.table_configuration.clustering_columns[*] +1767 resources.pipelines.*.ingestion_definition.objects[*].report.table_configuration.enable_auto_clustering +1768 resources.pipelines.*.ingestion_definition.objects[*].report.table_configuration.enable_auto_clustering=true +1769 resources.pipelines.*.ingestion_definition.objects[*].report.table_configuration.enable_auto_clustering=false +1770 resources.pipelines.*.ingestion_definition.objects[*].report.table_configuration.exclude_columns +1771 resources.pipelines.*.ingestion_definition.objects[*].report.table_configuration.exclude_columns[*] +1772 resources.pipelines.*.ingestion_definition.objects[*].report.table_configuration.include_columns +1773 resources.pipelines.*.ingestion_definition.objects[*].report.table_configuration.include_columns[*] +1774 resources.pipelines.*.ingestion_definition.objects[*].report.table_configuration.primary_keys +1775 resources.pipelines.*.ingestion_definition.objects[*].report.table_configuration.primary_keys[*] +1776 resources.pipelines.*.ingestion_definition.objects[*].report.table_configuration.query_based_connector_config +1777 resources.pipelines.*.ingestion_definition.objects[*].report.table_configuration.query_based_connector_config.cursor_columns +1778 resources.pipelines.*.ingestion_definition.objects[*].report.table_configuration.query_based_connector_config.cursor_columns[*] +1779 resources.pipelines.*.ingestion_definition.objects[*].report.table_configuration.query_based_connector_config.deletion_condition +1780 resources.pipelines.*.ingestion_definition.objects[*].report.table_configuration.query_based_connector_config.hard_deletion_sync_min_interval_in_seconds +1781 resources.pipelines.*.ingestion_definition.objects[*].report.table_configuration.row_filter +1782 resources.pipelines.*.ingestion_definition.objects[*].report.table_configuration.salesforce_include_formula_fields +1783 resources.pipelines.*.ingestion_definition.objects[*].report.table_configuration.salesforce_include_formula_fields=true +1784 resources.pipelines.*.ingestion_definition.objects[*].report.table_configuration.salesforce_include_formula_fields=false +1785 resources.pipelines.*.ingestion_definition.objects[*].report.table_configuration.scd_type +1786 resources.pipelines.*.ingestion_definition.objects[*].report.table_configuration.sequence_by +1787 resources.pipelines.*.ingestion_definition.objects[*].report.table_configuration.sequence_by[*] +1788 resources.pipelines.*.ingestion_definition.objects[*].report.table_configuration.source_metadata_column +1789 resources.pipelines.*.ingestion_definition.objects[*].report.table_configuration.table_properties +1790 resources.pipelines.*.ingestion_definition.objects[*].report.table_configuration.table_properties.* +1791 resources.pipelines.*.ingestion_definition.objects[*].report.table_configuration.workday_report_parameters +1792 resources.pipelines.*.ingestion_definition.objects[*].report.table_configuration.workday_report_parameters.incremental +1793 resources.pipelines.*.ingestion_definition.objects[*].report.table_configuration.workday_report_parameters.incremental=true +1794 resources.pipelines.*.ingestion_definition.objects[*].report.table_configuration.workday_report_parameters.incremental=false +1795 resources.pipelines.*.ingestion_definition.objects[*].report.table_configuration.workday_report_parameters.parameters +1796 resources.pipelines.*.ingestion_definition.objects[*].report.table_configuration.workday_report_parameters.parameters.* +1797 resources.pipelines.*.ingestion_definition.objects[*].report.table_configuration.workday_report_parameters.report_parameters +1798 resources.pipelines.*.ingestion_definition.objects[*].report.table_configuration.workday_report_parameters.report_parameters[*] +1799 resources.pipelines.*.ingestion_definition.objects[*].report.table_configuration.workday_report_parameters.report_parameters[*].key +1800 resources.pipelines.*.ingestion_definition.objects[*].report.table_configuration.workday_report_parameters.report_parameters[*].value +1801 resources.pipelines.*.ingestion_definition.objects[*].schema +1802 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options +1803 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.api_source_connector_options +1804 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.api_source_connector_options.options +1805 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.api_source_connector_options.options.* +1806 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.confluence_options +1807 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.confluence_options.include_confluence_spaces +1808 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.confluence_options.include_confluence_spaces[*] +1809 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.gdrive_options +1810 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.gdrive_options.entity_type +1811 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.gdrive_options.file_ingestion_options +1812 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.gdrive_options.file_ingestion_options.corrupt_record_column +1813 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.gdrive_options.file_ingestion_options.file_filters +1814 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.gdrive_options.file_ingestion_options.file_filters[*] +1815 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.gdrive_options.file_ingestion_options.file_filters[*].modified_after +1816 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.gdrive_options.file_ingestion_options.file_filters[*].modified_before +1817 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.gdrive_options.file_ingestion_options.file_filters[*].path_filter +1818 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.gdrive_options.file_ingestion_options.format +1819 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.gdrive_options.file_ingestion_options.format_options +1820 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.gdrive_options.file_ingestion_options.format_options.* +1821 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.gdrive_options.file_ingestion_options.ignore_corrupt_files +1822 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.gdrive_options.file_ingestion_options.ignore_corrupt_files=true +1823 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.gdrive_options.file_ingestion_options.ignore_corrupt_files=false +1824 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.gdrive_options.file_ingestion_options.infer_column_types +1825 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.gdrive_options.file_ingestion_options.infer_column_types=true +1826 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.gdrive_options.file_ingestion_options.infer_column_types=false +1827 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.gdrive_options.file_ingestion_options.reader_case_sensitive +1828 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.gdrive_options.file_ingestion_options.reader_case_sensitive=true +1829 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.gdrive_options.file_ingestion_options.reader_case_sensitive=false +1830 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.gdrive_options.file_ingestion_options.rescued_data_column +1831 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.gdrive_options.file_ingestion_options.schema_evolution_mode +1832 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.gdrive_options.file_ingestion_options.schema_hints +1833 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.gdrive_options.file_ingestion_options.single_variant_column +1834 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.gdrive_options.url +1835 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.google_ads_options +1836 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.google_ads_options.custom_report_options +1837 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.google_ads_options.custom_report_options.metrics +1838 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.google_ads_options.custom_report_options.metrics[*] +1839 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.google_ads_options.custom_report_options.resource +1840 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.google_ads_options.custom_report_options.resource_fields +1841 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.google_ads_options.custom_report_options.resource_fields[*] +1842 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.google_ads_options.custom_report_options.segments +1843 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.google_ads_options.custom_report_options.segments[*] +1844 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.google_ads_options.lookback_window_days +1845 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.google_ads_options.manager_account_id +1846 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.google_ads_options.sync_start_date +1847 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.jira_options +1848 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.jira_options.include_jira_spaces +1849 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.jira_options.include_jira_spaces[*] +1850 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options +1851 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.client_config +1852 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.client_config.* +1853 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.key_transformer +1854 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.key_transformer.avro_options +1855 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.key_transformer.avro_options.parse_mode +1856 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.key_transformer.avro_options.schema +1857 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.key_transformer.avro_options.schema_file_path +1858 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.key_transformer.avro_options.schema_registry +1859 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.key_transformer.avro_options.schema_registry.confluent_options +1860 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.key_transformer.avro_options.schema_registry.confluent_options.subject +1861 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.key_transformer.avro_options.schema_registry.connection_name +1862 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.key_transformer.avro_options.schema_registry.protobuf_message_name +1863 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.key_transformer.format +1864 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.key_transformer.input_column +1865 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.key_transformer.json_options +1866 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.key_transformer.json_options.as_variant +1867 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.key_transformer.json_options.as_variant=true +1868 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.key_transformer.json_options.as_variant=false +1869 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.key_transformer.json_options.schema +1870 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.key_transformer.json_options.schema_evolution_mode +1871 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.key_transformer.json_options.schema_file_path +1872 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.key_transformer.json_options.schema_hints +1873 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.key_transformer.output_column +1874 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.key_transformer.protobuf_options +1875 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.key_transformer.protobuf_options.desc_file_path +1876 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.key_transformer.protobuf_options.message_name +1877 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.key_transformer.protobuf_options.parse_mode +1878 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.key_transformer.protobuf_options.recursive_fields_max_depth +1879 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.key_transformer.protobuf_options.schema_registry +1880 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.key_transformer.protobuf_options.schema_registry.confluent_options +1881 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.key_transformer.protobuf_options.schema_registry.confluent_options.subject +1882 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.key_transformer.protobuf_options.schema_registry.connection_name +1883 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.key_transformer.protobuf_options.schema_registry.protobuf_message_name +1884 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.max_offsets_per_trigger +1885 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.starting_offset +1886 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.topic_pattern +1887 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.topics +1888 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.topics[*] +1889 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.value_transformer +1890 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.value_transformer.avro_options +1891 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.value_transformer.avro_options.parse_mode +1892 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.value_transformer.avro_options.schema +1893 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.value_transformer.avro_options.schema_file_path +1894 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.value_transformer.avro_options.schema_registry +1895 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.value_transformer.avro_options.schema_registry.confluent_options +1896 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.value_transformer.avro_options.schema_registry.confluent_options.subject +1897 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.value_transformer.avro_options.schema_registry.connection_name +1898 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.value_transformer.avro_options.schema_registry.protobuf_message_name +1899 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.value_transformer.format +1900 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.value_transformer.input_column +1901 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.value_transformer.json_options +1902 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.value_transformer.json_options.as_variant +1903 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.value_transformer.json_options.as_variant=true +1904 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.value_transformer.json_options.as_variant=false +1905 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.value_transformer.json_options.schema +1906 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.value_transformer.json_options.schema_evolution_mode +1907 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.value_transformer.json_options.schema_file_path +1908 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.value_transformer.json_options.schema_hints +1909 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.value_transformer.output_column +1910 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.value_transformer.protobuf_options +1911 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.value_transformer.protobuf_options.desc_file_path +1912 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.value_transformer.protobuf_options.message_name +1913 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.value_transformer.protobuf_options.parse_mode +1914 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.value_transformer.protobuf_options.recursive_fields_max_depth +1915 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.value_transformer.protobuf_options.schema_registry +1916 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.value_transformer.protobuf_options.schema_registry.confluent_options +1917 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.value_transformer.protobuf_options.schema_registry.confluent_options.subject +1918 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.value_transformer.protobuf_options.schema_registry.connection_name +1919 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.value_transformer.protobuf_options.schema_registry.protobuf_message_name +1920 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.linkedin_ads_options +1921 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.linkedin_ads_options.custom_report_options +1922 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.linkedin_ads_options.custom_report_options.entity_granularity +1923 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.linkedin_ads_options.custom_report_options.entity_granularity[*] +1924 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.linkedin_ads_options.custom_report_options.finder +1925 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.linkedin_ads_options.custom_report_options.metrics +1926 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.linkedin_ads_options.custom_report_options.metrics[*] +1927 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.linkedin_ads_options.custom_report_options.time_granularity +1928 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.linkedin_ads_options.lookback_window_days +1929 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.linkedin_ads_options.sync_start_date +1930 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.marketo_options +1931 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.marketo_options.sync_start_date +1932 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.meta_ads_options +1933 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.meta_ads_options.action_attribution_windows +1934 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.meta_ads_options.action_attribution_windows[*] +1935 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.meta_ads_options.action_breakdowns +1936 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.meta_ads_options.action_breakdowns[*] +1937 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.meta_ads_options.action_report_time +1938 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.meta_ads_options.breakdowns +1939 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.meta_ads_options.breakdowns[*] +1940 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.meta_ads_options.custom_insights_lookback_window +1941 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.meta_ads_options.custom_report_options +1942 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.meta_ads_options.custom_report_options.action_attribution_windows +1943 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.meta_ads_options.custom_report_options.action_attribution_windows[*] +1944 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.meta_ads_options.custom_report_options.action_breakdowns +1945 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.meta_ads_options.custom_report_options.action_breakdowns[*] +1946 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.meta_ads_options.custom_report_options.action_report_time +1947 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.meta_ads_options.custom_report_options.breakdowns +1948 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.meta_ads_options.custom_report_options.breakdowns[*] +1949 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.meta_ads_options.custom_report_options.level +1950 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.meta_ads_options.custom_report_options.time_increment +1951 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.meta_ads_options.level +1952 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.meta_ads_options.start_date +1953 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.meta_ads_options.time_increment +1954 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.onedrive_options +1955 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.onedrive_options.entity_type +1956 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.onedrive_options.file_ingestion_options +1957 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.onedrive_options.file_ingestion_options.corrupt_record_column +1958 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.onedrive_options.file_ingestion_options.file_filters +1959 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.onedrive_options.file_ingestion_options.file_filters[*] +1960 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.onedrive_options.file_ingestion_options.file_filters[*].modified_after +1961 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.onedrive_options.file_ingestion_options.file_filters[*].modified_before +1962 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.onedrive_options.file_ingestion_options.file_filters[*].path_filter +1963 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.onedrive_options.file_ingestion_options.format +1964 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.onedrive_options.file_ingestion_options.format_options +1965 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.onedrive_options.file_ingestion_options.format_options.* +1966 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.onedrive_options.file_ingestion_options.ignore_corrupt_files +1967 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.onedrive_options.file_ingestion_options.ignore_corrupt_files=true +1968 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.onedrive_options.file_ingestion_options.ignore_corrupt_files=false +1969 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.onedrive_options.file_ingestion_options.infer_column_types +1970 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.onedrive_options.file_ingestion_options.infer_column_types=true +1971 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.onedrive_options.file_ingestion_options.infer_column_types=false +1972 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.onedrive_options.file_ingestion_options.reader_case_sensitive +1973 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.onedrive_options.file_ingestion_options.reader_case_sensitive=true +1974 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.onedrive_options.file_ingestion_options.reader_case_sensitive=false +1975 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.onedrive_options.file_ingestion_options.rescued_data_column +1976 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.onedrive_options.file_ingestion_options.schema_evolution_mode +1977 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.onedrive_options.file_ingestion_options.schema_hints +1978 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.onedrive_options.file_ingestion_options.single_variant_column +1979 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.onedrive_options.url +1980 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.outlook_options +1981 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.outlook_options.attachment_mode +1982 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.outlook_options.body_format +1983 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.outlook_options.folder_filter +1984 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.outlook_options.folder_filter[*] +1985 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.outlook_options.include_folders +1986 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.outlook_options.include_folders[*] +1987 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.outlook_options.include_mailboxes +1988 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.outlook_options.include_mailboxes[*] +1989 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.outlook_options.include_senders +1990 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.outlook_options.include_senders[*] +1991 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.outlook_options.include_subjects +1992 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.outlook_options.include_subjects[*] +1993 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.outlook_options.sender_filter +1994 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.outlook_options.sender_filter[*] +1995 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.outlook_options.start_date +1996 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.outlook_options.subject_filter +1997 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.outlook_options.subject_filter[*] +1998 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.rabbitmq_options +1999 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.rabbitmq_options.queue +2000 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.reddit_ads_options +2001 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.reddit_ads_options.custom_report_options +2002 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.reddit_ads_options.custom_report_options.breakdowns +2003 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.reddit_ads_options.custom_report_options.breakdowns[*] +2004 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.reddit_ads_options.custom_report_options.fields +2005 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.reddit_ads_options.custom_report_options.fields[*] +2006 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.reddit_ads_options.lookback_window_days +2007 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.reddit_ads_options.sync_start_date +2008 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.sharepoint_options +2009 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.sharepoint_options.entity_type +2010 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.sharepoint_options.file_ingestion_options +2011 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.sharepoint_options.file_ingestion_options.corrupt_record_column +2012 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.sharepoint_options.file_ingestion_options.file_filters +2013 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.sharepoint_options.file_ingestion_options.file_filters[*] +2014 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.sharepoint_options.file_ingestion_options.file_filters[*].modified_after +2015 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.sharepoint_options.file_ingestion_options.file_filters[*].modified_before +2016 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.sharepoint_options.file_ingestion_options.file_filters[*].path_filter +2017 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.sharepoint_options.file_ingestion_options.format +2018 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.sharepoint_options.file_ingestion_options.format_options +2019 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.sharepoint_options.file_ingestion_options.format_options.* +2020 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.sharepoint_options.file_ingestion_options.ignore_corrupt_files +2021 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.sharepoint_options.file_ingestion_options.ignore_corrupt_files=true +2022 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.sharepoint_options.file_ingestion_options.ignore_corrupt_files=false +2023 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.sharepoint_options.file_ingestion_options.infer_column_types +2024 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.sharepoint_options.file_ingestion_options.infer_column_types=true +2025 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.sharepoint_options.file_ingestion_options.infer_column_types=false +2026 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.sharepoint_options.file_ingestion_options.reader_case_sensitive +2027 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.sharepoint_options.file_ingestion_options.reader_case_sensitive=true +2028 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.sharepoint_options.file_ingestion_options.reader_case_sensitive=false +2029 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.sharepoint_options.file_ingestion_options.rescued_data_column +2030 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.sharepoint_options.file_ingestion_options.schema_evolution_mode +2031 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.sharepoint_options.file_ingestion_options.schema_hints +2032 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.sharepoint_options.file_ingestion_options.single_variant_column +2033 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.sharepoint_options.url +2034 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.smartsheet_options +2035 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.smartsheet_options.enforce_schema +2036 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.smartsheet_options.enforce_schema=true +2037 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.smartsheet_options.enforce_schema=false +2038 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.tiktok_ads_options +2039 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.tiktok_ads_options.custom_report_options +2040 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.tiktok_ads_options.custom_report_options.data_level +2041 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.tiktok_ads_options.custom_report_options.dimensions +2042 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.tiktok_ads_options.custom_report_options.dimensions[*] +2043 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.tiktok_ads_options.custom_report_options.metrics +2044 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.tiktok_ads_options.custom_report_options.metrics[*] +2045 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.tiktok_ads_options.custom_report_options.query_lifetime +2046 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.tiktok_ads_options.custom_report_options.query_lifetime=true +2047 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.tiktok_ads_options.custom_report_options.query_lifetime=false +2048 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.tiktok_ads_options.custom_report_options.report_type +2049 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.tiktok_ads_options.data_level +2050 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.tiktok_ads_options.dimensions +2051 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.tiktok_ads_options.dimensions[*] +2052 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.tiktok_ads_options.lookback_window_days +2053 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.tiktok_ads_options.metrics +2054 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.tiktok_ads_options.metrics[*] +2055 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.tiktok_ads_options.query_lifetime +2056 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.tiktok_ads_options.query_lifetime=true +2057 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.tiktok_ads_options.query_lifetime=false +2058 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.tiktok_ads_options.report_type +2059 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.tiktok_ads_options.sync_start_date +2060 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.zendesk_support_options +2061 resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.zendesk_support_options.start_date +2062 resources.pipelines.*.ingestion_definition.objects[*].schema.destination_catalog +2063 resources.pipelines.*.ingestion_definition.objects[*].schema.destination_schema +2064 resources.pipelines.*.ingestion_definition.objects[*].schema.fanout_options +2065 resources.pipelines.*.ingestion_definition.objects[*].schema.fanout_options.fanout_by +2066 resources.pipelines.*.ingestion_definition.objects[*].schema.fanout_options.transforms +2067 resources.pipelines.*.ingestion_definition.objects[*].schema.fanout_options.transforms[*] +2068 resources.pipelines.*.ingestion_definition.objects[*].schema.fanout_options.transforms[*].avro_options +2069 resources.pipelines.*.ingestion_definition.objects[*].schema.fanout_options.transforms[*].avro_options.parse_mode +2070 resources.pipelines.*.ingestion_definition.objects[*].schema.fanout_options.transforms[*].avro_options.schema +2071 resources.pipelines.*.ingestion_definition.objects[*].schema.fanout_options.transforms[*].avro_options.schema_file_path +2072 resources.pipelines.*.ingestion_definition.objects[*].schema.fanout_options.transforms[*].avro_options.schema_registry +2073 resources.pipelines.*.ingestion_definition.objects[*].schema.fanout_options.transforms[*].avro_options.schema_registry.confluent_options +2074 resources.pipelines.*.ingestion_definition.objects[*].schema.fanout_options.transforms[*].avro_options.schema_registry.confluent_options.subject +2075 resources.pipelines.*.ingestion_definition.objects[*].schema.fanout_options.transforms[*].avro_options.schema_registry.connection_name +2076 resources.pipelines.*.ingestion_definition.objects[*].schema.fanout_options.transforms[*].avro_options.schema_registry.protobuf_message_name +2077 resources.pipelines.*.ingestion_definition.objects[*].schema.fanout_options.transforms[*].format +2078 resources.pipelines.*.ingestion_definition.objects[*].schema.fanout_options.transforms[*].input_column +2079 resources.pipelines.*.ingestion_definition.objects[*].schema.fanout_options.transforms[*].json_options +2080 resources.pipelines.*.ingestion_definition.objects[*].schema.fanout_options.transforms[*].json_options.as_variant +2081 resources.pipelines.*.ingestion_definition.objects[*].schema.fanout_options.transforms[*].json_options.as_variant=true +2082 resources.pipelines.*.ingestion_definition.objects[*].schema.fanout_options.transforms[*].json_options.as_variant=false +2083 resources.pipelines.*.ingestion_definition.objects[*].schema.fanout_options.transforms[*].json_options.schema +2084 resources.pipelines.*.ingestion_definition.objects[*].schema.fanout_options.transforms[*].json_options.schema_evolution_mode +2085 resources.pipelines.*.ingestion_definition.objects[*].schema.fanout_options.transforms[*].json_options.schema_file_path +2086 resources.pipelines.*.ingestion_definition.objects[*].schema.fanout_options.transforms[*].json_options.schema_hints +2087 resources.pipelines.*.ingestion_definition.objects[*].schema.fanout_options.transforms[*].output_column +2088 resources.pipelines.*.ingestion_definition.objects[*].schema.fanout_options.transforms[*].protobuf_options +2089 resources.pipelines.*.ingestion_definition.objects[*].schema.fanout_options.transforms[*].protobuf_options.desc_file_path +2090 resources.pipelines.*.ingestion_definition.objects[*].schema.fanout_options.transforms[*].protobuf_options.message_name +2091 resources.pipelines.*.ingestion_definition.objects[*].schema.fanout_options.transforms[*].protobuf_options.parse_mode +2092 resources.pipelines.*.ingestion_definition.objects[*].schema.fanout_options.transforms[*].protobuf_options.recursive_fields_max_depth +2093 resources.pipelines.*.ingestion_definition.objects[*].schema.fanout_options.transforms[*].protobuf_options.schema_registry +2094 resources.pipelines.*.ingestion_definition.objects[*].schema.fanout_options.transforms[*].protobuf_options.schema_registry.confluent_options +2095 resources.pipelines.*.ingestion_definition.objects[*].schema.fanout_options.transforms[*].protobuf_options.schema_registry.confluent_options.subject +2096 resources.pipelines.*.ingestion_definition.objects[*].schema.fanout_options.transforms[*].protobuf_options.schema_registry.connection_name +2097 resources.pipelines.*.ingestion_definition.objects[*].schema.fanout_options.transforms[*].protobuf_options.schema_registry.protobuf_message_name +2098 resources.pipelines.*.ingestion_definition.objects[*].schema.source_catalog +2099 resources.pipelines.*.ingestion_definition.objects[*].schema.source_schema +2100 resources.pipelines.*.ingestion_definition.objects[*].schema.table_configuration +2101 resources.pipelines.*.ingestion_definition.objects[*].schema.table_configuration.auto_full_refresh_policy +2102 resources.pipelines.*.ingestion_definition.objects[*].schema.table_configuration.auto_full_refresh_policy.enabled +2103 resources.pipelines.*.ingestion_definition.objects[*].schema.table_configuration.auto_full_refresh_policy.enabled=true +2104 resources.pipelines.*.ingestion_definition.objects[*].schema.table_configuration.auto_full_refresh_policy.enabled=false +2105 resources.pipelines.*.ingestion_definition.objects[*].schema.table_configuration.auto_full_refresh_policy.min_interval_hours +2106 resources.pipelines.*.ingestion_definition.objects[*].schema.table_configuration.clustering_columns +2107 resources.pipelines.*.ingestion_definition.objects[*].schema.table_configuration.clustering_columns[*] +2108 resources.pipelines.*.ingestion_definition.objects[*].schema.table_configuration.enable_auto_clustering +2109 resources.pipelines.*.ingestion_definition.objects[*].schema.table_configuration.enable_auto_clustering=true +2110 resources.pipelines.*.ingestion_definition.objects[*].schema.table_configuration.enable_auto_clustering=false +2111 resources.pipelines.*.ingestion_definition.objects[*].schema.table_configuration.exclude_columns +2112 resources.pipelines.*.ingestion_definition.objects[*].schema.table_configuration.exclude_columns[*] +2113 resources.pipelines.*.ingestion_definition.objects[*].schema.table_configuration.include_columns +2114 resources.pipelines.*.ingestion_definition.objects[*].schema.table_configuration.include_columns[*] +2115 resources.pipelines.*.ingestion_definition.objects[*].schema.table_configuration.primary_keys +2116 resources.pipelines.*.ingestion_definition.objects[*].schema.table_configuration.primary_keys[*] +2117 resources.pipelines.*.ingestion_definition.objects[*].schema.table_configuration.query_based_connector_config +2118 resources.pipelines.*.ingestion_definition.objects[*].schema.table_configuration.query_based_connector_config.cursor_columns +2119 resources.pipelines.*.ingestion_definition.objects[*].schema.table_configuration.query_based_connector_config.cursor_columns[*] +2120 resources.pipelines.*.ingestion_definition.objects[*].schema.table_configuration.query_based_connector_config.deletion_condition +2121 resources.pipelines.*.ingestion_definition.objects[*].schema.table_configuration.query_based_connector_config.hard_deletion_sync_min_interval_in_seconds +2122 resources.pipelines.*.ingestion_definition.objects[*].schema.table_configuration.row_filter +2123 resources.pipelines.*.ingestion_definition.objects[*].schema.table_configuration.salesforce_include_formula_fields +2124 resources.pipelines.*.ingestion_definition.objects[*].schema.table_configuration.salesforce_include_formula_fields=true +2125 resources.pipelines.*.ingestion_definition.objects[*].schema.table_configuration.salesforce_include_formula_fields=false +2126 resources.pipelines.*.ingestion_definition.objects[*].schema.table_configuration.scd_type +2127 resources.pipelines.*.ingestion_definition.objects[*].schema.table_configuration.sequence_by +2128 resources.pipelines.*.ingestion_definition.objects[*].schema.table_configuration.sequence_by[*] +2129 resources.pipelines.*.ingestion_definition.objects[*].schema.table_configuration.source_metadata_column +2130 resources.pipelines.*.ingestion_definition.objects[*].schema.table_configuration.table_properties +2131 resources.pipelines.*.ingestion_definition.objects[*].schema.table_configuration.table_properties.* +2132 resources.pipelines.*.ingestion_definition.objects[*].schema.table_configuration.workday_report_parameters +2133 resources.pipelines.*.ingestion_definition.objects[*].schema.table_configuration.workday_report_parameters.incremental +2134 resources.pipelines.*.ingestion_definition.objects[*].schema.table_configuration.workday_report_parameters.incremental=true +2135 resources.pipelines.*.ingestion_definition.objects[*].schema.table_configuration.workday_report_parameters.incremental=false +2136 resources.pipelines.*.ingestion_definition.objects[*].schema.table_configuration.workday_report_parameters.parameters +2137 resources.pipelines.*.ingestion_definition.objects[*].schema.table_configuration.workday_report_parameters.parameters.* +2138 resources.pipelines.*.ingestion_definition.objects[*].schema.table_configuration.workday_report_parameters.report_parameters +2139 resources.pipelines.*.ingestion_definition.objects[*].schema.table_configuration.workday_report_parameters.report_parameters[*] +2140 resources.pipelines.*.ingestion_definition.objects[*].schema.table_configuration.workday_report_parameters.report_parameters[*].key +2141 resources.pipelines.*.ingestion_definition.objects[*].schema.table_configuration.workday_report_parameters.report_parameters[*].value +2142 resources.pipelines.*.ingestion_definition.objects[*].table +2143 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options +2144 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.api_source_connector_options +2145 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.api_source_connector_options.options +2146 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.api_source_connector_options.options.* +2147 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.confluence_options +2148 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.confluence_options.include_confluence_spaces +2149 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.confluence_options.include_confluence_spaces[*] +2150 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.gdrive_options +2151 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.gdrive_options.entity_type +2152 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.gdrive_options.file_ingestion_options +2153 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.gdrive_options.file_ingestion_options.corrupt_record_column +2154 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.gdrive_options.file_ingestion_options.file_filters +2155 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.gdrive_options.file_ingestion_options.file_filters[*] +2156 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.gdrive_options.file_ingestion_options.file_filters[*].modified_after +2157 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.gdrive_options.file_ingestion_options.file_filters[*].modified_before +2158 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.gdrive_options.file_ingestion_options.file_filters[*].path_filter +2159 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.gdrive_options.file_ingestion_options.format +2160 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.gdrive_options.file_ingestion_options.format_options +2161 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.gdrive_options.file_ingestion_options.format_options.* +2162 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.gdrive_options.file_ingestion_options.ignore_corrupt_files +2163 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.gdrive_options.file_ingestion_options.ignore_corrupt_files=true +2164 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.gdrive_options.file_ingestion_options.ignore_corrupt_files=false +2165 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.gdrive_options.file_ingestion_options.infer_column_types +2166 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.gdrive_options.file_ingestion_options.infer_column_types=true +2167 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.gdrive_options.file_ingestion_options.infer_column_types=false +2168 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.gdrive_options.file_ingestion_options.reader_case_sensitive +2169 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.gdrive_options.file_ingestion_options.reader_case_sensitive=true +2170 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.gdrive_options.file_ingestion_options.reader_case_sensitive=false +2171 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.gdrive_options.file_ingestion_options.rescued_data_column +2172 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.gdrive_options.file_ingestion_options.schema_evolution_mode +2173 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.gdrive_options.file_ingestion_options.schema_hints +2174 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.gdrive_options.file_ingestion_options.single_variant_column +2175 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.gdrive_options.url +2176 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.google_ads_options +2177 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.google_ads_options.custom_report_options +2178 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.google_ads_options.custom_report_options.metrics +2179 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.google_ads_options.custom_report_options.metrics[*] +2180 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.google_ads_options.custom_report_options.resource +2181 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.google_ads_options.custom_report_options.resource_fields +2182 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.google_ads_options.custom_report_options.resource_fields[*] +2183 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.google_ads_options.custom_report_options.segments +2184 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.google_ads_options.custom_report_options.segments[*] +2185 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.google_ads_options.lookback_window_days +2186 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.google_ads_options.manager_account_id +2187 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.google_ads_options.sync_start_date +2188 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.jira_options +2189 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.jira_options.include_jira_spaces +2190 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.jira_options.include_jira_spaces[*] +2191 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options +2192 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.client_config +2193 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.client_config.* +2194 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.key_transformer +2195 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.key_transformer.avro_options +2196 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.key_transformer.avro_options.parse_mode +2197 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.key_transformer.avro_options.schema +2198 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.key_transformer.avro_options.schema_file_path +2199 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.key_transformer.avro_options.schema_registry +2200 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.key_transformer.avro_options.schema_registry.confluent_options +2201 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.key_transformer.avro_options.schema_registry.confluent_options.subject +2202 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.key_transformer.avro_options.schema_registry.connection_name +2203 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.key_transformer.avro_options.schema_registry.protobuf_message_name +2204 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.key_transformer.format +2205 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.key_transformer.input_column +2206 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.key_transformer.json_options +2207 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.key_transformer.json_options.as_variant +2208 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.key_transformer.json_options.as_variant=true +2209 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.key_transformer.json_options.as_variant=false +2210 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.key_transformer.json_options.schema +2211 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.key_transformer.json_options.schema_evolution_mode +2212 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.key_transformer.json_options.schema_file_path +2213 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.key_transformer.json_options.schema_hints +2214 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.key_transformer.output_column +2215 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.key_transformer.protobuf_options +2216 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.key_transformer.protobuf_options.desc_file_path +2217 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.key_transformer.protobuf_options.message_name +2218 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.key_transformer.protobuf_options.parse_mode +2219 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.key_transformer.protobuf_options.recursive_fields_max_depth +2220 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.key_transformer.protobuf_options.schema_registry +2221 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.key_transformer.protobuf_options.schema_registry.confluent_options +2222 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.key_transformer.protobuf_options.schema_registry.confluent_options.subject +2223 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.key_transformer.protobuf_options.schema_registry.connection_name +2224 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.key_transformer.protobuf_options.schema_registry.protobuf_message_name +2225 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.max_offsets_per_trigger +2226 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.starting_offset +2227 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.topic_pattern +2228 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.topics +2229 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.topics[*] +2230 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.value_transformer +2231 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.value_transformer.avro_options +2232 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.value_transformer.avro_options.parse_mode +2233 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.value_transformer.avro_options.schema +2234 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.value_transformer.avro_options.schema_file_path +2235 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.value_transformer.avro_options.schema_registry +2236 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.value_transformer.avro_options.schema_registry.confluent_options +2237 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.value_transformer.avro_options.schema_registry.confluent_options.subject +2238 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.value_transformer.avro_options.schema_registry.connection_name +2239 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.value_transformer.avro_options.schema_registry.protobuf_message_name +2240 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.value_transformer.format +2241 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.value_transformer.input_column +2242 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.value_transformer.json_options +2243 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.value_transformer.json_options.as_variant +2244 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.value_transformer.json_options.as_variant=true +2245 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.value_transformer.json_options.as_variant=false +2246 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.value_transformer.json_options.schema +2247 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.value_transformer.json_options.schema_evolution_mode +2248 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.value_transformer.json_options.schema_file_path +2249 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.value_transformer.json_options.schema_hints +2250 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.value_transformer.output_column +2251 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.value_transformer.protobuf_options +2252 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.value_transformer.protobuf_options.desc_file_path +2253 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.value_transformer.protobuf_options.message_name +2254 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.value_transformer.protobuf_options.parse_mode +2255 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.value_transformer.protobuf_options.recursive_fields_max_depth +2256 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.value_transformer.protobuf_options.schema_registry +2257 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.value_transformer.protobuf_options.schema_registry.confluent_options +2258 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.value_transformer.protobuf_options.schema_registry.confluent_options.subject +2259 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.value_transformer.protobuf_options.schema_registry.connection_name +2260 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.value_transformer.protobuf_options.schema_registry.protobuf_message_name +2261 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.linkedin_ads_options +2262 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.linkedin_ads_options.custom_report_options +2263 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.linkedin_ads_options.custom_report_options.entity_granularity +2264 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.linkedin_ads_options.custom_report_options.entity_granularity[*] +2265 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.linkedin_ads_options.custom_report_options.finder +2266 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.linkedin_ads_options.custom_report_options.metrics +2267 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.linkedin_ads_options.custom_report_options.metrics[*] +2268 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.linkedin_ads_options.custom_report_options.time_granularity +2269 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.linkedin_ads_options.lookback_window_days +2270 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.linkedin_ads_options.sync_start_date +2271 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.marketo_options +2272 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.marketo_options.sync_start_date +2273 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.meta_ads_options +2274 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.meta_ads_options.action_attribution_windows +2275 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.meta_ads_options.action_attribution_windows[*] +2276 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.meta_ads_options.action_breakdowns +2277 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.meta_ads_options.action_breakdowns[*] +2278 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.meta_ads_options.action_report_time +2279 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.meta_ads_options.breakdowns +2280 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.meta_ads_options.breakdowns[*] +2281 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.meta_ads_options.custom_insights_lookback_window +2282 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.meta_ads_options.custom_report_options +2283 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.meta_ads_options.custom_report_options.action_attribution_windows +2284 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.meta_ads_options.custom_report_options.action_attribution_windows[*] +2285 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.meta_ads_options.custom_report_options.action_breakdowns +2286 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.meta_ads_options.custom_report_options.action_breakdowns[*] +2287 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.meta_ads_options.custom_report_options.action_report_time +2288 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.meta_ads_options.custom_report_options.breakdowns +2289 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.meta_ads_options.custom_report_options.breakdowns[*] +2290 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.meta_ads_options.custom_report_options.level +2291 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.meta_ads_options.custom_report_options.time_increment +2292 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.meta_ads_options.level +2293 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.meta_ads_options.start_date +2294 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.meta_ads_options.time_increment +2295 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.onedrive_options +2296 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.onedrive_options.entity_type +2297 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.onedrive_options.file_ingestion_options +2298 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.onedrive_options.file_ingestion_options.corrupt_record_column +2299 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.onedrive_options.file_ingestion_options.file_filters +2300 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.onedrive_options.file_ingestion_options.file_filters[*] +2301 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.onedrive_options.file_ingestion_options.file_filters[*].modified_after +2302 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.onedrive_options.file_ingestion_options.file_filters[*].modified_before +2303 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.onedrive_options.file_ingestion_options.file_filters[*].path_filter +2304 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.onedrive_options.file_ingestion_options.format +2305 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.onedrive_options.file_ingestion_options.format_options +2306 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.onedrive_options.file_ingestion_options.format_options.* +2307 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.onedrive_options.file_ingestion_options.ignore_corrupt_files +2308 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.onedrive_options.file_ingestion_options.ignore_corrupt_files=true +2309 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.onedrive_options.file_ingestion_options.ignore_corrupt_files=false +2310 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.onedrive_options.file_ingestion_options.infer_column_types +2311 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.onedrive_options.file_ingestion_options.infer_column_types=true +2312 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.onedrive_options.file_ingestion_options.infer_column_types=false +2313 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.onedrive_options.file_ingestion_options.reader_case_sensitive +2314 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.onedrive_options.file_ingestion_options.reader_case_sensitive=true +2315 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.onedrive_options.file_ingestion_options.reader_case_sensitive=false +2316 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.onedrive_options.file_ingestion_options.rescued_data_column +2317 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.onedrive_options.file_ingestion_options.schema_evolution_mode +2318 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.onedrive_options.file_ingestion_options.schema_hints +2319 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.onedrive_options.file_ingestion_options.single_variant_column +2320 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.onedrive_options.url +2321 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.outlook_options +2322 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.outlook_options.attachment_mode +2323 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.outlook_options.body_format +2324 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.outlook_options.folder_filter +2325 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.outlook_options.folder_filter[*] +2326 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.outlook_options.include_folders +2327 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.outlook_options.include_folders[*] +2328 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.outlook_options.include_mailboxes +2329 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.outlook_options.include_mailboxes[*] +2330 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.outlook_options.include_senders +2331 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.outlook_options.include_senders[*] +2332 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.outlook_options.include_subjects +2333 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.outlook_options.include_subjects[*] +2334 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.outlook_options.sender_filter +2335 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.outlook_options.sender_filter[*] +2336 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.outlook_options.start_date +2337 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.outlook_options.subject_filter +2338 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.outlook_options.subject_filter[*] +2339 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.rabbitmq_options +2340 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.rabbitmq_options.queue +2341 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.reddit_ads_options +2342 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.reddit_ads_options.custom_report_options +2343 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.reddit_ads_options.custom_report_options.breakdowns +2344 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.reddit_ads_options.custom_report_options.breakdowns[*] +2345 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.reddit_ads_options.custom_report_options.fields +2346 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.reddit_ads_options.custom_report_options.fields[*] +2347 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.reddit_ads_options.lookback_window_days +2348 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.reddit_ads_options.sync_start_date +2349 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.sharepoint_options +2350 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.sharepoint_options.entity_type +2351 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.sharepoint_options.file_ingestion_options +2352 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.sharepoint_options.file_ingestion_options.corrupt_record_column +2353 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.sharepoint_options.file_ingestion_options.file_filters +2354 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.sharepoint_options.file_ingestion_options.file_filters[*] +2355 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.sharepoint_options.file_ingestion_options.file_filters[*].modified_after +2356 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.sharepoint_options.file_ingestion_options.file_filters[*].modified_before +2357 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.sharepoint_options.file_ingestion_options.file_filters[*].path_filter +2358 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.sharepoint_options.file_ingestion_options.format +2359 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.sharepoint_options.file_ingestion_options.format_options +2360 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.sharepoint_options.file_ingestion_options.format_options.* +2361 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.sharepoint_options.file_ingestion_options.ignore_corrupt_files +2362 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.sharepoint_options.file_ingestion_options.ignore_corrupt_files=true +2363 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.sharepoint_options.file_ingestion_options.ignore_corrupt_files=false +2364 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.sharepoint_options.file_ingestion_options.infer_column_types +2365 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.sharepoint_options.file_ingestion_options.infer_column_types=true +2366 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.sharepoint_options.file_ingestion_options.infer_column_types=false +2367 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.sharepoint_options.file_ingestion_options.reader_case_sensitive +2368 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.sharepoint_options.file_ingestion_options.reader_case_sensitive=true +2369 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.sharepoint_options.file_ingestion_options.reader_case_sensitive=false +2370 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.sharepoint_options.file_ingestion_options.rescued_data_column +2371 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.sharepoint_options.file_ingestion_options.schema_evolution_mode +2372 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.sharepoint_options.file_ingestion_options.schema_hints +2373 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.sharepoint_options.file_ingestion_options.single_variant_column +2374 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.sharepoint_options.url +2375 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.smartsheet_options +2376 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.smartsheet_options.enforce_schema +2377 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.smartsheet_options.enforce_schema=true +2378 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.smartsheet_options.enforce_schema=false +2379 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.tiktok_ads_options +2380 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.tiktok_ads_options.custom_report_options +2381 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.tiktok_ads_options.custom_report_options.data_level +2382 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.tiktok_ads_options.custom_report_options.dimensions +2383 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.tiktok_ads_options.custom_report_options.dimensions[*] +2384 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.tiktok_ads_options.custom_report_options.metrics +2385 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.tiktok_ads_options.custom_report_options.metrics[*] +2386 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.tiktok_ads_options.custom_report_options.query_lifetime +2387 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.tiktok_ads_options.custom_report_options.query_lifetime=true +2388 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.tiktok_ads_options.custom_report_options.query_lifetime=false +2389 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.tiktok_ads_options.custom_report_options.report_type +2390 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.tiktok_ads_options.data_level +2391 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.tiktok_ads_options.dimensions +2392 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.tiktok_ads_options.dimensions[*] +2393 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.tiktok_ads_options.lookback_window_days +2394 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.tiktok_ads_options.metrics +2395 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.tiktok_ads_options.metrics[*] +2396 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.tiktok_ads_options.query_lifetime +2397 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.tiktok_ads_options.query_lifetime=true +2398 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.tiktok_ads_options.query_lifetime=false +2399 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.tiktok_ads_options.report_type +2400 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.tiktok_ads_options.sync_start_date +2401 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.zendesk_support_options +2402 resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.zendesk_support_options.start_date +2403 resources.pipelines.*.ingestion_definition.objects[*].table.destination_catalog +2404 resources.pipelines.*.ingestion_definition.objects[*].table.destination_schema +2405 resources.pipelines.*.ingestion_definition.objects[*].table.destination_table +2406 resources.pipelines.*.ingestion_definition.objects[*].table.source_catalog +2407 resources.pipelines.*.ingestion_definition.objects[*].table.source_schema +2408 resources.pipelines.*.ingestion_definition.objects[*].table.source_table +2409 resources.pipelines.*.ingestion_definition.objects[*].table.table_configuration +2410 resources.pipelines.*.ingestion_definition.objects[*].table.table_configuration.auto_full_refresh_policy +2411 resources.pipelines.*.ingestion_definition.objects[*].table.table_configuration.auto_full_refresh_policy.enabled +2412 resources.pipelines.*.ingestion_definition.objects[*].table.table_configuration.auto_full_refresh_policy.enabled=true +2413 resources.pipelines.*.ingestion_definition.objects[*].table.table_configuration.auto_full_refresh_policy.enabled=false +2414 resources.pipelines.*.ingestion_definition.objects[*].table.table_configuration.auto_full_refresh_policy.min_interval_hours +2415 resources.pipelines.*.ingestion_definition.objects[*].table.table_configuration.clustering_columns +2416 resources.pipelines.*.ingestion_definition.objects[*].table.table_configuration.clustering_columns[*] +2417 resources.pipelines.*.ingestion_definition.objects[*].table.table_configuration.enable_auto_clustering +2418 resources.pipelines.*.ingestion_definition.objects[*].table.table_configuration.enable_auto_clustering=true +2419 resources.pipelines.*.ingestion_definition.objects[*].table.table_configuration.enable_auto_clustering=false +2420 resources.pipelines.*.ingestion_definition.objects[*].table.table_configuration.exclude_columns +2421 resources.pipelines.*.ingestion_definition.objects[*].table.table_configuration.exclude_columns[*] +2422 resources.pipelines.*.ingestion_definition.objects[*].table.table_configuration.include_columns +2423 resources.pipelines.*.ingestion_definition.objects[*].table.table_configuration.include_columns[*] +2424 resources.pipelines.*.ingestion_definition.objects[*].table.table_configuration.primary_keys +2425 resources.pipelines.*.ingestion_definition.objects[*].table.table_configuration.primary_keys[*] +2426 resources.pipelines.*.ingestion_definition.objects[*].table.table_configuration.query_based_connector_config +2427 resources.pipelines.*.ingestion_definition.objects[*].table.table_configuration.query_based_connector_config.cursor_columns +2428 resources.pipelines.*.ingestion_definition.objects[*].table.table_configuration.query_based_connector_config.cursor_columns[*] +2429 resources.pipelines.*.ingestion_definition.objects[*].table.table_configuration.query_based_connector_config.deletion_condition +2430 resources.pipelines.*.ingestion_definition.objects[*].table.table_configuration.query_based_connector_config.hard_deletion_sync_min_interval_in_seconds +2431 resources.pipelines.*.ingestion_definition.objects[*].table.table_configuration.row_filter +2432 resources.pipelines.*.ingestion_definition.objects[*].table.table_configuration.salesforce_include_formula_fields +2433 resources.pipelines.*.ingestion_definition.objects[*].table.table_configuration.salesforce_include_formula_fields=true +2434 resources.pipelines.*.ingestion_definition.objects[*].table.table_configuration.salesforce_include_formula_fields=false +2435 resources.pipelines.*.ingestion_definition.objects[*].table.table_configuration.scd_type +2436 resources.pipelines.*.ingestion_definition.objects[*].table.table_configuration.sequence_by +2437 resources.pipelines.*.ingestion_definition.objects[*].table.table_configuration.sequence_by[*] +2438 resources.pipelines.*.ingestion_definition.objects[*].table.table_configuration.source_metadata_column +2439 resources.pipelines.*.ingestion_definition.objects[*].table.table_configuration.table_properties +2440 resources.pipelines.*.ingestion_definition.objects[*].table.table_configuration.table_properties.* +2441 resources.pipelines.*.ingestion_definition.objects[*].table.table_configuration.workday_report_parameters +2442 resources.pipelines.*.ingestion_definition.objects[*].table.table_configuration.workday_report_parameters.incremental +2443 resources.pipelines.*.ingestion_definition.objects[*].table.table_configuration.workday_report_parameters.incremental=true +2444 resources.pipelines.*.ingestion_definition.objects[*].table.table_configuration.workday_report_parameters.incremental=false +2445 resources.pipelines.*.ingestion_definition.objects[*].table.table_configuration.workday_report_parameters.parameters +2446 resources.pipelines.*.ingestion_definition.objects[*].table.table_configuration.workday_report_parameters.parameters.* +2447 resources.pipelines.*.ingestion_definition.objects[*].table.table_configuration.workday_report_parameters.report_parameters +2448 resources.pipelines.*.ingestion_definition.objects[*].table.table_configuration.workday_report_parameters.report_parameters[*] +2449 resources.pipelines.*.ingestion_definition.objects[*].table.table_configuration.workday_report_parameters.report_parameters[*].key +2450 resources.pipelines.*.ingestion_definition.objects[*].table.table_configuration.workday_report_parameters.report_parameters[*].value +2451 resources.pipelines.*.ingestion_definition.source_configurations +2452 resources.pipelines.*.ingestion_definition.source_configurations[*] +2453 resources.pipelines.*.ingestion_definition.source_configurations[*].api_source_connector_config +2454 resources.pipelines.*.ingestion_definition.source_configurations[*].api_source_connector_config.configs +2455 resources.pipelines.*.ingestion_definition.source_configurations[*].api_source_connector_config.configs.* +2456 resources.pipelines.*.ingestion_definition.source_configurations[*].catalog +2457 resources.pipelines.*.ingestion_definition.source_configurations[*].catalog.postgres +2458 resources.pipelines.*.ingestion_definition.source_configurations[*].catalog.postgres.slot_config +2459 resources.pipelines.*.ingestion_definition.source_configurations[*].catalog.postgres.slot_config.publication_name +2460 resources.pipelines.*.ingestion_definition.source_configurations[*].catalog.postgres.slot_config.slot_name +2461 resources.pipelines.*.ingestion_definition.source_configurations[*].catalog.source_catalog +2462 resources.pipelines.*.ingestion_definition.source_configurations[*].google_ads_config +2463 resources.pipelines.*.ingestion_definition.source_configurations[*].google_ads_config.manager_account_id +2464 resources.pipelines.*.ingestion_definition.source_type +2465 resources.pipelines.*.ingestion_definition.table_configuration +2466 resources.pipelines.*.ingestion_definition.table_configuration.auto_full_refresh_policy +2467 resources.pipelines.*.ingestion_definition.table_configuration.auto_full_refresh_policy.enabled +2468 resources.pipelines.*.ingestion_definition.table_configuration.auto_full_refresh_policy.enabled=true +2469 resources.pipelines.*.ingestion_definition.table_configuration.auto_full_refresh_policy.enabled=false +2470 resources.pipelines.*.ingestion_definition.table_configuration.auto_full_refresh_policy.min_interval_hours +2471 resources.pipelines.*.ingestion_definition.table_configuration.clustering_columns +2472 resources.pipelines.*.ingestion_definition.table_configuration.clustering_columns[*] +2473 resources.pipelines.*.ingestion_definition.table_configuration.enable_auto_clustering +2474 resources.pipelines.*.ingestion_definition.table_configuration.enable_auto_clustering=true +2475 resources.pipelines.*.ingestion_definition.table_configuration.enable_auto_clustering=false +2476 resources.pipelines.*.ingestion_definition.table_configuration.exclude_columns +2477 resources.pipelines.*.ingestion_definition.table_configuration.exclude_columns[*] +2478 resources.pipelines.*.ingestion_definition.table_configuration.include_columns +2479 resources.pipelines.*.ingestion_definition.table_configuration.include_columns[*] +2480 resources.pipelines.*.ingestion_definition.table_configuration.primary_keys +2481 resources.pipelines.*.ingestion_definition.table_configuration.primary_keys[*] +2482 resources.pipelines.*.ingestion_definition.table_configuration.query_based_connector_config +2483 resources.pipelines.*.ingestion_definition.table_configuration.query_based_connector_config.cursor_columns +2484 resources.pipelines.*.ingestion_definition.table_configuration.query_based_connector_config.cursor_columns[*] +2485 resources.pipelines.*.ingestion_definition.table_configuration.query_based_connector_config.deletion_condition +2486 resources.pipelines.*.ingestion_definition.table_configuration.query_based_connector_config.hard_deletion_sync_min_interval_in_seconds +2487 resources.pipelines.*.ingestion_definition.table_configuration.row_filter +2488 resources.pipelines.*.ingestion_definition.table_configuration.salesforce_include_formula_fields +2489 resources.pipelines.*.ingestion_definition.table_configuration.salesforce_include_formula_fields=true +2490 resources.pipelines.*.ingestion_definition.table_configuration.salesforce_include_formula_fields=false +2491 resources.pipelines.*.ingestion_definition.table_configuration.scd_type +2492 resources.pipelines.*.ingestion_definition.table_configuration.sequence_by +2493 resources.pipelines.*.ingestion_definition.table_configuration.sequence_by[*] +2494 resources.pipelines.*.ingestion_definition.table_configuration.source_metadata_column +2495 resources.pipelines.*.ingestion_definition.table_configuration.table_properties +2496 resources.pipelines.*.ingestion_definition.table_configuration.table_properties.* +2497 resources.pipelines.*.ingestion_definition.table_configuration.workday_report_parameters +2498 resources.pipelines.*.ingestion_definition.table_configuration.workday_report_parameters.incremental +2499 resources.pipelines.*.ingestion_definition.table_configuration.workday_report_parameters.incremental=true +2500 resources.pipelines.*.ingestion_definition.table_configuration.workday_report_parameters.incremental=false +2501 resources.pipelines.*.ingestion_definition.table_configuration.workday_report_parameters.parameters +2502 resources.pipelines.*.ingestion_definition.table_configuration.workday_report_parameters.parameters.* +2503 resources.pipelines.*.ingestion_definition.table_configuration.workday_report_parameters.report_parameters +2504 resources.pipelines.*.ingestion_definition.table_configuration.workday_report_parameters.report_parameters[*] +2505 resources.pipelines.*.ingestion_definition.table_configuration.workday_report_parameters.report_parameters[*].key +2506 resources.pipelines.*.ingestion_definition.table_configuration.workday_report_parameters.report_parameters[*].value +2507 resources.pipelines.*.libraries +2508 resources.pipelines.*.libraries[*] +2509 resources.pipelines.*.libraries[*].file +2510 resources.pipelines.*.libraries[*].file.path +2511 resources.pipelines.*.libraries[*].glob +2512 resources.pipelines.*.libraries[*].glob.include +2513 resources.pipelines.*.libraries[*].jar +2514 resources.pipelines.*.libraries[*].maven +2515 resources.pipelines.*.libraries[*].maven.coordinates +2516 resources.pipelines.*.libraries[*].maven.exclusions +2517 resources.pipelines.*.libraries[*].maven.exclusions[*] +2518 resources.pipelines.*.libraries[*].maven.repo +2519 resources.pipelines.*.libraries[*].notebook +2520 resources.pipelines.*.libraries[*].notebook.path +2521 resources.pipelines.*.libraries[*].whl +2522 resources.pipelines.*.name +2523 resources.pipelines.*.notifications +2524 resources.pipelines.*.notifications[*] +2525 resources.pipelines.*.notifications[*].alerts +2526 resources.pipelines.*.notifications[*].alerts[*] +2527 resources.pipelines.*.notifications[*].email_recipients +2528 resources.pipelines.*.notifications[*].email_recipients[*] +2529 resources.pipelines.*.parameters +2530 resources.pipelines.*.parameters.* +2531 resources.pipelines.*.parent_path +2532 resources.pipelines.*.photon +2533 resources.pipelines.*.photon=true +2534 resources.pipelines.*.photon=false +2535 resources.pipelines.*.restart_window +2536 resources.pipelines.*.restart_window.days_of_week +2537 resources.pipelines.*.restart_window.days_of_week[*] +2538 resources.pipelines.*.restart_window.start_hour +2539 resources.pipelines.*.restart_window.time_zone_id +2540 resources.pipelines.*.root_path +2541 resources.pipelines.*.run_as +2542 resources.pipelines.*.run_as.group_name +2543 resources.pipelines.*.run_as.service_principal_name +2544 resources.pipelines.*.run_as.user_name +2545 resources.pipelines.*.schema +2546 resources.pipelines.*.serverless +2547 resources.pipelines.*.serverless=true +2548 resources.pipelines.*.serverless=false +2549 resources.pipelines.*.serverless_compute_id +2550 resources.pipelines.*.storage +2551 resources.pipelines.*.tags +2552 resources.pipelines.*.tags.* +2553 resources.pipelines.*.target +2554 resources.pipelines.*.testing +2555 resources.pipelines.*.testing.environment +2556 resources.pipelines.*.testing.environment.dependencies +2557 resources.pipelines.*.testing.environment.dependencies[*] +2558 resources.pipelines.*.testing.sources +2559 resources.pipelines.*.testing.sources[*] +2560 resources.pipelines.*.testing.sources[*].glob +2561 resources.pipelines.*.testing.sources[*].glob.include +2562 resources.pipelines.*.trigger +2563 resources.pipelines.*.trigger.cron +2564 resources.pipelines.*.trigger.cron.quartz_cron_schedule +2565 resources.pipelines.*.trigger.cron.timezone_id +2566 resources.pipelines.*.trigger.manual +2567 resources.pipelines.*.usage_policy_id +2568 resources.pipelines.*.permissions +2569 resources.pipelines.*.permissions[*] +2570 resources.pipelines.*.permissions[*].level +2571 resources.pipelines.*.permissions[*].user_name +2572 resources.pipelines.*.permissions[*].service_principal_name +2573 resources.pipelines.*.permissions[*].group_name +2574 resources.pipelines.*.cascade_on_destroy +2575 resources.pipelines.*.cascade_on_destroy=true +2576 resources.pipelines.*.cascade_on_destroy=false diff --git a/acceptance/bundle/bitmap/full/output/postgres_branches.txt b/acceptance/bundle/bitmap/full/output/postgres_branches.txt new file mode 100644 index 00000000000..16081032884 --- /dev/null +++ b/acceptance/bundle/bitmap/full/output/postgres_branches.txt @@ -0,0 +1,32 @@ +4632 resources.postgres_branches +4633 resources.postgres_branches.* +4634 resources.postgres_branches.*.url +4635 resources.postgres_branches.*.modified_status +4636 resources.postgres_branches.*.lifecycle +4637 resources.postgres_branches.*.lifecycle.prevent_destroy +4638 resources.postgres_branches.*.lifecycle.prevent_destroy=true +4639 resources.postgres_branches.*.lifecycle.prevent_destroy=false +4640 resources.postgres_branches.*.lifecycle.depends_on +4641 resources.postgres_branches.*.lifecycle.depends_on[*] +4642 resources.postgres_branches.*.lifecycle.depends_on[*].resource +4643 resources.postgres_branches.*.id +4644 resources.postgres_branches.*.expire_time +4645 resources.postgres_branches.*.is_protected +4646 resources.postgres_branches.*.is_protected=true +4647 resources.postgres_branches.*.is_protected=false +4648 resources.postgres_branches.*.no_expiry +4649 resources.postgres_branches.*.no_expiry=true +4650 resources.postgres_branches.*.no_expiry=false +4651 resources.postgres_branches.*.source_branch +4652 resources.postgres_branches.*.source_branch_lsn +4653 resources.postgres_branches.*.source_branch_time +4654 resources.postgres_branches.*.source_snapshot +4655 resources.postgres_branches.*.ttl +4656 resources.postgres_branches.*.branch_id +4657 resources.postgres_branches.*.parent +4658 resources.postgres_branches.*.replace_existing +4659 resources.postgres_branches.*.replace_existing=true +4660 resources.postgres_branches.*.replace_existing=false +4661 resources.postgres_branches.*.purge_on_delete +4662 resources.postgres_branches.*.purge_on_delete=true +4663 resources.postgres_branches.*.purge_on_delete=false diff --git a/acceptance/bundle/bitmap/full/output/postgres_catalogs.txt b/acceptance/bundle/bitmap/full/output/postgres_catalogs.txt new file mode 100644 index 00000000000..c5c55fca20f --- /dev/null +++ b/acceptance/bundle/bitmap/full/output/postgres_catalogs.txt @@ -0,0 +1,18 @@ +4700 resources.postgres_catalogs +4701 resources.postgres_catalogs.* +4702 resources.postgres_catalogs.*.url +4703 resources.postgres_catalogs.*.modified_status +4704 resources.postgres_catalogs.*.lifecycle +4705 resources.postgres_catalogs.*.lifecycle.prevent_destroy +4706 resources.postgres_catalogs.*.lifecycle.prevent_destroy=true +4707 resources.postgres_catalogs.*.lifecycle.prevent_destroy=false +4708 resources.postgres_catalogs.*.lifecycle.depends_on +4709 resources.postgres_catalogs.*.lifecycle.depends_on[*] +4710 resources.postgres_catalogs.*.lifecycle.depends_on[*].resource +4711 resources.postgres_catalogs.*.id +4712 resources.postgres_catalogs.*.branch +4713 resources.postgres_catalogs.*.create_database_if_missing +4714 resources.postgres_catalogs.*.create_database_if_missing=true +4715 resources.postgres_catalogs.*.create_database_if_missing=false +4716 resources.postgres_catalogs.*.postgres_database +4717 resources.postgres_catalogs.*.catalog_id diff --git a/acceptance/bundle/bitmap/full/output/postgres_data_apis.txt b/acceptance/bundle/bitmap/full/output/postgres_data_apis.txt new file mode 100644 index 00000000000..69a208be780 --- /dev/null +++ b/acceptance/bundle/bitmap/full/output/postgres_data_apis.txt @@ -0,0 +1,29 @@ +4836 resources.postgres_data_apis +4837 resources.postgres_data_apis.* +4838 resources.postgres_data_apis.*.url +4839 resources.postgres_data_apis.*.modified_status +4840 resources.postgres_data_apis.*.lifecycle +4841 resources.postgres_data_apis.*.lifecycle.prevent_destroy +4842 resources.postgres_data_apis.*.lifecycle.prevent_destroy=true +4843 resources.postgres_data_apis.*.lifecycle.prevent_destroy=false +4844 resources.postgres_data_apis.*.lifecycle.depends_on +4845 resources.postgres_data_apis.*.lifecycle.depends_on[*] +4846 resources.postgres_data_apis.*.lifecycle.depends_on[*].resource +4847 resources.postgres_data_apis.*.id +4848 resources.postgres_data_apis.*.db_aggregates_enabled +4849 resources.postgres_data_apis.*.db_aggregates_enabled=true +4850 resources.postgres_data_apis.*.db_aggregates_enabled=false +4851 resources.postgres_data_apis.*.db_extra_search_path +4852 resources.postgres_data_apis.*.db_extra_search_path[*] +4853 resources.postgres_data_apis.*.db_max_rows +4854 resources.postgres_data_apis.*.db_schemas +4855 resources.postgres_data_apis.*.db_schemas[*] +4856 resources.postgres_data_apis.*.jwt_cache_max_lifetime +4857 resources.postgres_data_apis.*.jwt_role_claim_key +4858 resources.postgres_data_apis.*.openapi_mode +4859 resources.postgres_data_apis.*.server_cors_allowed_origins +4860 resources.postgres_data_apis.*.server_cors_allowed_origins[*] +4861 resources.postgres_data_apis.*.server_timing_enabled +4862 resources.postgres_data_apis.*.server_timing_enabled=true +4863 resources.postgres_data_apis.*.server_timing_enabled=false +4864 resources.postgres_data_apis.*.parent diff --git a/acceptance/bundle/bitmap/full/output/postgres_databases.txt b/acceptance/bundle/bitmap/full/output/postgres_databases.txt new file mode 100644 index 00000000000..c1e31c1e0ef --- /dev/null +++ b/acceptance/bundle/bitmap/full/output/postgres_databases.txt @@ -0,0 +1,19 @@ +4718 resources.postgres_databases +4719 resources.postgres_databases.* +4720 resources.postgres_databases.*.url +4721 resources.postgres_databases.*.modified_status +4722 resources.postgres_databases.*.lifecycle +4723 resources.postgres_databases.*.lifecycle.prevent_destroy +4724 resources.postgres_databases.*.lifecycle.prevent_destroy=true +4725 resources.postgres_databases.*.lifecycle.prevent_destroy=false +4726 resources.postgres_databases.*.lifecycle.depends_on +4727 resources.postgres_databases.*.lifecycle.depends_on[*] +4728 resources.postgres_databases.*.lifecycle.depends_on[*].resource +4729 resources.postgres_databases.*.id +4730 resources.postgres_databases.*.postgres_database +4731 resources.postgres_databases.*.role +4732 resources.postgres_databases.*.database_id +4733 resources.postgres_databases.*.parent +4734 resources.postgres_databases.*.replace_existing +4735 resources.postgres_databases.*.replace_existing=true +4736 resources.postgres_databases.*.replace_existing=false diff --git a/acceptance/bundle/bitmap/full/output/postgres_endpoints.txt b/acceptance/bundle/bitmap/full/output/postgres_endpoints.txt new file mode 100644 index 00000000000..00d7b90ece5 --- /dev/null +++ b/acceptance/bundle/bitmap/full/output/postgres_endpoints.txt @@ -0,0 +1,36 @@ +4664 resources.postgres_endpoints +4665 resources.postgres_endpoints.* +4666 resources.postgres_endpoints.*.url +4667 resources.postgres_endpoints.*.modified_status +4668 resources.postgres_endpoints.*.lifecycle +4669 resources.postgres_endpoints.*.lifecycle.prevent_destroy +4670 resources.postgres_endpoints.*.lifecycle.prevent_destroy=true +4671 resources.postgres_endpoints.*.lifecycle.prevent_destroy=false +4672 resources.postgres_endpoints.*.lifecycle.depends_on +4673 resources.postgres_endpoints.*.lifecycle.depends_on[*] +4674 resources.postgres_endpoints.*.lifecycle.depends_on[*].resource +4675 resources.postgres_endpoints.*.id +4676 resources.postgres_endpoints.*.autoscaling_limit_max_cu +4677 resources.postgres_endpoints.*.autoscaling_limit_min_cu +4678 resources.postgres_endpoints.*.disabled +4679 resources.postgres_endpoints.*.disabled=true +4680 resources.postgres_endpoints.*.disabled=false +4681 resources.postgres_endpoints.*.endpoint_type +4682 resources.postgres_endpoints.*.group +4683 resources.postgres_endpoints.*.group.enable_readable_secondaries +4684 resources.postgres_endpoints.*.group.enable_readable_secondaries=true +4685 resources.postgres_endpoints.*.group.enable_readable_secondaries=false +4686 resources.postgres_endpoints.*.group.max +4687 resources.postgres_endpoints.*.group.min +4688 resources.postgres_endpoints.*.no_suspension +4689 resources.postgres_endpoints.*.no_suspension=true +4690 resources.postgres_endpoints.*.no_suspension=false +4691 resources.postgres_endpoints.*.settings +4692 resources.postgres_endpoints.*.settings.pg_settings +4693 resources.postgres_endpoints.*.settings.pg_settings.* +4694 resources.postgres_endpoints.*.suspend_timeout_duration +4695 resources.postgres_endpoints.*.endpoint_id +4696 resources.postgres_endpoints.*.parent +4697 resources.postgres_endpoints.*.replace_existing +4698 resources.postgres_endpoints.*.replace_existing=true +4699 resources.postgres_endpoints.*.replace_existing=false diff --git a/acceptance/bundle/bitmap/full/output/postgres_projects.txt b/acceptance/bundle/bitmap/full/output/postgres_projects.txt new file mode 100644 index 00000000000..76965b7f2d9 --- /dev/null +++ b/acceptance/bundle/bitmap/full/output/postgres_projects.txt @@ -0,0 +1,43 @@ +4589 resources.postgres_projects +4590 resources.postgres_projects.* +4591 resources.postgres_projects.*.url +4592 resources.postgres_projects.*.modified_status +4593 resources.postgres_projects.*.lifecycle +4594 resources.postgres_projects.*.lifecycle.prevent_destroy +4595 resources.postgres_projects.*.lifecycle.prevent_destroy=true +4596 resources.postgres_projects.*.lifecycle.prevent_destroy=false +4597 resources.postgres_projects.*.lifecycle.depends_on +4598 resources.postgres_projects.*.lifecycle.depends_on[*] +4599 resources.postgres_projects.*.lifecycle.depends_on[*].resource +4600 resources.postgres_projects.*.id +4601 resources.postgres_projects.*.budget_policy_id +4602 resources.postgres_projects.*.custom_tags +4603 resources.postgres_projects.*.custom_tags[*] +4604 resources.postgres_projects.*.custom_tags[*].key +4605 resources.postgres_projects.*.custom_tags[*].value +4606 resources.postgres_projects.*.default_branch +4607 resources.postgres_projects.*.default_endpoint_settings +4608 resources.postgres_projects.*.default_endpoint_settings.autoscaling_limit_max_cu +4609 resources.postgres_projects.*.default_endpoint_settings.autoscaling_limit_min_cu +4610 resources.postgres_projects.*.default_endpoint_settings.no_suspension +4611 resources.postgres_projects.*.default_endpoint_settings.no_suspension=true +4612 resources.postgres_projects.*.default_endpoint_settings.no_suspension=false +4613 resources.postgres_projects.*.default_endpoint_settings.pg_settings +4614 resources.postgres_projects.*.default_endpoint_settings.pg_settings.* +4615 resources.postgres_projects.*.default_endpoint_settings.suspend_timeout_duration +4616 resources.postgres_projects.*.display_name +4617 resources.postgres_projects.*.enable_pg_native_login +4618 resources.postgres_projects.*.enable_pg_native_login=true +4619 resources.postgres_projects.*.enable_pg_native_login=false +4620 resources.postgres_projects.*.history_retention_duration +4621 resources.postgres_projects.*.pg_version +4622 resources.postgres_projects.*.project_id +4623 resources.postgres_projects.*.purge_on_delete +4624 resources.postgres_projects.*.purge_on_delete=true +4625 resources.postgres_projects.*.purge_on_delete=false +4626 resources.postgres_projects.*.permissions +4627 resources.postgres_projects.*.permissions[*] +4628 resources.postgres_projects.*.permissions[*].level +4629 resources.postgres_projects.*.permissions[*].user_name +4630 resources.postgres_projects.*.permissions[*].service_principal_name +4631 resources.postgres_projects.*.permissions[*].group_name diff --git a/acceptance/bundle/bitmap/full/output/postgres_roles.txt b/acceptance/bundle/bitmap/full/output/postgres_roles.txt new file mode 100644 index 00000000000..83c173aff61 --- /dev/null +++ b/acceptance/bundle/bitmap/full/output/postgres_roles.txt @@ -0,0 +1,32 @@ +4737 resources.postgres_roles +4738 resources.postgres_roles.* +4739 resources.postgres_roles.*.url +4740 resources.postgres_roles.*.modified_status +4741 resources.postgres_roles.*.lifecycle +4742 resources.postgres_roles.*.lifecycle.prevent_destroy +4743 resources.postgres_roles.*.lifecycle.prevent_destroy=true +4744 resources.postgres_roles.*.lifecycle.prevent_destroy=false +4745 resources.postgres_roles.*.lifecycle.depends_on +4746 resources.postgres_roles.*.lifecycle.depends_on[*] +4747 resources.postgres_roles.*.lifecycle.depends_on[*].resource +4748 resources.postgres_roles.*.id +4749 resources.postgres_roles.*.attributes +4750 resources.postgres_roles.*.attributes.bypassrls +4751 resources.postgres_roles.*.attributes.bypassrls=true +4752 resources.postgres_roles.*.attributes.bypassrls=false +4753 resources.postgres_roles.*.attributes.createdb +4754 resources.postgres_roles.*.attributes.createdb=true +4755 resources.postgres_roles.*.attributes.createdb=false +4756 resources.postgres_roles.*.attributes.createrole +4757 resources.postgres_roles.*.attributes.createrole=true +4758 resources.postgres_roles.*.attributes.createrole=false +4759 resources.postgres_roles.*.auth_method +4760 resources.postgres_roles.*.identity_type +4761 resources.postgres_roles.*.membership_roles +4762 resources.postgres_roles.*.membership_roles[*] +4763 resources.postgres_roles.*.postgres_role +4764 resources.postgres_roles.*.role_id +4765 resources.postgres_roles.*.parent +4766 resources.postgres_roles.*.replace_existing +4767 resources.postgres_roles.*.replace_existing=true +4768 resources.postgres_roles.*.replace_existing=false diff --git a/acceptance/bundle/bitmap/full/output/postgres_snapshot_schedules.txt b/acceptance/bundle/bitmap/full/output/postgres_snapshot_schedules.txt new file mode 100644 index 00000000000..4cb3bae9d4f --- /dev/null +++ b/acceptance/bundle/bitmap/full/output/postgres_snapshot_schedules.txt @@ -0,0 +1,24 @@ +4812 resources.postgres_snapshot_schedules +4813 resources.postgres_snapshot_schedules.* +4814 resources.postgres_snapshot_schedules.*.url +4815 resources.postgres_snapshot_schedules.*.modified_status +4816 resources.postgres_snapshot_schedules.*.lifecycle +4817 resources.postgres_snapshot_schedules.*.lifecycle.prevent_destroy +4818 resources.postgres_snapshot_schedules.*.lifecycle.prevent_destroy=true +4819 resources.postgres_snapshot_schedules.*.lifecycle.prevent_destroy=false +4820 resources.postgres_snapshot_schedules.*.lifecycle.depends_on +4821 resources.postgres_snapshot_schedules.*.lifecycle.depends_on[*] +4822 resources.postgres_snapshot_schedules.*.lifecycle.depends_on[*].resource +4823 resources.postgres_snapshot_schedules.*.id +4824 resources.postgres_snapshot_schedules.*.branch +4825 resources.postgres_snapshot_schedules.*.schedule +4826 resources.postgres_snapshot_schedules.*.schedule[*] +4827 resources.postgres_snapshot_schedules.*.schedule[*].daily_schedule +4828 resources.postgres_snapshot_schedules.*.schedule[*].daily_schedule.hour +4829 resources.postgres_snapshot_schedules.*.schedule[*].monthly_schedule +4830 resources.postgres_snapshot_schedules.*.schedule[*].monthly_schedule.day +4831 resources.postgres_snapshot_schedules.*.schedule[*].monthly_schedule.hour +4832 resources.postgres_snapshot_schedules.*.schedule[*].retention +4833 resources.postgres_snapshot_schedules.*.schedule[*].weekly_schedule +4834 resources.postgres_snapshot_schedules.*.schedule[*].weekly_schedule.day_of_week +4835 resources.postgres_snapshot_schedules.*.schedule[*].weekly_schedule.hour diff --git a/acceptance/bundle/bitmap/full/output/postgres_synced_tables.txt b/acceptance/bundle/bitmap/full/output/postgres_synced_tables.txt new file mode 100644 index 00000000000..50c60c5e18d --- /dev/null +++ b/acceptance/bundle/bitmap/full/output/postgres_synced_tables.txt @@ -0,0 +1,43 @@ +4769 resources.postgres_synced_tables +4770 resources.postgres_synced_tables.* +4771 resources.postgres_synced_tables.*.url +4772 resources.postgres_synced_tables.*.modified_status +4773 resources.postgres_synced_tables.*.lifecycle +4774 resources.postgres_synced_tables.*.lifecycle.prevent_destroy +4775 resources.postgres_synced_tables.*.lifecycle.prevent_destroy=true +4776 resources.postgres_synced_tables.*.lifecycle.prevent_destroy=false +4777 resources.postgres_synced_tables.*.lifecycle.depends_on +4778 resources.postgres_synced_tables.*.lifecycle.depends_on[*] +4779 resources.postgres_synced_tables.*.lifecycle.depends_on[*].resource +4780 resources.postgres_synced_tables.*.id +4781 resources.postgres_synced_tables.*.accelerated_sync +4782 resources.postgres_synced_tables.*.accelerated_sync=true +4783 resources.postgres_synced_tables.*.accelerated_sync=false +4784 resources.postgres_synced_tables.*.branch +4785 resources.postgres_synced_tables.*.create_database_objects_if_missing +4786 resources.postgres_synced_tables.*.create_database_objects_if_missing=true +4787 resources.postgres_synced_tables.*.create_database_objects_if_missing=false +4788 resources.postgres_synced_tables.*.existing_pipeline_id +4789 resources.postgres_synced_tables.*.extra_columns +4790 resources.postgres_synced_tables.*.extra_columns[*] +4791 resources.postgres_synced_tables.*.extra_columns[*].column_name +4792 resources.postgres_synced_tables.*.extra_columns[*].column_type +4793 resources.postgres_synced_tables.*.extra_columns[*].compute +4794 resources.postgres_synced_tables.*.extra_columns[*].maintenance +4795 resources.postgres_synced_tables.*.new_pipeline_spec +4796 resources.postgres_synced_tables.*.new_pipeline_spec.budget_policy_id +4797 resources.postgres_synced_tables.*.new_pipeline_spec.pipeline_channel +4798 resources.postgres_synced_tables.*.new_pipeline_spec.storage_catalog +4799 resources.postgres_synced_tables.*.new_pipeline_spec.storage_schema +4800 resources.postgres_synced_tables.*.postgres_database +4801 resources.postgres_synced_tables.*.primary_key_columns +4802 resources.postgres_synced_tables.*.primary_key_columns[*] +4803 resources.postgres_synced_tables.*.scheduling_policy +4804 resources.postgres_synced_tables.*.source_table_full_name +4805 resources.postgres_synced_tables.*.timeseries_key +4806 resources.postgres_synced_tables.*.type_overrides +4807 resources.postgres_synced_tables.*.type_overrides[*] +4808 resources.postgres_synced_tables.*.type_overrides[*].column_name +4809 resources.postgres_synced_tables.*.type_overrides[*].pg_type +4810 resources.postgres_synced_tables.*.type_overrides[*].size +4811 resources.postgres_synced_tables.*.synced_table_id diff --git a/acceptance/bundle/bitmap/full/output/quality_monitors.txt b/acceptance/bundle/bitmap/full/output/quality_monitors.txt new file mode 100644 index 00000000000..3cb7d8f3c87 --- /dev/null +++ b/acceptance/bundle/bitmap/full/output/quality_monitors.txt @@ -0,0 +1,60 @@ +3457 resources.quality_monitors +3458 resources.quality_monitors.* +3459 resources.quality_monitors.*.url +3460 resources.quality_monitors.*.modified_status +3461 resources.quality_monitors.*.lifecycle +3462 resources.quality_monitors.*.lifecycle.prevent_destroy +3463 resources.quality_monitors.*.lifecycle.prevent_destroy=true +3464 resources.quality_monitors.*.lifecycle.prevent_destroy=false +3465 resources.quality_monitors.*.lifecycle.depends_on +3466 resources.quality_monitors.*.lifecycle.depends_on[*] +3467 resources.quality_monitors.*.lifecycle.depends_on[*].resource +3468 resources.quality_monitors.*.id +3469 resources.quality_monitors.*.assets_dir +3470 resources.quality_monitors.*.baseline_table_name +3471 resources.quality_monitors.*.custom_metrics +3472 resources.quality_monitors.*.custom_metrics[*] +3473 resources.quality_monitors.*.custom_metrics[*].definition +3474 resources.quality_monitors.*.custom_metrics[*].input_columns +3475 resources.quality_monitors.*.custom_metrics[*].input_columns[*] +3476 resources.quality_monitors.*.custom_metrics[*].name +3477 resources.quality_monitors.*.custom_metrics[*].output_data_type +3478 resources.quality_monitors.*.custom_metrics[*].type +3479 resources.quality_monitors.*.data_classification_config +3480 resources.quality_monitors.*.data_classification_config.enabled +3481 resources.quality_monitors.*.data_classification_config.enabled=true +3482 resources.quality_monitors.*.data_classification_config.enabled=false +3483 resources.quality_monitors.*.inference_log +3484 resources.quality_monitors.*.inference_log.granularities +3485 resources.quality_monitors.*.inference_log.granularities[*] +3486 resources.quality_monitors.*.inference_log.label_col +3487 resources.quality_monitors.*.inference_log.model_id_col +3488 resources.quality_monitors.*.inference_log.prediction_col +3489 resources.quality_monitors.*.inference_log.prediction_proba_col +3490 resources.quality_monitors.*.inference_log.problem_type +3491 resources.quality_monitors.*.inference_log.timestamp_col +3492 resources.quality_monitors.*.latest_monitor_failure_msg +3493 resources.quality_monitors.*.notifications +3494 resources.quality_monitors.*.notifications.on_failure +3495 resources.quality_monitors.*.notifications.on_failure.email_addresses +3496 resources.quality_monitors.*.notifications.on_failure.email_addresses[*] +3497 resources.quality_monitors.*.notifications.on_new_classification_tag_detected +3498 resources.quality_monitors.*.notifications.on_new_classification_tag_detected.email_addresses +3499 resources.quality_monitors.*.notifications.on_new_classification_tag_detected.email_addresses[*] +3500 resources.quality_monitors.*.output_schema_name +3501 resources.quality_monitors.*.schedule +3502 resources.quality_monitors.*.schedule.pause_status +3503 resources.quality_monitors.*.schedule.quartz_cron_expression +3504 resources.quality_monitors.*.schedule.timezone_id +3505 resources.quality_monitors.*.skip_builtin_dashboard +3506 resources.quality_monitors.*.skip_builtin_dashboard=true +3507 resources.quality_monitors.*.skip_builtin_dashboard=false +3508 resources.quality_monitors.*.slicing_exprs +3509 resources.quality_monitors.*.slicing_exprs[*] +3510 resources.quality_monitors.*.snapshot +3511 resources.quality_monitors.*.time_series +3512 resources.quality_monitors.*.time_series.granularities +3513 resources.quality_monitors.*.time_series.granularities[*] +3514 resources.quality_monitors.*.time_series.timestamp_col +3515 resources.quality_monitors.*.warehouse_id +3516 resources.quality_monitors.*.table_name diff --git a/acceptance/bundle/bitmap/full/output/registered_models.txt b/acceptance/bundle/bitmap/full/output/registered_models.txt new file mode 100644 index 00000000000..45816aebb52 --- /dev/null +++ b/acceptance/bundle/bitmap/full/output/registered_models.txt @@ -0,0 +1,40 @@ +3417 resources.registered_models +3418 resources.registered_models.* +3419 resources.registered_models.*.url +3420 resources.registered_models.*.modified_status +3421 resources.registered_models.*.lifecycle +3422 resources.registered_models.*.lifecycle.prevent_destroy +3423 resources.registered_models.*.lifecycle.prevent_destroy=true +3424 resources.registered_models.*.lifecycle.prevent_destroy=false +3425 resources.registered_models.*.lifecycle.depends_on +3426 resources.registered_models.*.lifecycle.depends_on[*] +3427 resources.registered_models.*.lifecycle.depends_on[*].resource +3428 resources.registered_models.*.id +3429 resources.registered_models.*.aliases +3430 resources.registered_models.*.aliases[*] +3431 resources.registered_models.*.aliases[*].alias_name +3432 resources.registered_models.*.aliases[*].catalog_name +3433 resources.registered_models.*.aliases[*].id +3434 resources.registered_models.*.aliases[*].model_name +3435 resources.registered_models.*.aliases[*].schema_name +3436 resources.registered_models.*.aliases[*].version_num +3437 resources.registered_models.*.browse_only +3438 resources.registered_models.*.browse_only=true +3439 resources.registered_models.*.browse_only=false +3440 resources.registered_models.*.catalog_name +3441 resources.registered_models.*.comment +3442 resources.registered_models.*.created_at +3443 resources.registered_models.*.created_by +3444 resources.registered_models.*.full_name +3445 resources.registered_models.*.metastore_id +3446 resources.registered_models.*.name +3447 resources.registered_models.*.owner +3448 resources.registered_models.*.schema_name +3449 resources.registered_models.*.storage_location +3450 resources.registered_models.*.updated_at +3451 resources.registered_models.*.updated_by +3452 resources.registered_models.*.grants +3453 resources.registered_models.*.grants[*] +3454 resources.registered_models.*.grants[*].principal +3455 resources.registered_models.*.grants[*].privileges +3456 resources.registered_models.*.grants[*].privileges[*] diff --git a/acceptance/bundle/bitmap/full/output/root.txt b/acceptance/bundle/bitmap/full/output/root.txt new file mode 100644 index 00000000000..efd611fabf3 --- /dev/null +++ b/acceptance/bundle/bitmap/full/output/root.txt @@ -0,0 +1,369 @@ + 0 variables + 1 variables.* + 2 variables.*.type + 3 variables.*.default + 4 variables.*.description + 5 variables.*.sensitive + 6 variables.*.sensitive=true + 7 variables.*.sensitive=false + 8 variables.*.value + 9 variables.*.lookup + 10 variables.*.lookup.alert + 11 variables.*.lookup.cluster_policy + 12 variables.*.lookup.cluster + 13 variables.*.lookup.dashboard + 14 variables.*.lookup.instance_pool + 15 variables.*.lookup.job + 16 variables.*.lookup.metastore + 17 variables.*.lookup.notification_destination + 18 variables.*.lookup.pipeline + 19 variables.*.lookup.query + 20 variables.*.lookup.service_principal + 21 variables.*.lookup.warehouse + 22 bundle + 23 bundle.name + 24 bundle.terraform + 25 bundle.terraform.exec_path + 26 bundle.force + 27 bundle.force=true + 28 bundle.force=false + 29 bundle.git + 30 bundle.git.branch + 31 bundle.git.origin_url + 32 bundle.git.commit + 33 bundle.git.bundle_root_path + 34 bundle.git.actual_branch + 35 bundle.mode + 36 bundle.compute_id + 37 bundle.cluster_id + 38 bundle.engine + 39 bundle.deployment + 40 bundle.deployment.fail_on_active_runs + 41 bundle.deployment.fail_on_active_runs=true + 42 bundle.deployment.fail_on_active_runs=false + 43 bundle.deployment.lock + 44 bundle.deployment.lock.enabled + 45 bundle.deployment.lock.enabled=true + 46 bundle.deployment.lock.enabled=false + 47 bundle.deployment.lock.force + 48 bundle.deployment.lock.force=true + 49 bundle.deployment.lock.force=false + 50 bundle.deployment.deployment_id + 51 bundle.deployment.latest_version_id + 52 bundle.databricks_cli_version + 53 bundle.uuid + 54 include + 55 include[*] + 56 workspace + 57 workspace.host + 58 workspace.profile + 59 workspace.auth_type + 60 workspace.metadata_service_url + 61 workspace.client_id + 62 workspace.google_service_account + 63 workspace.azure_workspace_resource_id + 64 workspace.azure_use_msi + 65 workspace.azure_use_msi=true + 66 workspace.azure_use_msi=false + 67 workspace.azure_client_id + 68 workspace.azure_tenant_id + 69 workspace.azure_environment + 70 workspace.azure_login_app_id + 71 workspace.experimental_is_unified_host + 72 workspace.experimental_is_unified_host=true + 73 workspace.experimental_is_unified_host=false + 74 workspace.account_id + 75 workspace.workspace_id + 76 workspace.current_user + 77 workspace.current_user.short_name + 78 workspace.current_user.domain_friendly_name + 79 workspace.current_user.active + 80 workspace.current_user.active=true + 81 workspace.current_user.active=false + 82 workspace.current_user.displayName + 83 workspace.current_user.emails + 84 workspace.current_user.emails[*] + 85 workspace.current_user.emails[*].display + 86 workspace.current_user.emails[*].primary + 87 workspace.current_user.emails[*].primary=true + 88 workspace.current_user.emails[*].primary=false + 89 workspace.current_user.emails[*].$ref + 90 workspace.current_user.emails[*].type + 91 workspace.current_user.emails[*].value + 92 workspace.current_user.entitlements + 93 workspace.current_user.entitlements[*] + 94 workspace.current_user.entitlements[*].display + 95 workspace.current_user.entitlements[*].primary + 96 workspace.current_user.entitlements[*].primary=true + 97 workspace.current_user.entitlements[*].primary=false + 98 workspace.current_user.entitlements[*].$ref + 99 workspace.current_user.entitlements[*].type + 100 workspace.current_user.entitlements[*].value + 101 workspace.current_user.externalId + 102 workspace.current_user.groups + 103 workspace.current_user.groups[*] + 104 workspace.current_user.groups[*].display + 105 workspace.current_user.groups[*].primary + 106 workspace.current_user.groups[*].primary=true + 107 workspace.current_user.groups[*].primary=false + 108 workspace.current_user.groups[*].$ref + 109 workspace.current_user.groups[*].type + 110 workspace.current_user.groups[*].value + 111 workspace.current_user.id + 112 workspace.current_user.name + 113 workspace.current_user.name.familyName + 114 workspace.current_user.name.givenName + 115 workspace.current_user.roles + 116 workspace.current_user.roles[*] + 117 workspace.current_user.roles[*].display + 118 workspace.current_user.roles[*].primary + 119 workspace.current_user.roles[*].primary=true + 120 workspace.current_user.roles[*].primary=false + 121 workspace.current_user.roles[*].$ref + 122 workspace.current_user.roles[*].type + 123 workspace.current_user.roles[*].value + 124 workspace.current_user.schemas + 125 workspace.current_user.schemas[*] + 126 workspace.current_user.userName + 127 workspace.root_path + 128 workspace.file_path + 129 workspace.resource_path + 130 workspace.artifact_path + 131 workspace.state_path + 132 workspace.sync_mode + 133 artifacts + 134 artifacts.* + 135 artifacts.*.type + 136 artifacts.*.path + 137 artifacts.*.files + 138 artifacts.*.files[*] + 139 artifacts.*.files[*].source + 140 artifacts.*.files[*].patched + 141 artifacts.*.files[*].remote_path + 142 artifacts.*.build + 143 artifacts.*.executable + 144 artifacts.*.dynamic_version + 145 artifacts.*.dynamic_version=true + 146 artifacts.*.dynamic_version=false + 147 artifacts.*.include + 148 artifacts.*.include[*] + 149 artifacts.*.git + 150 artifacts.*.git.branch + 151 artifacts.*.git.commit + 152 resources +5090 sync +5091 sync.paths +5092 sync.paths[*] +5093 sync.include +5094 sync.include[*] +5095 sync.exclude +5096 sync.exclude[*] +5097 run_as +5098 run_as.group_name +5099 run_as.service_principal_name +5100 run_as.user_name +5101 presets +5102 presets.name_prefix +5103 presets.pipelines_development +5104 presets.pipelines_development=true +5105 presets.pipelines_development=false +5106 presets.trigger_pause_status +5107 presets.jobs_max_concurrent_runs +5108 presets.source_linked_deployment +5109 presets.source_linked_deployment=true +5110 presets.source_linked_deployment=false +5111 presets.artifacts_dynamic_version +5112 presets.artifacts_dynamic_version=true +5113 presets.artifacts_dynamic_version=false +5114 presets.tags +5115 presets.tags.* +5116 experimental +5117 experimental.scripts +5118 experimental.scripts.* +5119 experimental.immutable_folder +5120 experimental.immutable_folder=true +5121 experimental.immutable_folder=false +5122 experimental.python_wheel_wrapper +5123 experimental.python_wheel_wrapper=true +5124 experimental.python_wheel_wrapper=false +5125 experimental.use_legacy_run_as +5126 experimental.use_legacy_run_as=true +5127 experimental.use_legacy_run_as=false +5128 experimental.pydabs +5129 experimental.pydabs.enabled +5130 experimental.pydabs.enabled=true +5131 experimental.pydabs.enabled=false +5132 experimental.python +5133 experimental.python.resources +5134 experimental.python.resources[*] +5135 experimental.python.mutators +5136 experimental.python.mutators[*] +5137 experimental.python.venv_path +5138 experimental.skip_artifact_cleanup +5139 experimental.skip_artifact_cleanup=true +5140 experimental.skip_artifact_cleanup=false +5141 experimental.skip_name_prefix_for_schema +5142 experimental.skip_name_prefix_for_schema=true +5143 experimental.skip_name_prefix_for_schema=false +5144 experimental.deployment_history +5145 experimental.deployment_history=true +5146 experimental.deployment_history=false +5147 permissions +5148 permissions[*] +5149 permissions[*].level +5150 permissions[*].user_name +5151 permissions[*].service_principal_name +5152 permissions[*].group_name +5153 scripts +5154 scripts.* +5155 scripts.*.content +5156 scripts.*.env +5157 scripts.*.env.* +5158 python +5159 python.resources +5160 python.resources[*] +5161 python.mutators +5162 python.mutators[*] +5163 python.venv_path +5164 definitions +5165 telemetry.PythonWheelWrapperIsSet +5166 telemetry.PythonWheelWrapperIsSet=true +5167 telemetry.PythonWheelWrapperIsSet=false +5168 telemetry.ArtifactDynamicVersionIsSet +5169 telemetry.ArtifactDynamicVersionIsSet=true +5170 telemetry.ArtifactDynamicVersionIsSet=false +5171 telemetry.ArtifactBuildCommandIsSet +5172 telemetry.ArtifactBuildCommandIsSet=true +5173 telemetry.ArtifactBuildCommandIsSet=false +5174 telemetry.ArtifactFilesIsSet +5175 telemetry.ArtifactFilesIsSet=true +5176 telemetry.ArtifactFilesIsSet=false +5177 telemetry.PresetsNamePrefixIsSet +5178 telemetry.PresetsNamePrefixIsSet=true +5179 telemetry.PresetsNamePrefixIsSet=false +5180 telemetry.AppLifecycleStarted +5181 telemetry.AppLifecycleStarted=true +5182 telemetry.AppLifecycleStarted=false +5183 telemetry.ClusterLifecycleStarted +5184 telemetry.ClusterLifecycleStarted=true +5185 telemetry.ClusterLifecycleStarted=false +5186 telemetry.DeploymentHistoryEnabled +5187 telemetry.DeploymentHistoryEnabled=true +5188 telemetry.DeploymentHistoryEnabled=false +5189 telemetry.SqlWarehouseLifecycleStarted +5190 telemetry.SqlWarehouseLifecycleStarted=true +5191 telemetry.SqlWarehouseLifecycleStarted=false +5192 telemetry.SelectUsed +5193 telemetry.SelectUsed=true +5194 telemetry.SelectUsed=false +5195 telemetry.ImmutableSnapshotUsed +5196 telemetry.ImmutableSnapshotUsed=true +5197 telemetry.ImmutableSnapshotUsed=false +5198 telemetry.DirectMigrateError +5199 telemetry.DirectMigrateError=true +5200 telemetry.DirectMigrateError=false +5201 telemetry.DirectMigrateWarnings +5202 telemetry.DirectMigrateWarnings=true +5203 telemetry.DirectMigrateWarnings=false +5204 telemetry.DirectMigratePlanError +5205 telemetry.DirectMigratePlanError=true +5206 telemetry.DirectMigratePlanError=false +5207 telemetry.DirectMigrateRecreatePlanned +5208 telemetry.DirectMigrateRecreatePlanned=true +5209 telemetry.DirectMigrateRecreatePlanned=false +5210 telemetry.DirectAutoMigrateViaConfig +5211 telemetry.DirectAutoMigrateViaConfig=true +5212 telemetry.DirectAutoMigrateViaConfig=false +5213 telemetry.DirectAutoMigrateViaEnv +5214 telemetry.DirectAutoMigrateViaEnv=true +5215 telemetry.DirectAutoMigrateViaEnv=false +5216 telemetry.DirectAutoMigrateViaDefault +5217 telemetry.DirectAutoMigrateViaDefault=true +5218 telemetry.DirectAutoMigrateViaDefault=false +5219 telemetry.EngineTerraformConfig +5220 telemetry.EngineTerraformConfig=true +5221 telemetry.EngineTerraformConfig=false +5222 telemetry.EngineTerraformEnv +5223 telemetry.EngineTerraformEnv=true +5224 telemetry.EngineTerraformEnv=false +5225 telemetry.StatePathIsShared +5226 telemetry.StatePathIsShared=true +5227 telemetry.StatePathIsShared=false +5228 telemetry.DMSCompatAuto +5229 telemetry.DMSCompatAuto=true +5230 telemetry.DMSCompatAuto=false +5231 telemetry.DMSCompatOnlySelfUndeclared +5232 telemetry.DMSCompatOnlySelfUndeclared=true +5233 telemetry.DMSCompatOnlySelfUndeclared=false +5234 telemetry.DMSCompatNot +5235 telemetry.DMSCompatNot=true +5236 telemetry.DMSCompatNot=false +5237 telemetry.PermissionsSectionSet +5238 telemetry.PermissionsSectionSet=true +5239 telemetry.PermissionsSectionSet=false +5240 telemetry.StatePathInDeployerHome +5241 telemetry.StatePathInDeployerHome=true +5242 telemetry.StatePathInDeployerHome=false +5243 telemetry.StatePathInOtherUserHome +5244 telemetry.StatePathInOtherUserHome=true +5245 telemetry.StatePathInOtherUserHome=false +5246 telemetry.StatePathOther +5247 telemetry.StatePathOther=true +5248 telemetry.StatePathOther=false +5249 telemetry.DMSUndeclaredDeployingUser +5250 telemetry.DMSUndeclaredDeployingUser=true +5251 telemetry.DMSUndeclaredDeployingUser=false +5252 telemetry.DMSUndeclaredOtherUser +5253 telemetry.DMSUndeclaredOtherUser=true +5254 telemetry.DMSUndeclaredOtherUser=false +5255 telemetry.DMSUndeclaredServicePrincipal +5256 telemetry.DMSUndeclaredServicePrincipal=true +5257 telemetry.DMSUndeclaredServicePrincipal=false +5258 telemetry.DMSUndeclaredGroup +5259 telemetry.DMSUndeclaredGroup=true +5260 telemetry.DMSUndeclaredGroup=false +5261 telemetry.AiRuntimeTaskScheduled +5262 telemetry.AiRuntimeTaskScheduled=true +5263 telemetry.AiRuntimeTaskScheduled=false +5264 telemetry.AiRuntimeTaskMultitask +5265 telemetry.AiRuntimeTaskMultitask=true +5266 telemetry.AiRuntimeTaskMultitask=false +5267 telemetry.SkipArtifactCleanup +5268 telemetry.SkipArtifactCleanup=true +5269 telemetry.SkipArtifactCleanup=false +5270 telemetry.SourceLinkedSetForNonDevelopment +5271 telemetry.SourceLinkedSetForNonDevelopment=true +5272 telemetry.SourceLinkedSetForNonDevelopment=false +5273 telemetry.ConfigHasDoubleDollarBrace +5274 telemetry.ConfigHasDoubleDollarBrace=true +5275 telemetry.ConfigHasDoubleDollarBrace=false +5276 telemetry.ConfigHasDoubleDollar +5277 telemetry.ConfigHasDoubleDollar=true +5278 telemetry.ConfigHasDoubleDollar=false +5279 telemetry.ConfigHasBackslashDollarBrace +5280 telemetry.ConfigHasBackslashDollarBrace=true +5281 telemetry.ConfigHasBackslashDollarBrace=false +5282 telemetry.ConfigHasBackslashDollar +5283 telemetry.ConfigHasBackslashDollar=true +5284 telemetry.ConfigHasBackslashDollar=false +5285 telemetry.ArtifactsReferenceUsed +5286 telemetry.ArtifactsReferenceUsed=true +5287 telemetry.ArtifactsReferenceUsed=false +5288 telemetry.UseLegacyRunAs +5289 telemetry.UseLegacyRunAs=true +5290 telemetry.UseLegacyRunAs=false +5291 telemetry.RunAsSet +5292 telemetry.RunAsSet=true +5293 telemetry.RunAsSet=false +5294 telemetry.HasTfOnlyReferences +5295 telemetry.HasTfOnlyReferences=true +5296 telemetry.HasTfOnlyReferences=false +5297 telemetry.HasServerlessCompute +5298 telemetry.HasServerlessCompute=true +5299 telemetry.HasServerlessCompute=false +5300 telemetry.HasClassicJobCompute +5301 telemetry.HasClassicJobCompute=true +5302 telemetry.HasClassicJobCompute=false +5303 telemetry.HasClassicInteractiveCompute +5304 telemetry.HasClassicInteractiveCompute=true +5305 telemetry.HasClassicInteractiveCompute=false diff --git a/acceptance/bundle/bitmap/full/output/schemas.txt b/acceptance/bundle/bitmap/full/output/schemas.txt new file mode 100644 index 00000000000..6c3b85aa10e --- /dev/null +++ b/acceptance/bundle/bitmap/full/output/schemas.txt @@ -0,0 +1,24 @@ +3643 resources.schemas +3644 resources.schemas.* +3645 resources.schemas.*.url +3646 resources.schemas.*.modified_status +3647 resources.schemas.*.lifecycle +3648 resources.schemas.*.lifecycle.prevent_destroy +3649 resources.schemas.*.lifecycle.prevent_destroy=true +3650 resources.schemas.*.lifecycle.prevent_destroy=false +3651 resources.schemas.*.lifecycle.depends_on +3652 resources.schemas.*.lifecycle.depends_on[*] +3653 resources.schemas.*.lifecycle.depends_on[*].resource +3654 resources.schemas.*.id +3655 resources.schemas.*.catalog_name +3656 resources.schemas.*.comment +3657 resources.schemas.*.custom_max_retention_hours +3658 resources.schemas.*.name +3659 resources.schemas.*.properties +3660 resources.schemas.*.properties.* +3661 resources.schemas.*.storage_root +3662 resources.schemas.*.grants +3663 resources.schemas.*.grants[*] +3664 resources.schemas.*.grants[*].principal +3665 resources.schemas.*.grants[*].privileges +3666 resources.schemas.*.grants[*].privileges[*] diff --git a/acceptance/bundle/bitmap/full/output/secret_scopes.txt b/acceptance/bundle/bitmap/full/output/secret_scopes.txt new file mode 100644 index 00000000000..6473a8ad405 --- /dev/null +++ b/acceptance/bundle/bitmap/full/output/secret_scopes.txt @@ -0,0 +1,23 @@ +4270 resources.secret_scopes +4271 resources.secret_scopes.* +4272 resources.secret_scopes.*.url +4273 resources.secret_scopes.*.modified_status +4274 resources.secret_scopes.*.lifecycle +4275 resources.secret_scopes.*.lifecycle.prevent_destroy +4276 resources.secret_scopes.*.lifecycle.prevent_destroy=true +4277 resources.secret_scopes.*.lifecycle.prevent_destroy=false +4278 resources.secret_scopes.*.lifecycle.depends_on +4279 resources.secret_scopes.*.lifecycle.depends_on[*] +4280 resources.secret_scopes.*.lifecycle.depends_on[*].resource +4281 resources.secret_scopes.*.id +4282 resources.secret_scopes.*.name +4283 resources.secret_scopes.*.permissions +4284 resources.secret_scopes.*.permissions[*] +4285 resources.secret_scopes.*.permissions[*].level +4286 resources.secret_scopes.*.permissions[*].user_name +4287 resources.secret_scopes.*.permissions[*].service_principal_name +4288 resources.secret_scopes.*.permissions[*].group_name +4289 resources.secret_scopes.*.backend_type +4290 resources.secret_scopes.*.keyvault_metadata +4291 resources.secret_scopes.*.keyvault_metadata.dns_name +4292 resources.secret_scopes.*.keyvault_metadata.resource_id diff --git a/acceptance/bundle/bitmap/full/output/secrets.txt b/acceptance/bundle/bitmap/full/output/secrets.txt new file mode 100644 index 00000000000..fd47fc92a78 --- /dev/null +++ b/acceptance/bundle/bitmap/full/output/secrets.txt @@ -0,0 +1,32 @@ +5001 resources.secrets +5002 resources.secrets.* +5003 resources.secrets.*.url +5004 resources.secrets.*.modified_status +5005 resources.secrets.*.lifecycle +5006 resources.secrets.*.lifecycle.prevent_destroy +5007 resources.secrets.*.lifecycle.prevent_destroy=true +5008 resources.secrets.*.lifecycle.prevent_destroy=false +5009 resources.secrets.*.lifecycle.depends_on +5010 resources.secrets.*.lifecycle.depends_on[*] +5011 resources.secrets.*.lifecycle.depends_on[*].resource +5012 resources.secrets.*.id +5013 resources.secrets.*.catalog_name +5014 resources.secrets.*.comment +5015 resources.secrets.*.create_time +5016 resources.secrets.*.created_by +5017 resources.secrets.*.effective_owner +5018 resources.secrets.*.effective_value +5019 resources.secrets.*.expire_time +5020 resources.secrets.*.full_name +5021 resources.secrets.*.metastore_id +5022 resources.secrets.*.name +5023 resources.secrets.*.owner +5024 resources.secrets.*.schema_name +5025 resources.secrets.*.update_time +5026 resources.secrets.*.updated_by +5027 resources.secrets.*.value +5028 resources.secrets.*.grants +5029 resources.secrets.*.grants[*] +5030 resources.secrets.*.grants[*].principal +5031 resources.secrets.*.grants[*].privileges +5032 resources.secrets.*.grants[*].privileges[*] diff --git a/acceptance/bundle/bitmap/full/output/sql_warehouses.txt b/acceptance/bundle/bitmap/full/output/sql_warehouses.txt new file mode 100644 index 00000000000..0464771e7aa --- /dev/null +++ b/acceptance/bundle/bitmap/full/output/sql_warehouses.txt @@ -0,0 +1,48 @@ +4368 resources.sql_warehouses +4369 resources.sql_warehouses.* +4370 resources.sql_warehouses.*.url +4371 resources.sql_warehouses.*.modified_status +4372 resources.sql_warehouses.*.lifecycle +4373 resources.sql_warehouses.*.lifecycle.prevent_destroy +4374 resources.sql_warehouses.*.lifecycle.prevent_destroy=true +4375 resources.sql_warehouses.*.lifecycle.prevent_destroy=false +4376 resources.sql_warehouses.*.lifecycle.depends_on +4377 resources.sql_warehouses.*.lifecycle.depends_on[*] +4378 resources.sql_warehouses.*.lifecycle.depends_on[*].resource +4379 resources.sql_warehouses.*.id +4380 resources.sql_warehouses.*.auto_stop_mins +4381 resources.sql_warehouses.*.channel +4382 resources.sql_warehouses.*.channel.dbsql_version +4383 resources.sql_warehouses.*.channel.name +4384 resources.sql_warehouses.*.cluster_size +4385 resources.sql_warehouses.*.creator_name +4386 resources.sql_warehouses.*.customer_version_label +4387 resources.sql_warehouses.*.enable_photon +4388 resources.sql_warehouses.*.enable_photon=true +4389 resources.sql_warehouses.*.enable_photon=false +4390 resources.sql_warehouses.*.enable_serverless_compute +4391 resources.sql_warehouses.*.enable_serverless_compute=true +4392 resources.sql_warehouses.*.enable_serverless_compute=false +4393 resources.sql_warehouses.*.instance_profile_arn +4394 resources.sql_warehouses.*.max_dbu +4395 resources.sql_warehouses.*.max_num_clusters +4396 resources.sql_warehouses.*.min_num_clusters +4397 resources.sql_warehouses.*.name +4398 resources.sql_warehouses.*.reyden_cluster_size_multiplier +4399 resources.sql_warehouses.*.spot_instance_policy +4400 resources.sql_warehouses.*.statement_timeout +4401 resources.sql_warehouses.*.tags +4402 resources.sql_warehouses.*.tags.custom_tags +4403 resources.sql_warehouses.*.tags.custom_tags[*] +4404 resources.sql_warehouses.*.tags.custom_tags[*].key +4405 resources.sql_warehouses.*.tags.custom_tags[*].value +4406 resources.sql_warehouses.*.warehouse_type +4407 resources.sql_warehouses.*.lifecycle.started +4408 resources.sql_warehouses.*.lifecycle.started=true +4409 resources.sql_warehouses.*.lifecycle.started=false +4410 resources.sql_warehouses.*.permissions +4411 resources.sql_warehouses.*.permissions[*] +4412 resources.sql_warehouses.*.permissions[*].level +4413 resources.sql_warehouses.*.permissions[*].user_name +4414 resources.sql_warehouses.*.permissions[*].service_principal_name +4415 resources.sql_warehouses.*.permissions[*].group_name diff --git a/acceptance/bundle/bitmap/full/output/synced_database_tables.txt b/acceptance/bundle/bitmap/full/output/synced_database_tables.txt new file mode 100644 index 00000000000..c5ddd5c530f --- /dev/null +++ b/acceptance/bundle/bitmap/full/output/synced_database_tables.txt @@ -0,0 +1,81 @@ +4508 resources.synced_database_tables +4509 resources.synced_database_tables.* +4510 resources.synced_database_tables.*.url +4511 resources.synced_database_tables.*.modified_status +4512 resources.synced_database_tables.*.lifecycle +4513 resources.synced_database_tables.*.lifecycle.prevent_destroy +4514 resources.synced_database_tables.*.lifecycle.prevent_destroy=true +4515 resources.synced_database_tables.*.lifecycle.prevent_destroy=false +4516 resources.synced_database_tables.*.lifecycle.depends_on +4517 resources.synced_database_tables.*.lifecycle.depends_on[*] +4518 resources.synced_database_tables.*.lifecycle.depends_on[*].resource +4519 resources.synced_database_tables.*.id +4520 resources.synced_database_tables.*.data_synchronization_status +4521 resources.synced_database_tables.*.data_synchronization_status.continuous_update_status +4522 resources.synced_database_tables.*.data_synchronization_status.continuous_update_status.initial_pipeline_sync_progress +4523 resources.synced_database_tables.*.data_synchronization_status.continuous_update_status.initial_pipeline_sync_progress.estimated_completion_time_seconds +4524 resources.synced_database_tables.*.data_synchronization_status.continuous_update_status.initial_pipeline_sync_progress.latest_version_currently_processing +4525 resources.synced_database_tables.*.data_synchronization_status.continuous_update_status.initial_pipeline_sync_progress.provisioning_phase +4526 resources.synced_database_tables.*.data_synchronization_status.continuous_update_status.initial_pipeline_sync_progress.sync_progress_completion +4527 resources.synced_database_tables.*.data_synchronization_status.continuous_update_status.initial_pipeline_sync_progress.synced_row_count +4528 resources.synced_database_tables.*.data_synchronization_status.continuous_update_status.initial_pipeline_sync_progress.total_row_count +4529 resources.synced_database_tables.*.data_synchronization_status.continuous_update_status.last_processed_commit_version +4530 resources.synced_database_tables.*.data_synchronization_status.continuous_update_status.timestamp +4531 resources.synced_database_tables.*.data_synchronization_status.detailed_state +4532 resources.synced_database_tables.*.data_synchronization_status.failed_status +4533 resources.synced_database_tables.*.data_synchronization_status.failed_status.last_processed_commit_version +4534 resources.synced_database_tables.*.data_synchronization_status.failed_status.timestamp +4535 resources.synced_database_tables.*.data_synchronization_status.last_sync +4536 resources.synced_database_tables.*.data_synchronization_status.last_sync.delta_table_sync_info +4537 resources.synced_database_tables.*.data_synchronization_status.last_sync.delta_table_sync_info.delta_commit_timestamp +4538 resources.synced_database_tables.*.data_synchronization_status.last_sync.delta_table_sync_info.delta_commit_version +4539 resources.synced_database_tables.*.data_synchronization_status.last_sync.sync_end_timestamp +4540 resources.synced_database_tables.*.data_synchronization_status.last_sync.sync_start_timestamp +4541 resources.synced_database_tables.*.data_synchronization_status.message +4542 resources.synced_database_tables.*.data_synchronization_status.pipeline_id +4543 resources.synced_database_tables.*.data_synchronization_status.provisioning_status +4544 resources.synced_database_tables.*.data_synchronization_status.provisioning_status.initial_pipeline_sync_progress +4545 resources.synced_database_tables.*.data_synchronization_status.provisioning_status.initial_pipeline_sync_progress.estimated_completion_time_seconds +4546 resources.synced_database_tables.*.data_synchronization_status.provisioning_status.initial_pipeline_sync_progress.latest_version_currently_processing +4547 resources.synced_database_tables.*.data_synchronization_status.provisioning_status.initial_pipeline_sync_progress.provisioning_phase +4548 resources.synced_database_tables.*.data_synchronization_status.provisioning_status.initial_pipeline_sync_progress.sync_progress_completion +4549 resources.synced_database_tables.*.data_synchronization_status.provisioning_status.initial_pipeline_sync_progress.synced_row_count +4550 resources.synced_database_tables.*.data_synchronization_status.provisioning_status.initial_pipeline_sync_progress.total_row_count +4551 resources.synced_database_tables.*.data_synchronization_status.triggered_update_status +4552 resources.synced_database_tables.*.data_synchronization_status.triggered_update_status.last_processed_commit_version +4553 resources.synced_database_tables.*.data_synchronization_status.triggered_update_status.timestamp +4554 resources.synced_database_tables.*.data_synchronization_status.triggered_update_status.triggered_update_progress +4555 resources.synced_database_tables.*.data_synchronization_status.triggered_update_status.triggered_update_progress.estimated_completion_time_seconds +4556 resources.synced_database_tables.*.data_synchronization_status.triggered_update_status.triggered_update_progress.latest_version_currently_processing +4557 resources.synced_database_tables.*.data_synchronization_status.triggered_update_status.triggered_update_progress.provisioning_phase +4558 resources.synced_database_tables.*.data_synchronization_status.triggered_update_status.triggered_update_progress.sync_progress_completion +4559 resources.synced_database_tables.*.data_synchronization_status.triggered_update_status.triggered_update_progress.synced_row_count +4560 resources.synced_database_tables.*.data_synchronization_status.triggered_update_status.triggered_update_progress.total_row_count +4561 resources.synced_database_tables.*.database_instance_name +4562 resources.synced_database_tables.*.effective_database_instance_name +4563 resources.synced_database_tables.*.effective_logical_database_name +4564 resources.synced_database_tables.*.logical_database_name +4565 resources.synced_database_tables.*.name +4566 resources.synced_database_tables.*.spec +4567 resources.synced_database_tables.*.spec.accelerated_sync +4568 resources.synced_database_tables.*.spec.accelerated_sync=true +4569 resources.synced_database_tables.*.spec.accelerated_sync=false +4570 resources.synced_database_tables.*.spec.create_database_objects_if_missing +4571 resources.synced_database_tables.*.spec.create_database_objects_if_missing=true +4572 resources.synced_database_tables.*.spec.create_database_objects_if_missing=false +4573 resources.synced_database_tables.*.spec.existing_pipeline_id +4574 resources.synced_database_tables.*.spec.new_pipeline_spec +4575 resources.synced_database_tables.*.spec.new_pipeline_spec.budget_policy_id +4576 resources.synced_database_tables.*.spec.new_pipeline_spec.storage_catalog +4577 resources.synced_database_tables.*.spec.new_pipeline_spec.storage_schema +4578 resources.synced_database_tables.*.spec.primary_key_columns +4579 resources.synced_database_tables.*.spec.primary_key_columns[*] +4580 resources.synced_database_tables.*.spec.scheduling_policy +4581 resources.synced_database_tables.*.spec.source_table_full_name +4582 resources.synced_database_tables.*.spec.timeseries_key +4583 resources.synced_database_tables.*.spec.type_overrides +4584 resources.synced_database_tables.*.spec.type_overrides[*] +4585 resources.synced_database_tables.*.spec.type_overrides[*].column_name +4586 resources.synced_database_tables.*.spec.type_overrides[*].pg_type +4587 resources.synced_database_tables.*.spec.type_overrides[*].size +4588 resources.synced_database_tables.*.unity_catalog_provisioning_state diff --git a/acceptance/bundle/bitmap/full/output/vector_search_endpoints.txt b/acceptance/bundle/bitmap/full/output/vector_search_endpoints.txt new file mode 100644 index 00000000000..2f78f75fc23 --- /dev/null +++ b/acceptance/bundle/bitmap/full/output/vector_search_endpoints.txt @@ -0,0 +1,23 @@ +4865 resources.vector_search_endpoints +4866 resources.vector_search_endpoints.* +4867 resources.vector_search_endpoints.*.url +4868 resources.vector_search_endpoints.*.modified_status +4869 resources.vector_search_endpoints.*.lifecycle +4870 resources.vector_search_endpoints.*.lifecycle.prevent_destroy +4871 resources.vector_search_endpoints.*.lifecycle.prevent_destroy=true +4872 resources.vector_search_endpoints.*.lifecycle.prevent_destroy=false +4873 resources.vector_search_endpoints.*.lifecycle.depends_on +4874 resources.vector_search_endpoints.*.lifecycle.depends_on[*] +4875 resources.vector_search_endpoints.*.lifecycle.depends_on[*].resource +4876 resources.vector_search_endpoints.*.id +4877 resources.vector_search_endpoints.*.budget_policy_id +4878 resources.vector_search_endpoints.*.endpoint_type +4879 resources.vector_search_endpoints.*.name +4880 resources.vector_search_endpoints.*.target_qps +4881 resources.vector_search_endpoints.*.usage_policy_id +4882 resources.vector_search_endpoints.*.permissions +4883 resources.vector_search_endpoints.*.permissions[*] +4884 resources.vector_search_endpoints.*.permissions[*].level +4885 resources.vector_search_endpoints.*.permissions[*].user_name +4886 resources.vector_search_endpoints.*.permissions[*].service_principal_name +4887 resources.vector_search_endpoints.*.permissions[*].group_name diff --git a/acceptance/bundle/bitmap/full/output/vector_search_indexes.txt b/acceptance/bundle/bitmap/full/output/vector_search_indexes.txt new file mode 100644 index 00000000000..be69f50a4bd --- /dev/null +++ b/acceptance/bundle/bitmap/full/output/vector_search_indexes.txt @@ -0,0 +1,50 @@ +4888 resources.vector_search_indexes +4889 resources.vector_search_indexes.* +4890 resources.vector_search_indexes.*.url +4891 resources.vector_search_indexes.*.modified_status +4892 resources.vector_search_indexes.*.lifecycle +4893 resources.vector_search_indexes.*.lifecycle.prevent_destroy +4894 resources.vector_search_indexes.*.lifecycle.prevent_destroy=true +4895 resources.vector_search_indexes.*.lifecycle.prevent_destroy=false +4896 resources.vector_search_indexes.*.lifecycle.depends_on +4897 resources.vector_search_indexes.*.lifecycle.depends_on[*] +4898 resources.vector_search_indexes.*.lifecycle.depends_on[*].resource +4899 resources.vector_search_indexes.*.id +4900 resources.vector_search_indexes.*.delta_sync_index_spec +4901 resources.vector_search_indexes.*.delta_sync_index_spec.columns_to_index +4902 resources.vector_search_indexes.*.delta_sync_index_spec.columns_to_index[*] +4903 resources.vector_search_indexes.*.delta_sync_index_spec.columns_to_sync +4904 resources.vector_search_indexes.*.delta_sync_index_spec.columns_to_sync[*] +4905 resources.vector_search_indexes.*.delta_sync_index_spec.embedding_source_columns +4906 resources.vector_search_indexes.*.delta_sync_index_spec.embedding_source_columns[*] +4907 resources.vector_search_indexes.*.delta_sync_index_spec.embedding_source_columns[*].embedding_model_endpoint_name +4908 resources.vector_search_indexes.*.delta_sync_index_spec.embedding_source_columns[*].model_endpoint_name_for_query +4909 resources.vector_search_indexes.*.delta_sync_index_spec.embedding_source_columns[*].name +4910 resources.vector_search_indexes.*.delta_sync_index_spec.embedding_vector_columns +4911 resources.vector_search_indexes.*.delta_sync_index_spec.embedding_vector_columns[*] +4912 resources.vector_search_indexes.*.delta_sync_index_spec.embedding_vector_columns[*].embedding_dimension +4913 resources.vector_search_indexes.*.delta_sync_index_spec.embedding_vector_columns[*].name +4914 resources.vector_search_indexes.*.delta_sync_index_spec.embedding_writeback_table +4915 resources.vector_search_indexes.*.delta_sync_index_spec.pipeline_type +4916 resources.vector_search_indexes.*.delta_sync_index_spec.source_table +4917 resources.vector_search_indexes.*.direct_access_index_spec +4918 resources.vector_search_indexes.*.direct_access_index_spec.embedding_source_columns +4919 resources.vector_search_indexes.*.direct_access_index_spec.embedding_source_columns[*] +4920 resources.vector_search_indexes.*.direct_access_index_spec.embedding_source_columns[*].embedding_model_endpoint_name +4921 resources.vector_search_indexes.*.direct_access_index_spec.embedding_source_columns[*].model_endpoint_name_for_query +4922 resources.vector_search_indexes.*.direct_access_index_spec.embedding_source_columns[*].name +4923 resources.vector_search_indexes.*.direct_access_index_spec.embedding_vector_columns +4924 resources.vector_search_indexes.*.direct_access_index_spec.embedding_vector_columns[*] +4925 resources.vector_search_indexes.*.direct_access_index_spec.embedding_vector_columns[*].embedding_dimension +4926 resources.vector_search_indexes.*.direct_access_index_spec.embedding_vector_columns[*].name +4927 resources.vector_search_indexes.*.direct_access_index_spec.schema_json +4928 resources.vector_search_indexes.*.endpoint_name +4929 resources.vector_search_indexes.*.index_subtype +4930 resources.vector_search_indexes.*.index_type +4931 resources.vector_search_indexes.*.name +4932 resources.vector_search_indexes.*.primary_key +4933 resources.vector_search_indexes.*.grants +4934 resources.vector_search_indexes.*.grants[*] +4935 resources.vector_search_indexes.*.grants[*].principal +4936 resources.vector_search_indexes.*.grants[*].privileges +4937 resources.vector_search_indexes.*.grants[*].privileges[*] diff --git a/acceptance/bundle/bitmap/full/output/volumes.txt b/acceptance/bundle/bitmap/full/output/volumes.txt new file mode 100644 index 00000000000..09b4fabc965 --- /dev/null +++ b/acceptance/bundle/bitmap/full/output/volumes.txt @@ -0,0 +1,24 @@ +3667 resources.volumes +3668 resources.volumes.* +3669 resources.volumes.*.url +3670 resources.volumes.*.modified_status +3671 resources.volumes.*.lifecycle +3672 resources.volumes.*.lifecycle.prevent_destroy +3673 resources.volumes.*.lifecycle.prevent_destroy=true +3674 resources.volumes.*.lifecycle.prevent_destroy=false +3675 resources.volumes.*.lifecycle.depends_on +3676 resources.volumes.*.lifecycle.depends_on[*] +3677 resources.volumes.*.lifecycle.depends_on[*].resource +3678 resources.volumes.*.id +3679 resources.volumes.*.catalog_name +3680 resources.volumes.*.comment +3681 resources.volumes.*.name +3682 resources.volumes.*.schema_name +3683 resources.volumes.*.storage_location +3684 resources.volumes.*.volume_type +3685 resources.volumes.*.volume_path +3686 resources.volumes.*.grants +3687 resources.volumes.*.grants[*] +3688 resources.volumes.*.grants[*].principal +3689 resources.volumes.*.grants[*].privileges +3690 resources.volumes.*.grants[*].privileges[*] diff --git a/acceptance/bundle/bitmap/full/script b/acceptance/bundle/bitmap/full/script new file mode 100644 index 00000000000..54553baef6f --- /dev/null +++ b/acceptance/bundle/bitmap/full/script @@ -0,0 +1,4 @@ +# Records every schema line with its bit, split per resource type, so a PR that adds a config +# field shows where it landed in the append-only schema. Refreshed by 'task generate-bitmap-schema'; +# ../presence pins the semantics of the bits. +$CLI bundle bitmap schema | python3 split.py diff --git a/acceptance/bundle/bitmap/full/split.py b/acceptance/bundle/bitmap/full/split.py new file mode 100644 index 00000000000..e2d0ed2f0b3 --- /dev/null +++ b/acceptance/bundle/bitmap/full/split.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python3 +"""Split the bitmap schema from stdin: resources..* goes to output/.txt, the rest to output/root.txt. + +Each line is " ", so a file still shows where each of its fields sits in the schema. Bits +are right-aligned to the width of the largest one, the same in every file.""" + +import os +import sys + +paths = sys.stdin.read().splitlines() +width = len(str(len(paths) - 1)) +files = {} +os.makedirs("output", exist_ok=True) +for bit, path in enumerate(paths): + parts = path.split(".") + name = parts[1] if parts[0] == "resources" and len(parts) > 1 else "root" + if name not in files: + files[name] = open(f"output/{name}.txt", "w", newline="\n") + files[name].write(f"{bit:>{width}} {path}\n") +for f in files.values(): + f.close() diff --git a/acceptance/bundle/bitmap/full/test.toml b/acceptance/bundle/bitmap/full/test.toml new file mode 100644 index 00000000000..d3a556b4329 --- /dev/null +++ b/acceptance/bundle/bitmap/full/test.toml @@ -0,0 +1,2 @@ +# The schema does not depend on the bundle or the DMS variant. +EnvMatrix.DMS = [] diff --git a/acceptance/bundle/bitmap/presence/databricks.yml b/acceptance/bundle/bitmap/presence/databricks.yml new file mode 100644 index 00000000000..3b47e4a45c2 --- /dev/null +++ b/acceptance/bundle/bitmap/presence/databricks.yml @@ -0,0 +1,102 @@ +bundle: + name: test-bundle + +variables: + scalar_var: + description: a variable + default: abc + complex_var: + type: complex + default: + key: value + flag: + default: true + # Set through BUNDLE_VAR_off_flag in test.toml, so its value arrives as a string. + off_flag: + description: no default + clusters: + type: complex + default: + - job_cluster_key: shared + new_cluster: + spark_version: 15.4.x-scala2.12 + num_workers: 1 + +# An explicit false on an optional bool of a CLI-defined struct (*bool) is recorded, +# as it is for SDK bools (see disable_auto_optimization below). +experimental: + use_legacy_run_as: false + +presets: + name_prefix: dev_ + pipelines_development: false + +run_as: + service_principal_name: sp-1234 + +artifacts: + # One artifact with a build command and one with files only, so each + # per-artifact flag records both outcomes. + with_build: + type: whl + build: "true" + with_files: + type: whl + files: + - source: wheels/*.whl + +resources: + jobs: + my_job: + name: My Job + description: "escaped $${var.scalar_var}" + timeout_seconds: 0 + email_notifications: {} + parameters: [] + tags: + team: data + "a.b": c + # Set through a variable: the variable's value is recorded. + notification_settings: + no_alert_for_skipped_runs: ${var.flag} + # true here and false in other_job: both value bits are set. + queue: + enabled: true + tasks: + - task_key: a + # SDK bool with omitempty: an explicit false is kept via ForceSendFields. + disable_auto_optimization: false + notebook_task: + notebook_path: ./notebook.py + - task_key: b + depends_on: + - task_key: a + notebook_task: + notebook_path: ./notebook.py + other_job: + name: Other Job + queue: + enabled: false + email_notifications: + no_alert_for_skipped_runs: ${var.off_flag} + # A complex variable: the fields inside its value are recorded. + job_clusters: ${var.clusters} + clusters: + # lifecycle.started is a *bool, so the explicit false is kept: true on one cluster + # and false on the other sets both value bits. + started_cluster: + cluster_name: started + lifecycle: + started: true + stopped_cluster: + cluster_name: stopped + lifecycle: + started: false + +targets: + dev: + default: true + resources: + jobs: + my_job: + max_concurrent_runs: 2 diff --git a/acceptance/bundle/bitmap/presence/future.py b/acceptance/bundle/bitmap/presence/future.py new file mode 100755 index 00000000000..dce0440a869 --- /dev/null +++ b/acceptance/bundle/bitmap/presence/future.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python3 +"""Read an encoded bundle bitmap from stdin and print it as a newer CLI, whose schema has two more +fields, would encode it with the last of those set. + +Wire format: base64 of raw DEFLATE of "DBTB" | size (uint32, big-endian) | context (uint16, +big-endian) | the bits, MSB first.""" + +import base64 +import struct +import sys +import zlib + +raw = zlib.decompress(base64.b64decode(sys.stdin.read().strip()), -15) +size, context = struct.unpack(">IH", raw[4:10]) +bits = [(raw[10 + i // 8] >> (7 - i % 8)) & 1 for i in range(size)] + [0, 1] +bitmap = bytearray((len(bits) + 7) // 8) +for i, bit in enumerate(bits): + bitmap[i // 8] |= bit << (7 - i % 8) +payload = raw[:4] + struct.pack(">IH", len(bits), context) + bytes(bitmap) +compressor = zlib.compressobj(wbits=-15) +print(base64.b64encode(compressor.compress(payload) + compressor.flush()).decode()) diff --git a/acceptance/bundle/bitmap/presence/invalid/databricks.yml b/acceptance/bundle/bitmap/presence/invalid/databricks.yml new file mode 100644 index 00000000000..cb7920d4386 --- /dev/null +++ b/acceptance/bundle/bitmap/presence/invalid/databricks.yml @@ -0,0 +1,5 @@ +bundle: + name: test-bundle + +artifacts: + broken: diff --git a/acceptance/bundle/bitmap/presence/notebook.py b/acceptance/bundle/bitmap/presence/notebook.py new file mode 100644 index 00000000000..1645e04b1de --- /dev/null +++ b/acceptance/bundle/bitmap/presence/notebook.py @@ -0,0 +1 @@ +# Databricks notebook source diff --git a/acceptance/bundle/bitmap/presence/out.bitmap.b64 b/acceptance/bundle/bitmap/presence/out.bitmap.b64 new file mode 100644 index 00000000000..fa022d6ac16 --- /dev/null +++ b/acceptance/bundle/bitmap/presence/out.bitmap.b64 @@ -0,0 +1 @@ +cnEKcWJgENnFwPCDgZkBGbAXMTxgYFBrYGBgYGBgsEGWYmJkYGCX4mfgZ0ADVxloAzgxRBTAaBQQBicYGBgYGFgYEODB4HNkuAMHKwMDAwODPLLjEhh2MDAABgA= diff --git a/acceptance/bundle/bitmap/presence/out.bitmap.json b/acceptance/bundle/bitmap/presence/out.bitmap.json new file mode 100644 index 00000000000..86b3c60b48d --- /dev/null +++ b/acceptance/bundle/bitmap/presence/out.bitmap.json @@ -0,0 +1,154 @@ +{ + "size": [SCHEMA_SIZE], + "context": 0, + "presence": { + "artifacts": { + "*": { + "build": true, + "files[*]": { + "source": true + }, + "type": true + } + }, + "bundle": { + "name": true + }, + "experimental": { + "use_legacy_run_as": true + }, + "presets": { + "name_prefix": true, + "pipelines_development": true + }, + "resources": { + "clusters": { + "*": { + "cluster_name": true, + "lifecycle": { + "started": true + } + } + }, + "jobs": { + "*": { + "description": true, + "email_notifications": { + "no_alert_for_skipped_runs": true + }, + "job_clusters[*]": { + "job_cluster_key": true, + "new_cluster": { + "num_workers": true, + "spark_version": true + } + }, + "max_concurrent_runs": true, + "name": true, + "notification_settings": { + "no_alert_for_skipped_runs": true + }, + "parameters": true, + "queue": { + "enabled": true + }, + "tags": { + "*": true + }, + "tasks[*]": { + "depends_on[*]": { + "task_key": true + }, + "disable_auto_optimization": true, + "notebook_task": { + "notebook_path": true + }, + "task_key": true + }, + "timeout_seconds": true + } + } + }, + "run_as": { + "service_principal_name": true + }, + "telemetry": { + "ArtifactBuildCommandIsSet": true, + "ArtifactFilesIsSet": true, + "ConfigHasDoubleDollarBrace": true, + "PresetsNamePrefixIsSet": true, + "RunAsSet": true, + "UseLegacyRunAs": true + }, + "variables": { + "*": { + "default": true, + "description": true, + "type": true + } + } + }, + "true": { + "resources": { + "clusters": { + "*": { + "lifecycle": { + "started": true + } + } + }, + "jobs": { + "*": { + "notification_settings": { + "no_alert_for_skipped_runs": true + }, + "queue": { + "enabled": true + } + } + } + }, + "telemetry": { + "ArtifactBuildCommandIsSet": true, + "ArtifactFilesIsSet": true, + "ConfigHasDoubleDollarBrace": true, + "PresetsNamePrefixIsSet": true, + "RunAsSet": true + } + }, + "false": { + "experimental": { + "use_legacy_run_as": true + }, + "presets": { + "pipelines_development": true + }, + "resources": { + "clusters": { + "*": { + "lifecycle": { + "started": true + } + } + }, + "jobs": { + "*": { + "email_notifications": { + "no_alert_for_skipped_runs": true + }, + "queue": { + "enabled": true + }, + "tasks[*]": { + "disable_auto_optimization": true + } + } + } + }, + "telemetry": { + "ArtifactBuildCommandIsSet": true, + "ArtifactFilesIsSet": true, + "UseLegacyRunAs": true + } + } +} diff --git a/acceptance/bundle/bitmap/presence/out.bitmap.txt b/acceptance/bundle/bitmap/presence/out.bitmap.txt new file mode 100644 index 00000000000..08a03ad4b49 --- /dev/null +++ b/acceptance/bundle/bitmap/presence/out.bitmap.txt @@ -0,0 +1,64 @@ +size=[SCHEMA_SIZE] context=0 +P variables +P variables.* +P variables.*.type +P variables.*.default +P variables.*.description +P bundle +P bundle.name +P artifacts +P artifacts.* +P artifacts.*.type +P artifacts.*.files +P artifacts.*.files[*] +P artifacts.*.files[*].source +P artifacts.*.build +P resources +P resources.jobs +P resources.jobs.* +P resources.jobs.*.description +P resources.jobs.*.email_notifications +P F resources.jobs.*.email_notifications.no_alert_for_skipped_runs +P resources.jobs.*.job_clusters +P resources.jobs.*.job_clusters[*] +P resources.jobs.*.job_clusters[*].job_cluster_key +P resources.jobs.*.job_clusters[*].new_cluster +P resources.jobs.*.job_clusters[*].new_cluster.num_workers +P resources.jobs.*.job_clusters[*].new_cluster.spark_version +P resources.jobs.*.max_concurrent_runs +P resources.jobs.*.name +P resources.jobs.*.notification_settings +PT resources.jobs.*.notification_settings.no_alert_for_skipped_runs +P resources.jobs.*.parameters +P resources.jobs.*.queue +PTF resources.jobs.*.queue.enabled +P resources.jobs.*.tags +P resources.jobs.*.tags.* +P resources.jobs.*.tasks +P resources.jobs.*.tasks[*] +P resources.jobs.*.tasks[*].depends_on +P resources.jobs.*.tasks[*].depends_on[*] +P resources.jobs.*.tasks[*].depends_on[*].task_key +P F resources.jobs.*.tasks[*].disable_auto_optimization +P resources.jobs.*.tasks[*].notebook_task +P resources.jobs.*.tasks[*].notebook_task.notebook_path +P resources.jobs.*.tasks[*].task_key +P resources.jobs.*.timeout_seconds +P resources.clusters +P resources.clusters.* +P resources.clusters.*.lifecycle +P resources.clusters.*.cluster_name +PTF resources.clusters.*.lifecycle.started +P run_as +P run_as.service_principal_name +P presets +P presets.name_prefix +P F presets.pipelines_development +P experimental +P F experimental.use_legacy_run_as +PTF telemetry.ArtifactBuildCommandIsSet +PTF telemetry.ArtifactFilesIsSet +PT telemetry.PresetsNamePrefixIsSet +PT telemetry.ConfigHasDoubleDollarBrace +P F telemetry.UseLegacyRunAs +PT telemetry.RunAsSet diff --git a/acceptance/bundle/bitmap/presence/out.test.toml b/acceptance/bundle/bitmap/presence/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/bitmap/presence/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/bitmap/presence/output.txt b/acceptance/bundle/bitmap/presence/output.txt new file mode 100644 index 00000000000..b21bd0e539b --- /dev/null +++ b/acceptance/bundle/bitmap/presence/output.txt @@ -0,0 +1,94 @@ + +>>> [CLI] bundle bitmap bitmap -o text + +>>> [CLI] bundle bitmap bitmap -o json + +>>> [CLI] bundle bitmap bitmap + +>>> [CLI] bundle bitmap decode + +>>> diff.py out.bitmap.txt decoded.txt + +>>> [CLI] bundle bitmap decode -o json + +>>> diff.py out.bitmap.json decoded.json + +>>> [CLI] bundle bitmap decode -o compressed + +>>> diff.py out.bitmap.b64 recoded.b64 + +>>> uv run --isolated --no-project --reinstall-package databricks-bundles --with [DATABRICKS_BUNDLES_WHEEL] -q python -m databricks.bundles._bitmap + +>>> diff.py out.bitmap.txt decoded-python.txt + +>>> uv run --isolated --no-project --reinstall-package databricks-bundles --with [DATABRICKS_BUNDLES_WHEEL] -q python -m databricks.bundles._bitmap -o json + +>>> diff.py out.bitmap.json decoded-python.json + +>>> [CLI] bundle bitmap bitmap -o text --var off_flag=false + +>>> diff.py out.bitmap.txt var-flag.txt + +=== A bitmap from a newer CLI decodes, with a placeholder for the field this schema lacks + + +>>> [CLI] bundle bitmap decode + +>>> [CLI] bundle bitmap decode -o json + +>>> uv run --isolated --no-project --reinstall-package databricks-bundles --with [DATABRICKS_BUNDLES_WHEEL] -q python -m databricks.bundles._bitmap + +>>> uv run --isolated --no-project --reinstall-package databricks-bundles --with [DATABRICKS_BUNDLES_WHEEL] -q python -m databricks.bundles._bitmap -o json + +>>> diff.py LOG.future.txt LOG.future-python.txt + +>>> diff.py LOG.future.json LOG.future-python.json +size=[FUTURE_SIZE] context=0 +P future #[FUTURE_BIT] +{ + "future #[FUTURE_BIT]": true +} + +>>> [CLI] bundle bitmap update-schema --validate + +=== decode rejects input that is not an encoded bitmap + + +>>> musterr [CLI] bundle bitmap decode +Error: base64: illegal base64 data at input byte 3 + +=== -o accepts compressed, text and json + + +>>> musterr [CLI] bundle bitmap bitmap -o yaml +Error: invalid argument "yaml" for "-o, --output" flag: accepted values are compressed, text and json + +Usage: + databricks bundle bitmap bitmap [flags] + +Flags: + -h, --help[=true|false] help for bitmap + -o, --output format output format: + compressed: base64 of the DEFLATE-compressed wire format + text: a "size=N context=C" header, then one "PTF " line per set + field: "P" if present, "T" and "F" if a boolean field held true + and false, a space for each that is not; unset fields are omitted + json: size, context, and trees of the fields present ("presence") and of + the boolean fields seen holding "true" and "false" (default compressed) + +Global Flags: + --debug[=true|false] enable debug logging + -p, --profile string ~/.databrickscfg profile + -t, --target string bundle target to use (if applicable) + --var strings set values for variables defined in bundle config. Example: --var="foo=bar" + + +=== An invalid bundle fails instead of printing a bitmap + + +>>> musterr [CLI] bundle bitmap bitmap +Error: Artifact not properly configured + in databricks.yml:5:10 + +please specify artifact properties + diff --git a/acceptance/bundle/bitmap/presence/script b/acceptance/bundle/bitmap/presence/script new file mode 100644 index 00000000000..7cc62a4815e --- /dev/null +++ b/acceptance/bundle/bitmap/presence/script @@ -0,0 +1,58 @@ +# The size grows with every field appended to the schema; replacing it keeps these goldens +# stable while still asserting that the bitmap covers the whole schema (../full records it). +schema_size=$($CLI bundle bitmap schema | wc -l | tr -d ' ') +add_repl "$schema_size" SCHEMA_SIZE + +# One "PTF " line per set field (P present, T/F the values a bool held) after a +# size/context header. +trace $CLI bundle bitmap bitmap -o text > out.bitmap.txt +trace $CLI bundle bitmap bitmap -o json > out.bitmap.json + +# The encoded bitmap decodes back to the same fields, with the CLI and with the +# self-contained decoder in databricks-bundles. --reinstall-package: the wheel's +# version does not change with the code, and uv would reuse a cached install. +trace $CLI bundle bitmap bitmap > out.bitmap.b64 +trace $CLI bundle bitmap decode < out.bitmap.b64 > decoded.txt +trace diff.py out.bitmap.txt decoded.txt +trace $CLI bundle bitmap decode -o json < out.bitmap.b64 > decoded.json +trace diff.py out.bitmap.json decoded.json +# Encoding is deterministic, so re-encoding a decoded bitmap gives the same bytes. +trace $CLI bundle bitmap decode -o compressed < out.bitmap.b64 > recoded.b64 +trace diff.py out.bitmap.b64 recoded.b64 +trace uv run --isolated --no-project --reinstall-package databricks-bundles --with "$DATABRICKS_BUNDLES_WHEEL" -q python -m databricks.bundles._bitmap < out.bitmap.b64 > decoded-python.txt +trace diff.py out.bitmap.txt decoded-python.txt +trace uv run --isolated --no-project --reinstall-package databricks-bundles --with "$DATABRICKS_BUNDLES_WHEEL" -q python -m databricks.bundles._bitmap -o json < out.bitmap.b64 > decoded-python.json +trace diff.py out.bitmap.json decoded-python.json + +# A value passed with --var gives the same bits as BUNDLE_VAR_off_flag (test.toml): the bits +# record what the config sets, not how a variable got its value. +trace $CLI bundle bitmap bitmap -o text --var off_flag=false > var-flag.txt +trace diff.py out.bitmap.txt var-flag.txt + +title "A bitmap from a newer CLI decodes, with a placeholder for the field this schema lacks\n" +# future.py re-encodes the bitmap with two more fields, the last one set. +add_repl "$((schema_size + 2))" FUTURE_SIZE +add_repl "$((schema_size + 1))" FUTURE_BIT +python3 future.py < out.bitmap.b64 > LOG.future.b64 +trace $CLI bundle bitmap decode < LOG.future.b64 > LOG.future.txt +trace $CLI bundle bitmap decode -o json < LOG.future.b64 > LOG.future.json +trace uv run --isolated --no-project --reinstall-package databricks-bundles --with "$DATABRICKS_BUNDLES_WHEEL" -q python -m databricks.bundles._bitmap < LOG.future.b64 > LOG.future-python.txt +trace uv run --isolated --no-project --reinstall-package databricks-bundles --with "$DATABRICKS_BUNDLES_WHEEL" -q python -m databricks.bundles._bitmap -o json < LOG.future.b64 > LOG.future-python.json +trace diff.py LOG.future.txt LOG.future-python.txt +trace diff.py LOG.future.json LOG.future-python.json +grep -E '^size=|future' LOG.future.txt +jq '.presence | with_entries(select(.key | startswith("future")))' LOG.future.json + +trace $CLI bundle bitmap update-schema --validate + +title "decode rejects input that is not an encoded bitmap\n" +echo not-a-bitmap | trace musterr $CLI bundle bitmap decode + +title "-o accepts compressed, text and json\n" +trace musterr $CLI bundle bitmap bitmap -o yaml +# The message comes from Python's base64 module and varies by version, so it is logged, not diffed. +echo not-a-bitmap | trace musterr uv run --isolated --no-project --reinstall-package databricks-bundles --with "$DATABRICKS_BUNDLES_WHEEL" -q python -m databricks.bundles._bitmap 2> LOG.decode_python + +title "An invalid bundle fails instead of printing a bitmap\n" +cd invalid +trace musterr $CLI bundle bitmap bitmap diff --git a/acceptance/bundle/bitmap/presence/test.toml b/acceptance/bundle/bitmap/presence/test.toml new file mode 100644 index 00000000000..9f1c6d5c205 --- /dev/null +++ b/acceptance/bundle/bitmap/presence/test.toml @@ -0,0 +1,4 @@ +Env.BUNDLE_VAR_off_flag = "false" + +# Scratch files of the round-trip check; out.bitmap.txt and out.bitmap.json are the goldens. +Ignore = [".databricks", "recoded.b64", "decoded.txt", "decoded.json", "decoded-python.txt", "decoded-python.json", "var-flag.txt"] diff --git a/acceptance/bundle/bitmap/presence/wheels/test_bundle-0.1-py3-none-any.whl b/acceptance/bundle/bitmap/presence/wheels/test_bundle-0.1-py3-none-any.whl new file mode 100644 index 00000000000..e69de29bb2d diff --git a/acceptance/bundle/invariant/validate_bitmap/out.test.toml b/acceptance/bundle/invariant/validate_bitmap/out.test.toml new file mode 100644 index 00000000000..cf622d40c4d --- /dev/null +++ b/acceptance/bundle/invariant/validate_bitmap/out.test.toml @@ -0,0 +1,71 @@ +Cloud = false +EnvMatrix.DMS = [] +EnvMatrix.INPUT_CONFIG = [ + "alert.yml.tmpl", + "app.yml.tmpl", + "catalog.yml.tmpl", + "catalog_optional_fields.yml.tmpl", + "cluster.yml.tmpl", + "cluster_apply_policy_default_values.yml.tmpl", + "cluster_libraries.yml.tmpl", + "cluster_policy.yml.tmpl", + "dashboard.yml.tmpl", + "job_apply_policy_default_values_job_cluster.yml.tmpl", + "job_apply_policy_default_values_task_cluster.yml.tmpl", + "job_apply_policy_default_values_for_each_task.yml.tmpl", + "data_quality_monitor.yml.tmpl", + "database_catalog.yml.tmpl", + "database_instance.yml.tmpl", + "experiment.yml.tmpl", + "external_location.yml.tmpl", + "feature.yml.tmpl", + "feature_spec.yml.tmpl", + "genie_space.yml.tmpl", + "instance_pool.yml.tmpl", + "job.yml.tmpl", + "job_pydabs_10_tasks.yml.tmpl", + "job_pydabs_1000_tasks.yml.tmpl", + "job_cross_resource_ref.yml.tmpl", + "job_escaped_refs.yml.tmpl", + "job_permission_ref.yml.tmpl", + "job_run.yml.tmpl", + "job_run_job_ref.yml.tmpl", + "job_table_update_trigger.yml.tmpl", + "job_with_depends_on.yml.tmpl", + "job_with_task.yml.tmpl", + "model.yml.tmpl", + "model_with_permissions.yml.tmpl", + "model_serving_endpoint.yml.tmpl", + "model_serving_endpoint_telemetry.yml.tmpl", + "pipeline.yml.tmpl", + "pipeline_allow_duplicate_names.yml.tmpl", + "pipeline_apply_policy_default_values.yml.tmpl", + "pipeline_config_dots.yml.tmpl", + "postgres_branch.yml.tmpl", + "postgres_catalog.yml.tmpl", + "postgres_data_api.yml.tmpl", + "postgres_database.yml.tmpl", + "postgres_endpoint.yml.tmpl", + "postgres_project.yml.tmpl", + "postgres_role.yml.tmpl", + "postgres_synced_table.yml.tmpl", + "registered_model.yml.tmpl", + "schema.yml.tmpl", + "schema_tf_compat.yml.tmpl", + "schema_empty_grants.yml.tmpl", + "schema_grant_ref.yml.tmpl", + "schema_uppercase_name.yml.tmpl", + "secret.yml.tmpl", + "secret_scope.yml.tmpl", + "secret_scope_default_backend_type.yml.tmpl", + "sql_warehouse.yml.tmpl", + "synced_database_table.yml.tmpl", + "uc_trailing_slash.yml.tmpl", + "vector_search_endpoint.yml.tmpl", + "vector_search_index.yml.tmpl", + "volume.yml.tmpl", + "volume_external.yml.tmpl", + "volume_path_job_ref.yml.tmpl", + "volume_uppercase_name.yml.tmpl", + "model_service.yml.tmpl" +] diff --git a/acceptance/bundle/invariant/validate_bitmap/output.txt b/acceptance/bundle/invariant/validate_bitmap/output.txt new file mode 100644 index 00000000000..7a28cb73a58 --- /dev/null +++ b/acceptance/bundle/invariant/validate_bitmap/output.txt @@ -0,0 +1 @@ +INPUT_CONFIG_OK diff --git a/acceptance/bundle/invariant/validate_bitmap/script b/acceptance/bundle/invariant/validate_bitmap/script new file mode 100644 index 00000000000..8f84e596a0c --- /dev/null +++ b/acceptance/bundle/invariant/validate_bitmap/script @@ -0,0 +1,38 @@ +# Invariant to test: every config validates, and its bitmap survives encoding: the CLI decoder +# and the databricks-bundles Python decoder (text and JSON each) reproduce what the bitmap +# command prints before encoding. +# Nothing is deployed, so there is nothing to clean up. + +invariant_render + +# Validation output differs per config: log it, check for panics. +$CLI bundle validate &> LOG.validate +cat LOG.validate | contains.py '!panic:' '!internal error' > /dev/null +$CLI bundle validate -o json > LOG.validate_json 2> LOG.validate_json_err +cat LOG.validate_json_err | contains.py '!panic:' '!internal error' > /dev/null + +# Tells the fuzzer the generated config was valid; failures after this count as bugs. +echo INPUT_CONFIG_OK + +$CLI bundle bitmap bitmap -o text > LOG.bitmap_text 2> LOG.bitmap_text_err +cat LOG.bitmap_text_err | contains.py '!panic:' '!internal error' > /dev/null +# Every config names its bundle; guards against comparing two empty decodes. +cat LOG.bitmap_text | contains.py 'P bundle.name' > /dev/null + +$CLI bundle bitmap bitmap -o json > LOG.bitmap_json 2> LOG.bitmap_json_err +cat LOG.bitmap_json_err | contains.py '!panic:' '!internal error' > /dev/null + +$CLI bundle bitmap bitmap > LOG.bitmap 2> LOG.bitmap_err +cat LOG.bitmap_err | contains.py '!panic:' '!internal error' > /dev/null + +$CLI bundle bitmap decode < LOG.bitmap > LOG.bitmap_decoded +$CLI bundle bitmap decode -o json < LOG.bitmap > LOG.bitmap_decoded_json +# --reinstall-package: the wheel's version does not change with the code, and uv would reuse a +# cached install. +uv run --isolated --no-project --reinstall-package databricks-bundles --with "$DATABRICKS_BUNDLES_WHEEL" -q python -m databricks.bundles._bitmap < LOG.bitmap > LOG.bitmap_decoded_python +uv run --isolated --no-project --reinstall-package databricks-bundles --with "$DATABRICKS_BUNDLES_WHEEL" -q python -m databricks.bundles._bitmap -o json < LOG.bitmap > LOG.bitmap_decoded_python_json + +diff.py LOG.bitmap_text LOG.bitmap_decoded +diff.py LOG.bitmap_json LOG.bitmap_decoded_json +diff.py LOG.bitmap_text LOG.bitmap_decoded_python +diff.py LOG.bitmap_json LOG.bitmap_decoded_python_json diff --git a/acceptance/bundle/invariant/validate_bitmap/test.toml b/acceptance/bundle/invariant/validate_bitmap/test.toml new file mode 100644 index 00000000000..6ccc31b67aa --- /dev/null +++ b/acceptance/bundle/invariant/validate_bitmap/test.toml @@ -0,0 +1,5 @@ +# Nothing is deployed, so the local test server is enough. +Cloud = false + +# Neither validate nor the bitmap uses the deployment metadata service. +EnvMatrix.DMS = [] diff --git a/acceptance/bundle/python/telemetry/mutators.py b/acceptance/bundle/python/telemetry/mutators.py index 9335dd63f19..7f5392ab865 100644 --- a/acceptance/bundle/python/telemetry/mutators.py +++ b/acceptance/bundle/python/telemetry/mutators.py @@ -8,4 +8,4 @@ def update_job(job: Job) -> Job: assert isinstance(job.name, str) - return replace(job, name=f"{job.name} (updated)") + return replace(job, name=f"{job.name} (updated)", description="set by a mutator") diff --git a/acceptance/bundle/python/telemetry/output.txt b/acceptance/bundle/python/telemetry/output.txt index 46dfd6211a9..d1915fc7472 100644 --- a/acceptance/bundle/python/telemetry/output.txt +++ b/acceptance/bundle/python/telemetry/output.txt @@ -70,3 +70,14 @@ Resources: 0 created, 0 changed, 0 deleted, 4 unchanged "schemas" ] } + +=== Bitmap includes what Python wrote + +>>> uv run [UV_ARGS] -q [CLI] bundle bitmap bitmap -o text +P resources.jobs +P resources.jobs.* +P resources.jobs.*.description +P resources.jobs.*.name +P resources.pipelines +P resources.pipelines.* +P resources.pipelines.*.name diff --git a/acceptance/bundle/python/telemetry/script b/acceptance/bundle/python/telemetry/script index 311998c3e9c..c9228e1b261 100644 --- a/acceptance/bundle/python/telemetry/script +++ b/acceptance/bundle/python/telemetry/script @@ -17,4 +17,9 @@ trace update_file.py databricks.yml 'python: {resources: ["resources:load_resour trace uv run $UV_ARGS -q $CLI bundle deploy print_pydabs_telemetry +# The bitmap records what Python wrote: the pipeline it defines, and the description the mutator +# sets on the YAML job. The defaults the CLI fills in are left out. +title "Bitmap includes what Python wrote" +trace uv run $UV_ARGS -q $CLI bundle bitmap bitmap -o text | grep -E ' resources\.(jobs|pipelines)' + rm -fr .databricks __pycache__ diff --git a/acceptance/bundle/telemetry/deploy-app-lifecycle-started/databricks.yml b/acceptance/bundle/telemetry/deploy-app-lifecycle-started/databricks.yml index 73279b89f9b..cd237f9e6e5 100644 --- a/acceptance/bundle/telemetry/deploy-app-lifecycle-started/databricks.yml +++ b/acceptance/bundle/telemetry/deploy-app-lifecycle-started/databricks.yml @@ -3,8 +3,14 @@ bundle: resources: apps: + # One app started and one stopped: app_lifecycle_started reports each value once. myapp: name: my-app source_code_path: . lifecycle: started: true + stopped_app: + name: stopped-app + source_code_path: ./stopped + lifecycle: + started: false diff --git a/acceptance/bundle/telemetry/deploy-app-lifecycle-started/output.txt b/acceptance/bundle/telemetry/deploy-app-lifecycle-started/output.txt index 4362ebba960..6b828f7b641 100644 --- a/acceptance/bundle/telemetry/deploy-app-lifecycle-started/output.txt +++ b/acceptance/bundle/telemetry/deploy-app-lifecycle-started/output.txt @@ -1,46 +1,46 @@ >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -✓ Deployment succeeded Created apps.myapp -Files: 1 uploaded, 0 deleted -Resources: 1 created, 0 changed, 0 deleted, 0 unchanged +Created apps.stopped_app +Files: 2 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> cat [OUTPUT_DIR]/out.requests.txt { "bool_values": [ { - "key": "local.cache.attempt", - "value": true - }, - { - "key": "local.cache.miss", - "value": true + "key": "python_wheel_wrapper_is_set", + "value": false }, { - "key": "experimental.use_legacy_run_as", + "key": "presets_name_prefix_is_set", "value": false }, { - "key": "run_as_set", - "value": false + "key": "app_lifecycle_started", + "value": true }, { - "key": "presets_name_prefix_is_set", + "key": "app_lifecycle_started", "value": false }, { - "key": "python_wheel_wrapper_is_set", + "key": "deployment_history_enabled", "value": false }, { - "key": "skip_artifact_cleanup", + "key": "immutable_snapshot_used", "value": false }, { "key": "state_path_is_shared", "value": false }, + { + "key": "dms_compat_auto", + "value": true + }, { "key": "permissions_section_set", "value": false @@ -74,31 +74,35 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "value": false }, { - "key": "dms_compat_auto", - "value": true + "key": "skip_artifact_cleanup", + "value": false }, { - "key": "has_serverless_compute", + "key": "experimental.use_legacy_run_as", "value": false }, { - "key": "has_classic_job_compute", + "key": "run_as_set", "value": false }, { - "key": "has_classic_interactive_compute", + "key": "has_serverless_compute", "value": false }, { - "key": "deployment_history_enabled", + "key": "has_classic_job_compute", "value": false }, { - "key": "immutable_snapshot_used", + "key": "has_classic_interactive_compute", "value": false }, { - "key": "app_lifecycle_started", + "key": "local.cache.attempt", + "value": true + }, + { + "key": "local.cache.miss", "value": true } ] diff --git a/acceptance/bundle/telemetry/deploy-app-lifecycle-started/stopped/app.py b/acceptance/bundle/telemetry/deploy-app-lifecycle-started/stopped/app.py new file mode 100644 index 00000000000..81c2b1e610f --- /dev/null +++ b/acceptance/bundle/telemetry/deploy-app-lifecycle-started/stopped/app.py @@ -0,0 +1 @@ +print("stopped") diff --git a/acceptance/bundle/telemetry/deploy-app-lifecycle-started/test.toml b/acceptance/bundle/telemetry/deploy-app-lifecycle-started/test.toml new file mode 100644 index 00000000000..f03b652f20a --- /dev/null +++ b/acceptance/bundle/telemetry/deploy-app-lifecycle-started/test.toml @@ -0,0 +1,4 @@ +# The two apps deploy in parallel and only the started one prints its deployment +# message, which can land on either side of the stopped app's "Created" line. +# Suppressing it leaves the "Created" lines adjacent, so deploy_actions sorts them. +SuppressLines.app_deployment = '^✓ Deployment succeeded$' diff --git a/acceptance/bundle/telemetry/deploy-load-error/broken.yml b/acceptance/bundle/telemetry/deploy-load-error/broken.yml new file mode 100644 index 00000000000..6d156ba4843 --- /dev/null +++ b/acceptance/bundle/telemetry/deploy-load-error/broken.yml @@ -0,0 +1 @@ +resources: [ diff --git a/acceptance/bundle/telemetry/deploy-load-error/databricks.yml b/acceptance/bundle/telemetry/deploy-load-error/databricks.yml new file mode 100644 index 00000000000..8ac7c98b1a4 --- /dev/null +++ b/acceptance/bundle/telemetry/deploy-load-error/databricks.yml @@ -0,0 +1,5 @@ +bundle: + name: test-bundle + +include: + - broken.yml diff --git a/acceptance/bundle/telemetry/deploy-load-error/out.test.toml b/acceptance/bundle/telemetry/deploy-load-error/out.test.toml new file mode 100644 index 00000000000..19b5b15d11f --- /dev/null +++ b/acceptance/bundle/telemetry/deploy-load-error/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/telemetry/deploy-load-error/output.txt b/acceptance/bundle/telemetry/deploy-load-error/output.txt new file mode 100644 index 00000000000..7789f8b6606 --- /dev/null +++ b/acceptance/bundle/telemetry/deploy-load-error/output.txt @@ -0,0 +1,4 @@ + +>>> [CLI] bundle deploy +Error: failed to load [TEST_TMP_DIR]/broken.yml: yaml: line 1: did not find expected node content + diff --git a/acceptance/bundle/telemetry/deploy-load-error/script b/acceptance/bundle/telemetry/deploy-load-error/script new file mode 100644 index 00000000000..df2e79eba90 --- /dev/null +++ b/acceptance/bundle/telemetry/deploy-load-error/script @@ -0,0 +1,4 @@ +# The included broken.yml does not parse, so loading stops before the bundle bitmap is started. +# Deploy telemetry is still assembled on this path (nothing is uploaded: there is no workspace +# client yet), so the deploy must fail with the load error instead of crashing. +musterr trace $CLI bundle deploy diff --git a/acceptance/bundle/telemetry/deploy/out.bitmap.txt b/acceptance/bundle/telemetry/deploy/out.bitmap.txt new file mode 100644 index 00000000000..0ecf8df6420 --- /dev/null +++ b/acceptance/bundle/telemetry/deploy/out.bitmap.txt @@ -0,0 +1,33 @@ +P bundle +P bundle.name +P bundle.uuid +P resources +P resources.jobs +P resources.jobs.* +P resources.jobs.*.name +P resources.pipelines +P resources.pipelines.* +P resources.pipelines.*.name +P sync +P sync.paths +P sync.paths[*] +P F telemetry.PythonWheelWrapperIsSet +P F telemetry.PresetsNamePrefixIsSet +P F telemetry.DeploymentHistoryEnabled +P F telemetry.ImmutableSnapshotUsed +P F telemetry.StatePathIsShared +PT telemetry.DMSCompatAuto +P F telemetry.PermissionsSectionSet +PT telemetry.StatePathInDeployerHome +P F telemetry.StatePathInOtherUserHome +P F telemetry.StatePathOther +P F telemetry.DMSUndeclaredDeployingUser +P F telemetry.DMSUndeclaredOtherUser +P F telemetry.DMSUndeclaredServicePrincipal +P F telemetry.DMSUndeclaredGroup +P F telemetry.SkipArtifactCleanup +P F telemetry.UseLegacyRunAs +P F telemetry.RunAsSet +P F telemetry.HasServerlessCompute +P F telemetry.HasClassicJobCompute +P F telemetry.HasClassicInteractiveCompute diff --git a/acceptance/bundle/telemetry/deploy/out.telemetry.txt b/acceptance/bundle/telemetry/deploy/out.telemetry.txt index 1945afe889c..c8b737da327 100644 --- a/acceptance/bundle/telemetry/deploy/out.telemetry.txt +++ b/acceptance/bundle/telemetry/deploy/out.telemetry.txt @@ -44,19 +44,7 @@ "target_count": 1, "bool_values": [ { - "key": "local.cache.attempt", - "value": true - }, - { - "key": "local.cache.miss", - "value": true - }, - { - "key": "experimental.use_legacy_run_as", - "value": false - }, - { - "key": "run_as_set", + "key": "python_wheel_wrapper_is_set", "value": false }, { @@ -64,17 +52,21 @@ "value": false }, { - "key": "python_wheel_wrapper_is_set", + "key": "deployment_history_enabled", "value": false }, { - "key": "skip_artifact_cleanup", + "key": "immutable_snapshot_used", "value": false }, { "key": "state_path_is_shared", "value": false }, + { + "key": "dms_compat_auto", + "value": true + }, { "key": "permissions_section_set", "value": false @@ -108,8 +100,16 @@ "value": false }, { - "key": "dms_compat_auto", - "value": true + "key": "skip_artifact_cleanup", + "value": false + }, + { + "key": "experimental.use_legacy_run_as", + "value": false + }, + { + "key": "run_as_set", + "value": false }, { "key": "has_serverless_compute", @@ -124,12 +124,12 @@ "value": false }, { - "key": "deployment_history_enabled", - "value": false + "key": "local.cache.attempt", + "value": true }, { - "key": "immutable_snapshot_used", - "value": false + "key": "local.cache.miss", + "value": true } ], "bundle_mode": "TYPE_UNSPECIFIED", diff --git a/acceptance/bundle/telemetry/deploy/script b/acceptance/bundle/telemetry/deploy/script index ef428a814a1..c7488a0aba8 100644 --- a/acceptance/bundle/telemetry/deploy/script +++ b/acceptance/bundle/telemetry/deploy/script @@ -24,6 +24,11 @@ cat telemetry.json | jq '.entry.databricks_cli_log.bundle_deploy_event.resources title "Assert the terraform engine opt-in is reported" cat telemetry.json | jq '[.entry.databricks_cli_log.bundle_deploy_event.experimental.bool_values[] | select(.key | startswith("engine_terraform_"))]' > out.engine.txt +# The bundle bitmap that the deploy sends, decoded: the config fields the bundle sets and the telemetry +# flags measured, including those only a deploy measures. The encoded form and the size header grow +# with the schema, so they are left out. +cat telemetry.json | jq -r '.entry.databricks_cli_log.bundle_deploy_event.bundle_bitmap' | $CLI bundle bitmap decode | tail -n +2 > out.bitmap.txt + # bundle_mutator_execution_time_ms can have variable number of entries depending upon the runtime of the mutators. Thus we omit it from # being asserted here. # @@ -32,7 +37,7 @@ cat telemetry.json | jq '[.entry.databricks_cli_log.bundle_deploy_event.experime # The resource ids are listed in creation order, which varies because resources are created in # parallel. Sort them by resource key so the order is stable. ids=$($CLI bundle summary -o json | jq '[.resources[] | to_entries[] | {key: (.value.id | tostring), value: .key}] | from_entries') -cat telemetry.json | jq --argjson ids "$ids" 'del(.entry.databricks_cli_log.bundle_deploy_event.experimental.bundle_mutator_execution_time_ms, .entry.databricks_cli_log.bundle_deploy_event.resources_metadata) | .entry.databricks_cli_log.bundle_deploy_event.experimental.bool_values |= map(select(.key | startswith("engine_terraform_") | not)) | .entry.databricks_cli_log.bundle_deploy_event |= (.resource_job_ids |= sort_by($ids[.]) | .resource_pipeline_ids |= sort_by($ids[.]))' > out.telemetry.txt +cat telemetry.json | jq --argjson ids "$ids" 'del(.entry.databricks_cli_log.bundle_deploy_event.experimental.bundle_mutator_execution_time_ms, .entry.databricks_cli_log.bundle_deploy_event.resources_metadata, .entry.databricks_cli_log.bundle_deploy_event.bundle_bitmap) | .entry.databricks_cli_log.bundle_deploy_event.experimental.bool_values |= map(select(.key | startswith("engine_terraform_") | not)) | .entry.databricks_cli_log.bundle_deploy_event |= (.resource_job_ids |= sort_by($ids[.]) | .resource_pipeline_ids |= sort_by($ids[.]))' > out.telemetry.txt cmd_exec_id=$(extract_command_exec_id.py) deployment_id=$(cat .databricks/bundle/default/deployment.json | jq -r .id) diff --git a/acceptance/bundle/templates/default-python/classic/out.bitmap.txt b/acceptance/bundle/templates/default-python/classic/out.bitmap.txt new file mode 100644 index 00000000000..e1043d191f6 --- /dev/null +++ b/acceptance/bundle/templates/default-python/classic/out.bitmap.txt @@ -0,0 +1,76 @@ +P variables +P variables.* +P variables.*.default +P variables.*.description +P bundle +P bundle.name +P bundle.mode +P bundle.uuid +P include +P include[*] +P workspace +P workspace.host +P artifacts +P artifacts.* +P artifacts.*.type +P artifacts.*.build +P resources +P resources.jobs +P resources.jobs.* +P resources.jobs.*.job_clusters +P resources.jobs.*.job_clusters[*] +P resources.jobs.*.job_clusters[*].job_cluster_key +P resources.jobs.*.job_clusters[*].new_cluster +P resources.jobs.*.job_clusters[*].new_cluster.autoscale +P resources.jobs.*.job_clusters[*].new_cluster.autoscale.max_workers +P resources.jobs.*.job_clusters[*].new_cluster.autoscale.min_workers +P resources.jobs.*.job_clusters[*].new_cluster.data_security_mode +P resources.jobs.*.job_clusters[*].new_cluster.node_type_id +P resources.jobs.*.job_clusters[*].new_cluster.spark_version +P resources.jobs.*.name +P resources.jobs.*.parameters +P resources.jobs.*.parameters[*] +P resources.jobs.*.parameters[*].default +P resources.jobs.*.parameters[*].name +P resources.jobs.*.tasks +P resources.jobs.*.tasks[*] +P resources.jobs.*.tasks[*].depends_on +P resources.jobs.*.tasks[*].depends_on[*] +P resources.jobs.*.tasks[*].depends_on[*].task_key +P resources.jobs.*.tasks[*].job_cluster_key +P resources.jobs.*.tasks[*].libraries +P resources.jobs.*.tasks[*].libraries[*] +P resources.jobs.*.tasks[*].libraries[*].whl +P resources.jobs.*.tasks[*].notebook_task +P resources.jobs.*.tasks[*].notebook_task.notebook_path +P resources.jobs.*.tasks[*].pipeline_task +P resources.jobs.*.tasks[*].pipeline_task.pipeline_id +P resources.jobs.*.tasks[*].python_wheel_task +P resources.jobs.*.tasks[*].python_wheel_task.entry_point +P resources.jobs.*.tasks[*].python_wheel_task.package_name +P resources.jobs.*.tasks[*].python_wheel_task.parameters +P resources.jobs.*.tasks[*].python_wheel_task.parameters[*] +P resources.jobs.*.tasks[*].task_key +P resources.jobs.*.trigger +P resources.jobs.*.trigger.periodic +P resources.jobs.*.trigger.periodic.interval +P resources.jobs.*.trigger.periodic.unit +P resources.pipelines +P resources.pipelines.* +P resources.pipelines.*.environment +P resources.pipelines.*.environment.dependencies +P resources.pipelines.*.environment.dependencies[*] +P resources.pipelines.*.libraries +P resources.pipelines.*.libraries[*] +P resources.pipelines.*.libraries[*].glob +P resources.pipelines.*.libraries[*].glob.include +P resources.pipelines.*.name +P resources.pipelines.*.root_path +P resources.pipelines.*.schema +P presets +PT presets.artifacts_dynamic_version +PT telemetry.ArtifactBuildCommandIsSet +P F telemetry.ArtifactFilesIsSet +PT telemetry.PresetsNamePrefixIsSet +P F telemetry.UseLegacyRunAs +P F telemetry.RunAsSet diff --git a/acceptance/bundle/templates/default-python/classic/script b/acceptance/bundle/templates/default-python/classic/script index a1227e1fb0d..fa6049b2826 100644 --- a/acceptance/bundle/templates/default-python/classic/script +++ b/acceptance/bundle/templates/default-python/classic/script @@ -4,6 +4,10 @@ cd output/my_default_python trace $CLI bundle validate -t dev trace $CLI bundle validate -t prod +# Record which fields of this realistic template the bitmap reports as set. tail skips the +# size=N header: N grows with every field appended to the schema. +$CLI bundle bitmap bitmap -o text -t dev | tail -n +2 > ../../out.bitmap.txt + trace $CLI bundle plan -t dev trace $CLI bundle plan -t prod diff --git a/acceptance/selftest/selection/output.txt b/acceptance/selftest/selection/output.txt index 7e2c55b1c1d..2b0d00380dc 100644 --- a/acceptance/selftest/selection/output.txt +++ b/acceptance/selftest/selection/output.txt @@ -28,7 +28,7 @@ Selected 2 changed tests (limit=50, 0 not selected) === Touching an invariant config runs every invariant test, for that config only >>> selection M:acceptance/bundle/invariant/configs/job.yml.tmpl -Selected 6 changed tests (limit=50, 0 not selected) +Selected 7 changed tests (limit=50, 0 not selected) 5 ^bundle$/^invariant$/^auto-migrate$/^DMS=$/^INPUT_CONFIG=job\.yml\.tmpl$ 5 ^bundle$/^invariant$/^continue_293$/^DMS=$/^INPUT_CONFIG=job\.yml\.tmpl$ 5 ^bundle$/^invariant$/^delete_idempotent$/^DMS=$/^INPUT_CONFIG=job\.yml\.tmpl$ @@ -36,11 +36,12 @@ Selected 6 changed tests (limit=50, 0 not selected) 5 ^bundle$/^invariant$/^migrate$/^DMS=$/^INPUT_CONFIG=job\.yml\.tmpl$ 5 ^bundle$/^invariant$/^no_drift$/^DMS=$/^INPUT_CONFIG=job\.yml\.tmpl$ 5 ^bundle$/^invariant$/^no_drift$/^DMS=true$/^INPUT_CONFIG=job\.yml\.tmpl$ + 5 ^bundle$/^invariant$/^validate_bitmap$/^INPUT_CONFIG=job\.yml\.tmpl$ === Adding one invariant config and touching another: only the new variant is new >>> selection A:acceptance/bundle/invariant/configs/pipeline.yml.tmpl M:acceptance/bundle/invariant/configs/job.yml.tmpl -Selected 12 changed tests (limit=50, 0 not selected) +Selected 14 changed tests (limit=50, 0 not selected) 10 ^bundle$/^invariant$/^auto-migrate$/^DMS=$/^INPUT_CONFIG=pipeline\.yml\.tmpl$ 10 ^bundle$/^invariant$/^continue_293$/^DMS=$/^INPUT_CONFIG=pipeline\.yml\.tmpl$ 10 ^bundle$/^invariant$/^delete_idempotent$/^DMS=$/^INPUT_CONFIG=pipeline\.yml\.tmpl$ @@ -48,6 +49,7 @@ Selected 12 changed tests (limit=50, 0 not selected) 10 ^bundle$/^invariant$/^migrate$/^DMS=$/^INPUT_CONFIG=pipeline\.yml\.tmpl$ 10 ^bundle$/^invariant$/^no_drift$/^DMS=$/^INPUT_CONFIG=pipeline\.yml\.tmpl$ 10 ^bundle$/^invariant$/^no_drift$/^DMS=true$/^INPUT_CONFIG=pipeline\.yml\.tmpl$ + 10 ^bundle$/^invariant$/^validate_bitmap$/^INPUT_CONFIG=pipeline\.yml\.tmpl$ 5 ^bundle$/^invariant$/^auto-migrate$/^DMS=$/^INPUT_CONFIG=job\.yml\.tmpl$ 5 ^bundle$/^invariant$/^continue_293$/^DMS=$/^INPUT_CONFIG=job\.yml\.tmpl$ 5 ^bundle$/^invariant$/^delete_idempotent$/^DMS=$/^INPUT_CONFIG=job\.yml\.tmpl$ @@ -55,15 +57,17 @@ Selected 12 changed tests (limit=50, 0 not selected) 5 ^bundle$/^invariant$/^migrate$/^DMS=$/^INPUT_CONFIG=job\.yml\.tmpl$ 5 ^bundle$/^invariant$/^no_drift$/^DMS=$/^INPUT_CONFIG=job\.yml\.tmpl$ 5 ^bundle$/^invariant$/^no_drift$/^DMS=true$/^INPUT_CONFIG=job\.yml\.tmpl$ + 5 ^bundle$/^invariant$/^validate_bitmap$/^INPUT_CONFIG=job\.yml\.tmpl$ === A config an invariant test excludes does not select that test >>> selection M:acceptance/bundle/invariant/configs/vector_search_index.yml.tmpl -Selected 3 changed tests (limit=50, 0 not selected) +Selected 4 changed tests (limit=50, 0 not selected) 5 ^bundle$/^invariant$/^delete_idempotent$/^DMS=$/^INPUT_CONFIG=vector_search_index\.yml\.tmpl$ 5 ^bundle$/^invariant$/^destroy_idempotent$/^DMS=$/^INPUT_CONFIG=vector_search_index\.yml\.tmpl$ 5 ^bundle$/^invariant$/^no_drift$/^DMS=$/^INPUT_CONFIG=vector_search_index\.yml\.tmpl$ 5 ^bundle$/^invariant$/^no_drift$/^DMS=true$/^INPUT_CONFIG=vector_search_index\.yml\.tmpl$ + 5 ^bundle$/^invariant$/^validate_bitmap$/^INPUT_CONFIG=vector_search_index\.yml\.tmpl$ === Renaming a fixture out of a test selects the test that lost it @@ -74,7 +78,7 @@ Selected 1 changed tests (limit=50, 0 not selected) === The limit keeps the highest scoring tests >>> selection -limit 3 A:acceptance/selftest/basic/script M:acceptance/selftest/diff/script M:acceptance/bundle/invariant/configs/job.yml.tmpl -Selected 3 changed tests (limit=3, 5 not selected) +Selected 3 changed tests (limit=3, 6 not selected) 10 ^selftest$/^basic$ 5 ^bundle$/^invariant$/^auto-migrate$/^DMS=$/^INPUT_CONFIG=job\.yml\.tmpl$ 5 ^bundle$/^invariant$/^continue_293$/^DMS=$/^INPUT_CONFIG=job\.yml\.tmpl$ diff --git a/bundle/artifacts/build.go b/bundle/artifacts/build.go index 10d1f0aa03c..b3ff560d293 100644 --- a/bundle/artifacts/build.go +++ b/bundle/artifacts/build.go @@ -10,7 +10,6 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/bundle/metrics" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/exec" @@ -80,7 +79,7 @@ func (m *build) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { } if a.Type == "whl" && config.IsExplicitlyEnabled(a.DynamicVersion) && cacheDir != "" { - b.Metrics.AddBoolValue(metrics.ArtifactDynamicVersionIsSet, true) + b.Telemetry.ArtifactDynamicVersionIsSet.SetBool(true) for ind, artifactFile := range a.Files { patchedWheel := makePatchedWheel(ctx, cacheDir, artifactName, artifactFile.Source) log.Debugf(ctx, "Patching ind=%d artifactName=%s Source=%s patchedWheel=%s", ind, artifactName, artifactFile.Source, patchedWheel) diff --git a/bundle/artifacts/prepare.go b/bundle/artifacts/prepare.go index e285eb1ee30..8ac914b2eb8 100644 --- a/bundle/artifacts/prepare.go +++ b/bundle/artifacts/prepare.go @@ -12,7 +12,6 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/libraries" - "github.com/databricks/cli/bundle/metrics" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/logdiag" @@ -47,8 +46,8 @@ func (m *prepare) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics }) continue } - b.Metrics.AddBoolValue(metrics.ArtifactBuildCommandIsSet, artifact.BuildCommand != "") - b.Metrics.AddBoolValue(metrics.ArtifactFilesIsSet, len(artifact.Files) != 0) + b.Telemetry.ArtifactBuildCommandIsSet.SetBool(artifact.BuildCommand != "") + b.Telemetry.ArtifactFilesIsSet.SetBool(len(artifact.Files) != 0) // A `tgz` artifact with `include`/`git` is built by DABs itself in the build // phase (see artifacts.Build). `build` and `git`/`include` are mutually diff --git a/bundle/artifacts/upload.go b/bundle/artifacts/upload.go index c5b6e225d9f..35aaf0ad976 100644 --- a/bundle/artifacts/upload.go +++ b/bundle/artifacts/upload.go @@ -28,7 +28,7 @@ func (m *cleanUp) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics } skipArtifactsCleanup := b.Config.Experimental != nil && config.IsExplicitlyEnabled(b.Config.Experimental.SkipArtifactCleanup) - b.Metrics.AddBoolValue("skip_artifact_cleanup", skipArtifactsCleanup) + b.Telemetry.SkipArtifactCleanup = new(skipArtifactsCleanup) if skipArtifactsCleanup { log.Info(ctx, "Skip cleaning up artifacts folder") } else { diff --git a/bundle/bitmap/bitmap.go b/bundle/bitmap/bitmap.go new file mode 100644 index 00000000000..0d27489c7e6 --- /dev/null +++ b/bundle/bitmap/bitmap.go @@ -0,0 +1,623 @@ +// Package bitmap encodes the presence of every bundle configuration field as a +// single bit. The schema (the ordered list of field paths) is derived by +// reflecting over config.Root and recorded in schema.deflate, which is embedded +// into the binary (acceptance/bundle/bitmap/full lists it). The schema is +// append-only so that bit positions stay stable across releases: a newer binary +// can decode an older bitmap and vice versa. +// +// The bits describe the configuration as the user wrote it, not the result of +// Initialize, so defaults the CLI fills in are not counted (see Builder). +package bitmap + +import ( + "bytes" + "compress/flate" + "encoding/base64" + "encoding/binary" + "fmt" + "io" + "reflect" + "slices" + "strings" + + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/convert" + "github.com/databricks/cli/libs/dyn/dynvar" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structwalk" +) + +// magic identifies a bundle telemetry bitmap. The bytes are chosen so they do +// not collide with the leading bytes of the compression formats we wrap the +// payload in (gzip 0x1f 0x8b, zlib 0x78), so a decoder can tell a raw payload +// from a compressed one. +const magic = "DBTB" + +// ContextFullBundle marks a bitmap covering the whole user-authored config. Other +// context values are reserved for narrower bitmaps in the future. +const ContextFullBundle uint16 = 0 + +// headerSize is the size of the fixed header: magic (4) + size (4) + context (2). +const headerSize = 10 + +// EmbeddedSchema returns the schema baked into the binary at build time. +func EmbeddedSchema() []string { + text, err := io.ReadAll(flate.NewReader(bytes.NewReader(schemaDeflate))) + if err != nil { + // schema.deflate is generated and checked by TestEmbeddedSchemaIsCompleteAndUnique. + panic(fmt.Sprintf("bitmap: corrupt embedded schema: %s", err)) + } + return parseSchema(text) +} + +// CompressSchema encodes schema text, one field path per line, the way +// schema.deflate stores it: raw DEFLATE. compress/flate gives the same bytes on +// every platform, so the generated file does not depend on who generated it. +func CompressSchema(text []byte) ([]byte, error) { + var out bytes.Buffer + w, err := flate.NewWriter(&out, flate.BestCompression) + if err != nil { + return nil, err + } + if _, err := w.Write(text); err != nil { + return nil, err + } + if err := w.Close(); err != nil { + return nil, err + } + return out.Bytes(), nil +} + +// telemetryPrefix namespaces the internal telemetry struct's fields in the +// schema so they never collide with config.Root paths. +const telemetryPrefix = "telemetry" + +// Every bool field has two value entries after its presence entry, so the +// bitmap records true and false apart. A present field with neither value bit +// holds an unresolved variable reference. +const ( + trueSuffix = "=true" + falseSuffix = "=false" +) + +// listSuffix marks a list element in a schema path. +const listSuffix = "[*]" + +// WalkSchema reflects over config.Root and the internal telemetry struct and +// returns the ordered list of field paths, one per field, with map keys +// rendered as ".*" and slice elements as "[*]", and a "=true" and "=false" +// entry after each bool field. The telemetry struct's fields are namespaced +// under "telemetry.". This is the freshly computed schema; the +// committed schema.deflate is a prefix of it plus any fields since removed. +// +// telemetryType is the internal telemetry struct type (bundle.Telemetry), +// passed in so this package does not import the bundle package. +func WalkSchema(telemetryType reflect.Type) ([]string, error) { + paths, err := walkTypePaths(reflect.TypeFor[config.Root](), "", isPruned) + if err != nil { + return nil, err + } + tmPaths, err := walkTypePaths(telemetryType, telemetryPrefix+".", nil) + if err != nil { + return nil, err + } + // The walk visits some paths twice: a flattened embed at its parent's path, + // and a field together with an embedded field it shadows. + unique, _ := Merge(nil, append(paths, tmPaths...)) + return unique, nil +} + +// walkTypePaths walks a struct type and returns each field path, prefixed and +// with pruned subtrees removed. prune may be nil. +func walkTypePaths(t reflect.Type, prefix string, prune func(string) bool) ([]string, error) { + var paths []string + err := structwalk.WalkType(t, func(path *structpath.PatternNode, typ reflect.Type, _ *reflect.StructField) bool { + if path == nil { + return true + } + p := path.String() + if prune != nil && prune(p) { + return false + } + paths = append(paths, prefix+p) + if typ.Kind() == reflect.Pointer { + typ = typ.Elem() + } + if typ.Kind() == reflect.Bool || typ.Implements(boolValuesType) { + paths = append(paths, prefix+p+trueSuffix, prefix+p+falseSuffix) + } + return true + }) + if err != nil { + return nil, err + } + return paths, nil +} + +// isPruned reports whether a schema path should be excluded from the bitmap. +func isPruned(p string) bool { + // targets/environments are per-target overrides that are merged away and set + // to nil by the SelectTarget mutator, so they are never present at deploy + // time and would only duplicate the merged tree. __locations is an + // output-only internal field. + switch { + case p == "targets" || strings.HasPrefix(p, "targets."): + return true + case p == "environments" || strings.HasPrefix(p, "environments."): + return true + case p == "__locations" || strings.HasPrefix(p, "__locations."): + return true + // Target selection sets these on every bundle; users do not write them. + case p == "bundle.target" || p == "bundle.environment": + return true + } + return false +} + +// Merge appends to old any paths in fresh that are not already present, keeping +// old's order intact. Paths removed from config.Root remain in the merged schema +// so that existing bit positions never shift. It returns the merged schema and +// the list of newly added paths. +func Merge(old, fresh []string) (merged, added []string) { + seen := make(map[string]struct{}, len(old)) + for _, p := range old { + seen[p] = struct{}{} + } + merged = append(merged, old...) + for _, p := range fresh { + if _, ok := seen[p]; ok { + continue + } + seen[p] = struct{}{} + merged = append(merged, p) + added = append(added, p) + } + return merged, added +} + +// Builder computes the config bits of a bundle as the user wrote it. Mark runs +// on the loaded configuration, and on the resources Python adds, before any +// mutator fills in defaults. A field holding a ${var.*} reference has no value +// yet at that point, so Mark keeps it aside, and ResolveVariables marks the +// variable's value once it is known. Bits are only ever set, never cleared. +type Builder struct { + index map[string]int + bits []bool + + // refs are the fields Mark found holding a pure ${var.*} reference. + refs map[varRef]struct{} +} + +// varRef is a field that held a pure ${var.*} reference. +type varRef struct { + pattern string // the field's schema pattern + reference string // e.g. "${var.flag}" +} + +// varPrefix is the first component of a variable reference: ${var.name}. +var varPrefix = dyn.NewPath(dyn.Key("var")) + +// NewBuilder returns a Builder for schema with no bits set. +func NewBuilder(schema []string) *Builder { + return &Builder{ + index: indexSchema(schema), + bits: make([]bool, len(schema)), + refs: map[varRef]struct{}{}, + } +} + +// Mark sets the bits of the fields present in cfg, a configuration tree rooted +// at the configuration root. The config is walked as its dyn.Value rather than +// the typed struct so that presence is exactly what the config holds: an +// interface-typed field (variables.*.default) or an empty container ({} or []) +// counts, and a zero value counts only when it is written out. +func (b *Builder) Mark(cfg dyn.Value) { + b.markChildren(cfg, "") +} + +// MarkChanges sets the bits of what after holds that before does not: fields +// before lacks, values that differ, and the fields on the way to them. Both are +// rooted at the configuration root. List elements are compared by position. +func (b *Builder) MarkChanges(before, after dyn.Value) { + b.markChangedChildren(before, after, "") +} + +// ResolveVariables marks the values of the fields Mark found holding a +// ${var.*} reference, reading each variable's value from cfg, the processed +// configuration. The values are the ones the user supplied (default, --var, +// environment or file), whatever mutators later do to the field. A reference +// that does not resolve keeps only the presence bit Mark set. +func (b *Builder) ResolveVariables(cfg dyn.Value) { + refs := b.refs + b.refs = map[varRef]struct{}{} + for ref := range refs { + v, err := dynvar.Resolve(dyn.V(ref.reference), func(p dyn.Path) (dyn.Value, error) { + return variableValue(cfg, p) + }) + if err != nil { + continue + } + // A value from --var or the environment is a string; coerce it the way + // the loader does when it reaches a bool field. + if _, ok := b.index[ref.pattern+trueSuffix]; ok { + normalized, diags := convert.Normalize(false, v) + if diags.HasError() { + continue + } + v = normalized + } + b.mark(v, ref.pattern) + } +} + +// variableValue looks up the value of a ${var.name...} reference in cfg, which +// keeps it under variables.name.value. +func variableValue(cfg dyn.Value, p dyn.Path) (dyn.Value, error) { + if len(p) < 2 || !p.HasPrefix(varPrefix) { + return dyn.InvalidValue, dynvar.ErrSkipResolution + } + path := append(dyn.NewPath(dyn.Key("variables"), p[1], dyn.Key("value")), p[2:]...) + return dyn.GetByPath(cfg, path) +} + +// Bits returns a bit per schema entry: the bits Mark and ResolveVariables set, +// together with those of the measured telemetry fields, namespaced under +// "telemetry.". A "=true"/"=false" entry is set when the bool held that value. +func (b *Builder) Bits(telemetry any) ([]bool, error) { + bits := slices.Clone(b.bits) + if err := markTelemetry(telemetry, b.index, bits); err != nil { + return nil, err + } + return bits, nil +} + +// boolValues is implemented by bundle.BoolSet, a telemetry field that records +// each value it took. It is matched by method so that this package does not +// import the bundle package. +type boolValues interface{ Values() []bool } + +var boolValuesType = reflect.TypeFor[boolValues]() + +// markTelemetry sets the bits of the measured telemetry fields: the presence +// bit and the bit of each value taken. A bool counts only when true. +func markTelemetry(telemetry any, index map[string]int, bits []bool) error { + v := reflect.ValueOf(telemetry) + for i := range v.NumField() { + name := v.Type().Field(i).Name + var values []bool + switch f := v.Field(i).Interface().(type) { + case bool: + if f { + values = []bool{true} + } + case *bool: + if f != nil { + values = []bool{*f} + } + case boolValues: + values = f.Values() + default: + return fmt.Errorf("telemetry field %s has unsupported type %s", name, v.Field(i).Type()) + } + path := joinField(telemetryPrefix, name) + for _, value := range values { + setBit(index, bits, path) + setBit(index, bits, path+valueSuffix(value)) + } + } + return nil +} + +// markChildren marks every child of v whose schema pattern is known. A map key +// resolves to the struct field of that name when the schema has one and to the +// ".*" wildcard otherwise; a sequence element resolves to "[*]". +func (b *Builder) markChildren(v dyn.Value, pattern string) { + if m, ok := v.AsMap(); ok { + for _, pair := range m.Pairs() { + b.mark(pair.Value, b.childPattern(pattern, pair.Key.MustString())) + } + } + if s, ok := v.AsSequence(); ok { + for _, elem := range s { + b.mark(elem, pattern+listSuffix) + } + } +} + +// childPattern returns the schema pattern of the map entry key under pattern: +// the field of that name, or the ".*" wildcard of a map. +func (b *Builder) childPattern(pattern, key string) string { + child := joinField(pattern, key) + if _, ok := b.index[child]; !ok { + child = joinField(pattern, "*") + } + return child +} + +// markChanged marks after at pattern where it differs from before, and reports +// whether it marked anything. +func (b *Builder) markChanged(before, after dyn.Value, pattern string) bool { + if _, ok := b.index[pattern]; !ok || after.Kind() == dyn.KindNil { + return false + } + switch { + case before.Kind() != after.Kind(): + b.mark(after, pattern) + return true + case after.Kind() == dyn.KindMap || after.Kind() == dyn.KindSequence: + if !b.markChangedChildren(before, after, pattern) { + return false + } + setBit(b.index, b.bits, pattern) + return true + case before.AsAny() == after.AsAny(): + return false + default: + b.mark(after, pattern) + return true + } +} + +// markChangedChildren is markChanged for the map entries and list elements of +// after, and reports whether it marked any. +func (b *Builder) markChangedChildren(before, after dyn.Value, pattern string) bool { + changed := false + if m, ok := after.AsMap(); ok { + for _, pair := range m.Pairs() { + key := pair.Key.MustString() + changed = b.markChanged(before.Get(key), pair.Value, b.childPattern(pattern, key)) || changed + } + } + if s, ok := after.AsSequence(); ok { + prev, _ := before.AsSequence() + for i, elem := range s { + old := dyn.InvalidValue + if i < len(prev) { + old = prev[i] + } + changed = b.markChanged(old, elem, pattern+listSuffix) || changed + } + } + return changed +} + +// mark sets the bit for a present (non-null) value and recurses into it. A +// pattern outside the schema is not descended into: that covers the pruned +// subtrees and the contents of interface-typed fields, whose own bit is set. +func (b *Builder) mark(v dyn.Value, pattern string) { + i, ok := b.index[pattern] + if !ok || v.Kind() == dyn.KindNil { + return + } + b.bits[i] = true + if value, ok := v.AsBool(); ok { + setBit(b.index, b.bits, pattern+valueSuffix(value)) + } + if s, ok := v.AsString(); ok && isVariableReference(s) { + b.refs[varRef{pattern: pattern, reference: s}] = struct{}{} + } + b.markChildren(v, pattern) +} + +// isVariableReference reports whether s is exactly one ${var.*} reference. +func isVariableReference(s string) bool { + p, ok := dynvar.PureReferenceToPath(s) + return ok && p.HasPrefix(varPrefix) +} + +// setBit sets the bit of schema path p, if the schema has it. +func setBit(index map[string]int, bits []bool, p string) { + if i, ok := index[p]; ok { + bits[i] = true + } +} + +// valueSuffix returns the suffix of the schema entry for a bool value. +func valueSuffix(value bool) string { + if value { + return trueSuffix + } + return falseSuffix +} + +func joinField(pattern, key string) string { + if pattern == "" { + return key + } + return pattern + "." + key +} + +// Text renders the bitmap as a "size=N context=C" header followed by one +// "PTF " line per set field in schema order: "P" for a present field, "T" +// and "F" for a bool field seen holding true and false, and a space for each +// that is not set. A bool with "P" alone holds an unresolved variable +// reference; one with both "T" and "F" has both values under a wildcard. Unset +// fields are omitted. bits may be shorter than schema (a bitmap from an older +// CLI) or longer (a newer CLI): a set bit past the schema is a "P" line named +// by futureField. +func Text(schema []string, bits []bool, context uint16) []string { + schema = schema[:min(len(bits), len(schema))] + index := indexSchema(schema) + lines := []string{fmt.Sprintf("size=%d context=%d", len(bits), context)} + for i, p := range schema { + if strings.HasSuffix(p, trueSuffix) || strings.HasSuffix(p, falseSuffix) { + continue + } + isTrue := isSet(index, bits, p+trueSuffix) + isFalse := isSet(index, bits, p+falseSuffix) + if !bits[i] && !isTrue && !isFalse { + continue + } + lines = append(lines, fmt.Sprintf("%c%c%c %s", letter(bits[i], 'P'), letter(isTrue, 'T'), letter(isFalse, 'F'), p)) + } + for i := len(schema); i < len(bits); i++ { + if bits[i] { + lines = append(lines, "P "+futureField(i)) + } + } + return lines +} + +// futureField names a set bit past the end of the schema, a field that only a +// newer CLI can name. Whether it is a field or the value of a bool is unknown. +func futureField(bit int) string { + return fmt.Sprintf("future #%d", bit) +} + +// letter returns c if set, and a space otherwise, so that Text's flags stay aligned. +func letter(set bool, c byte) byte { + if set { + return c + } + return ' ' +} + +// JSON is the JSON rendering of a bitmap. Presence, True and False are trees +// keyed by path segment ("*" for any map key, and a list field's name plus +// "[*]" for its elements) with true leaves: Presence holds every set field, +// True and False the bool fields seen with that value. A field with set +// children is a subtree rather than a leaf, since a child is only set when its +// parent is; likewise "tasks[*]" replaces the leaf of "tasks". +type JSON struct { + Size int `json:"size"` + Context uint16 `json:"context"` + Presence map[string]any `json:"presence"` + True map[string]any `json:"true"` + False map[string]any `json:"false"` +} + +// ToJSON renders bits as JSON. bits may be shorter than schema or longer: a set +// bit past the schema is a top-level Presence entry named by futureField. +func ToJSON(schema []string, bits []bool, context uint16) JSON { + out := JSON{ + Size: len(bits), + Context: context, + Presence: map[string]any{}, + True: map[string]any{}, + False: map[string]any{}, + } + known := min(len(bits), len(schema)) + for i, p := range schema[:known] { + if !bits[i] { + continue + } + tree := out.Presence + if field, ok := strings.CutSuffix(p, trueSuffix); ok { + tree, p = out.True, field + } else if field, ok := strings.CutSuffix(p, falseSuffix); ok { + tree, p = out.False, field + } + insertPath(tree, strings.Split(p, ".")) + } + for i := known; i < len(bits); i++ { + if bits[i] { + out.Presence[futureField(i)] = true + } + } + return out +} + +// insertPath adds a true leaf at keys, turning a leaf on the way into a +// subtree. A "tasks[*]" key replaces the "tasks" leaf next to it, and a "tasks" +// leaf is not added next to "tasks[*]". +func insertPath(tree map[string]any, keys []string) { + for i, k := range keys { + if field, ok := strings.CutSuffix(k, listSuffix); ok { + delete(tree, field) + } + if i == len(keys)-1 { + if _, ok := tree[k]; !ok && tree[k+listSuffix] == nil { + tree[k] = true + } + return + } + child, ok := tree[k].(map[string]any) + if !ok { + child = map[string]any{} + tree[k] = child + } + tree = child + } +} + +func isSet(index map[string]int, bits []bool, p string) bool { + i, ok := index[p] + return ok && bits[i] +} + +// indexSchema maps each schema path to its bit position. +func indexSchema(schema []string) map[string]int { + index := make(map[string]int, len(schema)) + for i, p := range schema { + index[p] = i + } + return index +} + +// Encode packs bits into the wire format (magic, size, context, bitmap), +// compresses it with DEFLATE, and base64-encodes the result. DEFLATE is used +// rather than gzip because it has no timestamp, keeping the output deterministic. +func Encode(bits []bool, context uint16) (string, error) { + n := len(bits) + raw := make([]byte, headerSize+(n+7)/8) + copy(raw, magic) + binary.BigEndian.PutUint32(raw[4:], uint32(n)) + binary.BigEndian.PutUint16(raw[8:], context) + for i, set := range bits { + if set { + raw[headerSize+i/8] |= 1 << uint(7-i%8) + } + } + + var buf bytes.Buffer + w, err := flate.NewWriter(&buf, flate.BestCompression) + if err != nil { + return "", err + } + if _, err := w.Write(raw); err != nil { + return "", err + } + if err := w.Close(); err != nil { + return "", err + } + return base64.StdEncoding.EncodeToString(buf.Bytes()), nil +} + +// Decode reverses Encode: it base64-decodes, inflates, validates the header, and +// returns the bits and context. +func Decode(encoded string) (bits []bool, context uint16, err error) { + compressed, err := base64.StdEncoding.DecodeString(encoded) + if err != nil { + return nil, 0, fmt.Errorf("base64: %w", err) + } + raw, err := io.ReadAll(flate.NewReader(bytes.NewReader(compressed))) + if err != nil { + return nil, 0, fmt.Errorf("inflate: %w", err) + } + if len(raw) < headerSize { + return nil, 0, fmt.Errorf("payload too short: %d bytes", len(raw)) + } + if string(raw[:4]) != magic { + return nil, 0, fmt.Errorf("bad magic %q", raw[:4]) + } + n := binary.BigEndian.Uint32(raw[4:]) + context = binary.BigEndian.Uint16(raw[8:]) + // Compare in uint64 so a corrupt size near 2^32 cannot overflow int on 32-bit platforms. + if uint64(len(raw)-headerSize) != (uint64(n)+7)/8 { + return nil, 0, fmt.Errorf("payload size mismatch: %d bits, %d bytes", n, len(raw)) + } + bits = make([]bool, n) + for i := range bits { + bits[i] = raw[headerSize+i/8]&(1< 0 { + assert.Equal(t, bits, decoded) + } + } + } +} + +func TestEncodeBitOrder(t *testing.T) { + encoded, err := Encode([]bool{true, false, false, false, false, false, false, true, true}, ContextFullBundle) + require.NoError(t, err) + + compressed, err := base64.StdEncoding.DecodeString(encoded) + require.NoError(t, err) + var raw bytes.Buffer + _, err = raw.ReadFrom(flate.NewReader(bytes.NewReader(compressed))) + require.NoError(t, err) + + assert.Equal(t, []byte{'D', 'B', 'T', 'B', 0, 0, 0, 9, 0, 0, 0b1000_0001, 0b1000_0000}, raw.Bytes()) +} + +// deflateBase64 wraps a raw payload the way Encode does, so Decode's header +// checks can be exercised on hand-built payloads. +func deflateBase64(t *testing.T, raw []byte) string { + var buf bytes.Buffer + w, err := flate.NewWriter(&buf, flate.BestCompression) + require.NoError(t, err) + _, err = w.Write(raw) + require.NoError(t, err) + require.NoError(t, w.Close()) + return base64.StdEncoding.EncodeToString(buf.Bytes()) +} + +func header(magic string, size uint32) []byte { + raw := make([]byte, headerSize) + copy(raw, magic) + binary.BigEndian.PutUint32(raw[4:], size) + return raw +} + +func TestDecodeErrors(t *testing.T) { + for _, tc := range []struct { + name string + encoded string + err string + }{ + { + name: "bad base64", + encoded: "not-valid-base64!!!", + err: "base64", + }, + { + name: "not deflate", + encoded: base64.StdEncoding.EncodeToString([]byte{0xff, 0xff}), + err: "inflate", + }, + { + name: "short header", + encoded: deflateBase64(t, []byte("DBTB")), + err: "payload too short", + }, + { + name: "bad magic", + encoded: deflateBase64(t, append(header("XXXX", 1), 0x80)), + err: "bad magic", + }, + { + name: "missing bitmap bytes", + encoded: deflateBase64(t, header(magic, 9)), + err: "payload size mismatch", + }, + { + name: "extra bitmap bytes", + encoded: deflateBase64(t, append(header(magic, 1), 0x80, 0x00)), + err: "payload size mismatch", + }, + { + name: "size near 2^32", + encoded: deflateBase64(t, header(magic, 0xFFFFFFFF)), + err: "payload size mismatch", + }, + } { + t.Run(tc.name, func(t *testing.T) { + _, _, err := Decode(tc.encoded) + assert.ErrorContains(t, err, tc.err) + }) + } +} + +func BenchmarkBits(b *testing.B) { + schema := EmbeddedSchema() + root, diags := config.LoadFromBytes("databricks.yml", []byte(` +bundle: + name: bench +resources: + jobs: + j: + name: My Job + tags: {team: data, env: prod} + tasks: + - task_key: a + notebook_task: {notebook_path: /nb} + - task_key: b + new_cluster: {num_workers: 2} +`)) + require.Empty(b, diags) + tm := struct{ SelectUsed bool }{SelectUsed: true} + b.ReportAllocs() + for b.Loop() { + builder := NewBuilder(schema) + builder.Mark(root.Value()) + _, err := builder.Bits(tm) + if err != nil { + b.Fatal(err) + } + } +} + +func TestText(t *testing.T) { + schema := []string{ + "bundle", "bundle.name", + "flag", "flag=true", "flag=false", + "jobs.*.enabled", "jobs.*.enabled=true", "jobs.*.enabled=false", + "ref", "ref=true", "ref=false", + "later", + } + for _, tc := range []struct { + name string + set []string + want []string + }{ + { + name: "nothing set", + set: nil, + want: nil, + }, + { + name: "field", + set: []string{"bundle", "bundle.name"}, + want: []string{"P bundle", "P bundle.name"}, + }, + { + name: "bool true", + set: []string{"flag", "flag=true"}, + want: []string{"PT flag"}, + }, + { + name: "bool false", + set: []string{"flag", "flag=false"}, + want: []string{"P F flag"}, + }, + { + name: "both values under a wildcard", + set: []string{"jobs.*.enabled", "jobs.*.enabled=true", "jobs.*.enabled=false"}, + want: []string{"PTF jobs.*.enabled"}, + }, + { + name: "unresolved reference", + set: []string{"ref"}, + want: []string{"P ref"}, + }, + { + name: "value without presence", + set: []string{"flag=true"}, + want: []string{" T flag"}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + want := append([]string{"size=12 context=0"}, tc.want...) + assert.Equal(t, want, Text(schema, bitsFor(schema, tc.set), ContextFullBundle)) + }) + } + + // A bitmap from an older CLI has fewer bits; the rest of the schema is unset. + assert.Equal(t, []string{"size=1 context=7", "P bundle"}, Text(schema, []bool{true}, 7)) + + // A bitmap from a newer CLI has more bits; a set one past the schema is a placeholder. + newer := append(bitsFor(schema, []string{"bundle"}), false, true) + assert.Equal(t, []string{"size=14 context=0", "P bundle", "P future #13"}, Text(schema, newer, ContextFullBundle)) +} + +func TestToJSON(t *testing.T) { + schema := []string{ + "bundle", "bundle.name", + "jobs", "jobs.*", "jobs.*.tasks", "jobs.*.tasks[*]", + "jobs.*.tasks[*].enabled", "jobs.*.tasks[*].enabled=true", "jobs.*.tasks[*].enabled=false", + "later", + } + job := func(key string, leaf any) map[string]any { + return map[string]any{"jobs": map[string]any{"*": map[string]any{key: leaf}}} + } + enabled := job("tasks[*]", map[string]any{"enabled": true}) + for _, tc := range []struct { + name string + set []string + presence, isTrue, isF map[string]any + }{ + { + name: "nothing set", + set: nil, + presence: map[string]any{}, + isTrue: map[string]any{}, + isF: map[string]any{}, + }, + { + name: "a parent with set children is a subtree", + set: []string{"bundle", "bundle.name"}, + presence: map[string]any{"bundle": map[string]any{"name": true}}, + isTrue: map[string]any{}, + isF: map[string]any{}, + }, + { + name: "a set field without set children is a leaf", + set: []string{"jobs", "jobs.*", "jobs.*.tasks"}, + presence: job("tasks", true), + isTrue: map[string]any{}, + isF: map[string]any{}, + }, + { + name: "a list's elements replace its leaf", + set: []string{"jobs", "jobs.*", "jobs.*.tasks", "jobs.*.tasks[*]"}, + presence: job("tasks[*]", true), + isTrue: map[string]any{}, + isF: map[string]any{}, + }, + { + name: "bool false", + set: []string{"jobs", "jobs.*", "jobs.*.tasks", "jobs.*.tasks[*]", "jobs.*.tasks[*].enabled", "jobs.*.tasks[*].enabled=false"}, + presence: enabled, + isTrue: map[string]any{}, + isF: enabled, + }, + { + name: "both values under a wildcard", + set: []string{"jobs", "jobs.*", "jobs.*.tasks", "jobs.*.tasks[*]", "jobs.*.tasks[*].enabled", "jobs.*.tasks[*].enabled=true", "jobs.*.tasks[*].enabled=false"}, + presence: enabled, + isTrue: enabled, + isF: enabled, + }, + } { + t.Run(tc.name, func(t *testing.T) { + got := ToJSON(schema, bitsFor(schema, tc.set), ContextFullBundle) + assert.Equal(t, JSON{Size: len(schema), Presence: tc.presence, True: tc.isTrue, False: tc.isF}, got) + }) + } + + // A bitmap from an older CLI has fewer bits; the rest of the schema is unset. + got := ToJSON(schema, []bool{true}, 7) + assert.Equal(t, JSON{Size: 1, Context: 7, Presence: map[string]any{"bundle": true}, True: map[string]any{}, False: map[string]any{}}, got) + + // A bitmap from a newer CLI has more bits; a set one past the schema is a placeholder. + got = ToJSON(schema, append(bitsFor(schema, []string{"bundle"}), false, true), ContextFullBundle) + assert.Equal(t, JSON{Size: len(schema) + 2, Presence: map[string]any{"bundle": true, "future #11": true}, True: map[string]any{}, False: map[string]any{}}, got) + + // The list field's leaf is left out whichever of the two comes first. + got = ToJSON([]string{"x[*]", "x"}, []bool{true, true}, ContextFullBundle) + assert.Equal(t, map[string]any{"x[*]": true}, got.Presence) +} + +// TestToJSONIsLossless checks that every entry of the embedded schema maps to +// its own tree node, so the JSON form keeps every set bit. +func TestToJSONIsLossless(t *testing.T) { + schema := EmbeddedSchema() + bits := make([]bool, len(schema)) + for i := range bits { + bits[i] = true + } + got := ToJSON(schema, bits, ContextFullBundle) + + var presence, trueValues, falseValues int + for _, p := range schema { + switch { + case strings.HasSuffix(p, trueSuffix): + trueValues++ + case strings.HasSuffix(p, falseSuffix): + falseValues++ + default: + presence++ + } + } + // "telemetry" namespaces the telemetry fields but is not a schema entry. + assert.Equal(t, presence+1, countNodes(got.Presence)) + // A bool field has no children, so each value entry is a leaf. + assert.Equal(t, trueValues, countLeaves(got.True)) + assert.Equal(t, falseValues, countLeaves(got.False)) +} + +// countNodes counts the schema entries a tree stands for: a "tasks[*]" node +// also stands for "tasks". +func countNodes(tree map[string]any) int { + n := 0 + for k, v := range tree { + n += 1 + strings.Count(k, listSuffix) + if child, ok := v.(map[string]any); ok { + n += countNodes(child) + } + } + return n +} + +func countLeaves(tree map[string]any) int { + n := 0 + for _, v := range tree { + if child, ok := v.(map[string]any); ok { + n += countLeaves(child) + } else { + n++ + } + } + return n +} + +// bitsFor returns bits with the given schema paths set. +func bitsFor(schema, set []string) []bool { + bits := make([]bool, len(schema)) + for _, p := range set { + bits[slices.Index(schema, p)] = true + } + return bits +} diff --git a/bundle/bitmap/bundle_test.go b/bundle/bitmap/bundle_test.go new file mode 100644 index 00000000000..51eb3360b28 --- /dev/null +++ b/bundle/bitmap/bundle_test.go @@ -0,0 +1,171 @@ +package bitmap_test + +import ( + "reflect" + "slices" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/bitmap" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// These tests need bundle.Telemetry, and the bundle package imports this one, +// so they live in an external test package. + +func TestWalkSchema(t *testing.T) { + schema, err := bitmap.WalkSchema(reflect.TypeFor[bundle.Telemetry]()) + require.NoError(t, err) + + seen := map[string]bool{} + for _, p := range schema { + assert.False(t, seen[p], "duplicate path %s", p) + seen[p] = true + assert.NotEqual(t, "targets", p) + assert.NotContains(t, p, "targets.") + assert.NotContains(t, p, "environments.") + assert.NotContains(t, p, "__locations") + } + + assert.Contains(t, schema, "bundle.name") + assert.Contains(t, schema, "resources.jobs.*.name") + assert.Contains(t, schema, "resources.jobs.*.tasks[*].task_key") + assert.Contains(t, schema, "variables.*.default") + // Telemetry fields are namespaced under "telemetry." and, having no json + // tag, keep their Go field name. + assert.Contains(t, schema, "telemetry.SelectUsed") + assert.Contains(t, schema, "telemetry.RunAsSet") + assert.Contains(t, schema, "telemetry.ArtifactFilesIsSet") + assert.NotContains(t, schema, "telemetry.ArtifactFilesIsSet[*]") + + // Each bool field, whatever its Go shape, is followed by its value entries. + for _, p := range []string{ + "experimental.use_legacy_run_as", // bool, omitempty + "resources.jobs.*.queue.enabled", // bool, no omitempty + "presets.pipelines_development", // *bool + "telemetry.SelectUsed", // telemetry bool + "telemetry.RunAsSet", // telemetry *bool + "telemetry.ArtifactFilesIsSet", // telemetry BoolSet + "resources.jobs.*.tasks[*].disable_auto_optimization", // bool in a list element + } { + i := slices.Index(schema, p) + require.GreaterOrEqual(t, i, 0, p) + assert.Equal(t, []string{p + "=true", p + "=false"}, schema[i+1:i+3], p) + } + // Non-bool fields have no value entries. + assert.NotContains(t, schema, "bundle.name=true") + assert.NotContains(t, schema, "variables.*.default=true") +} + +func TestEmbeddedSchemaIsCompleteAndUnique(t *testing.T) { + schema := bitmap.EmbeddedSchema() + fresh, err := bitmap.WalkSchema(reflect.TypeFor[bundle.Telemetry]()) + require.NoError(t, err) + + _, added := bitmap.Merge(schema, fresh) + assert.Empty(t, added, "run 'task generate-bitmap-schema'") + + // A duplicate would leave its first bit permanently 0. + seen := map[string]bool{} + for _, p := range schema { + assert.False(t, seen[p], "duplicate schema entry %q", p) + seen[p] = true + } +} + +func TestBitsTelemetry(t *testing.T) { + schema := bitmap.EmbeddedSchema() + for _, tc := range []struct { + name string + tm bundle.Telemetry + want []string + }{ + // Every field of a zero struct is unmeasured; none may count as present. + { + name: "zero value", + tm: bundle.Telemetry{}, + want: nil, + }, + { + name: "bool false", + tm: bundle.Telemetry{SelectUsed: false}, + want: nil, + }, + { + name: "bool true", + tm: bundle.Telemetry{SelectUsed: true}, + want: []string{"telemetry.SelectUsed", "telemetry.SelectUsed=true"}, + }, + { + name: "*bool nil", + tm: bundle.Telemetry{RunAsSet: nil}, + want: nil, + }, + { + name: "*bool false", + tm: bundle.Telemetry{RunAsSet: new(false)}, + want: []string{"telemetry.RunAsSet", "telemetry.RunAsSet=false"}, + }, + { + name: "*bool true", + tm: bundle.Telemetry{RunAsSet: new(true)}, + want: []string{"telemetry.RunAsSet", "telemetry.RunAsSet=true"}, + }, + { + name: "BoolSet unset", + tm: bundle.Telemetry{ArtifactFilesIsSet: boolSet()}, + want: nil, + }, + { + name: "BoolSet true twice", + tm: bundle.Telemetry{ArtifactFilesIsSet: boolSet(true, true)}, + want: []string{ + "telemetry.ArtifactFilesIsSet", "telemetry.ArtifactFilesIsSet=true", + }, + }, + { + name: "BoolSet both values", + tm: bundle.Telemetry{ArtifactFilesIsSet: boolSet(true, false)}, + want: []string{ + "telemetry.ArtifactFilesIsSet", "telemetry.ArtifactFilesIsSet=false", "telemetry.ArtifactFilesIsSet=true", + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + bits, err := bitmap.NewBuilder(schema).Bits(tc.tm) + require.NoError(t, err) + assert.Equal(t, tc.want, setPaths(schema, bits)) + }) + } +} + +func boolSet(values ...bool) bundle.BoolSet { + var s bundle.BoolSet + for _, v := range values { + s.SetBool(v) + } + return s +} + +func BenchmarkWalkSchema(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + _, err := bitmap.WalkSchema(reflect.TypeFor[bundle.Telemetry]()) + if err != nil { + b.Fatal(err) + } + } +} + +// setPaths returns the schema paths whose bit is set, sorted. +func setPaths(schema []string, bits []bool) []string { + var out []string + for i, p := range schema { + if bits[i] { + out = append(out, p) + } + } + slices.Sort(out) + return out +} diff --git a/bundle/bitmap/embed.go b/bundle/bitmap/embed.go new file mode 100644 index 00000000000..6bcb343f30e --- /dev/null +++ b/bundle/bitmap/embed.go @@ -0,0 +1,9 @@ +package bitmap + +import _ "embed" + +// schemaDeflate is the schema, one field path per line, compressed by +// CompressSchema. Generated by 'task generate-bitmap-schema'. +// +//go:embed schema.deflate +var schemaDeflate []byte diff --git a/bundle/bitmap/genschema/main.go b/bundle/bitmap/genschema/main.go new file mode 100644 index 00000000000..56d98bb1f64 --- /dev/null +++ b/bundle/bitmap/genschema/main.go @@ -0,0 +1,40 @@ +// Command genschema compresses the bitmap schema text in INPUT (as printed by +// 'databricks bundle bitmap update-schema') and writes the same bytes to each +// OUTPUT: the copy the CLI embeds (bundle/bitmap/schema.deflate) and the one the +// databricks-bundles decoder reads (python/databricks/bundles/_bitmap). +package main + +import ( + "fmt" + "os" + + "github.com/databricks/cli/bundle/bitmap" +) + +func main() { + if len(os.Args) < 3 { + fmt.Fprintln(os.Stderr, "usage: genschema INPUT OUTPUT...") + os.Exit(2) + } + if err := generate(os.Args[1], os.Args[2:]); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func generate(input string, outputs []string) error { + text, err := os.ReadFile(input) + if err != nil { + return err + } + compressed, err := bitmap.CompressSchema(text) + if err != nil { + return err + } + for _, path := range outputs { + if err := os.WriteFile(path, compressed, 0o644); err != nil { + return err + } + } + return nil +} diff --git a/bundle/bitmap/schema.deflate b/bundle/bitmap/schema.deflate new file mode 100644 index 00000000000..21b84b572f3 Binary files /dev/null and b/bundle/bitmap/schema.deflate differ diff --git a/bundle/bundle.go b/bundle/bundle.go index 9f686a7e466..afa7289e93d 100644 --- a/bundle/bundle.go +++ b/bundle/bundle.go @@ -15,6 +15,7 @@ import ( "time" "uuid" + "github.com/databricks/cli/bundle/bitmap" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/engine" "github.com/databricks/cli/bundle/direct" @@ -67,9 +68,11 @@ const terraformStateFilename = "terraform.tfstate" // This struct is used as a communication channel to collect metrics // from all over the bundle codebase to finally be emitted as telemetry. type Metrics struct { - ConfigurationFileCount int64 - TargetCount int64 - DeploymentId uuid.UUID + ConfigurationFileCount int64 + TargetCount int64 + DeploymentId uuid.UUID + // BoolValues holds bool keys that are not fields of Telemetry: the dynamic + // has_ keys and the local.cache.* keys written by libs/cache. BoolValues []protos.BoolMapEntry ExecutionTimes []protos.IntMapEntry LocalCacheMeasurementsMs []protos.IntMapEntry // Local cache measurements stored as milliseconds @@ -109,10 +112,6 @@ func (m *Metrics) SetBoolValue(key string, value bool) { m.BoolValues = append(m.BoolValues, protos.BoolMapEntry{Key: key, Value: value}) } -func (m *Metrics) AddBoolValue(key string, value bool) { - m.BoolValues = append(m.BoolValues, protos.BoolMapEntry{Key: key, Value: value}) -} - // AddDurationValue sets the value of a duration metric in milliseconds. // The value is added to the list of measurements. func (m *Metrics) AddDurationValue(key string, value time.Duration) { @@ -230,6 +229,17 @@ type Bundle struct { Cache *cache.Cache Metrics Metrics + + // Telemetry holds the boolean deploy telemetry flags; they feed both the + // deploy event's bool_values and the bundle bitmap. + Telemetry Telemetry + + // Bitmap holds the config bits of the bundle bitmap: what users configure, + // not what the CLI derives. Loading with a target creates it from the files + // with the target merged in, before flags or Initialize set any field; the + // Python mutator adds the resources Python defines, and the end of Initialize + // adds the values of fields set through ${var.*} references. + Bitmap *bitmap.Builder } func Load(ctx context.Context, path string) (*Bundle, error) { diff --git a/bundle/config/mutator/apply_sync_mode.go b/bundle/config/mutator/apply_sync_mode.go index 57b564920cd..ad13ecc820e 100644 --- a/bundle/config/mutator/apply_sync_mode.go +++ b/bundle/config/mutator/apply_sync_mode.go @@ -75,7 +75,7 @@ func (m *applySyncMode) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagno return diags } - b.Metrics.AddBoolValue("source_linked_set_for_non_development", b.Config.Bundle.Mode != config.Development) + b.Telemetry.SourceLinkedSetForNonDevelopment = new(b.Config.Bundle.Mode != config.Development) if b.Config.Bundle.Mode != config.Development { diags = diags.Append( diff --git a/bundle/config/mutator/collect_escape_telemetry.go b/bundle/config/mutator/collect_escape_telemetry.go index b24922bb5b3..f8d49e60329 100644 --- a/bundle/config/mutator/collect_escape_telemetry.go +++ b/bundle/config/mutator/collect_escape_telemetry.go @@ -50,16 +50,16 @@ func (*collectEscapeTelemetry) Apply(ctx context.Context, b *bundle.Bundle) diag } if hasDoubleDollarBrace { - b.Metrics.SetBoolValue("config_has_double_dollar_brace", true) + b.Telemetry.ConfigHasDoubleDollarBrace = true } if hasDoubleDollar { - b.Metrics.SetBoolValue("config_has_double_dollar", true) + b.Telemetry.ConfigHasDoubleDollar = true } if hasBackslashDollarBrace { - b.Metrics.SetBoolValue("config_has_backslash_dollar_brace", true) + b.Telemetry.ConfigHasBackslashDollarBrace = true } if hasBackslashDollar { - b.Metrics.SetBoolValue("config_has_backslash_dollar", true) + b.Telemetry.ConfigHasBackslashDollar = true } return nil diff --git a/bundle/config/mutator/collect_escape_telemetry_test.go b/bundle/config/mutator/collect_escape_telemetry_test.go index 90bc259704e..533ac39940a 100644 --- a/bundle/config/mutator/collect_escape_telemetry_test.go +++ b/bundle/config/mutator/collect_escape_telemetry_test.go @@ -93,7 +93,7 @@ func TestCollectEscapeTelemetry(t *testing.T) { diags := bundle.Apply(t.Context(), b, mutator.CollectEscapeTelemetry()) require.Empty(t, diags) - assert.Equal(t, tt.expected, b.Metrics.BoolValues) + assert.Equal(t, tt.expected, b.Telemetry.BoolValues()) }) } } diff --git a/bundle/config/mutator/mark_select_used.go b/bundle/config/mutator/mark_select_used.go index 168eb59f396..cc1cf0fe353 100644 --- a/bundle/config/mutator/mark_select_used.go +++ b/bundle/config/mutator/mark_select_used.go @@ -4,7 +4,6 @@ import ( "context" "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/metrics" "github.com/databricks/cli/libs/diag" ) @@ -26,7 +25,7 @@ func (m *markSelectUsed) Name() string { func (m *markSelectUsed) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { if len(b.Select) > 0 { - b.Metrics.SetBoolValue(metrics.SelectUsed, true) + b.Telemetry.SelectUsed = true } return nil } diff --git a/bundle/config/mutator/mark_select_used_test.go b/bundle/config/mutator/mark_select_used_test.go index b2da87c681d..361a7450a5d 100644 --- a/bundle/config/mutator/mark_select_used_test.go +++ b/bundle/config/mutator/mark_select_used_test.go @@ -7,8 +7,6 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/cli/bundle/metrics" - "github.com/databricks/cli/libs/telemetry/protos" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -27,7 +25,7 @@ func TestMarkSelectUsed_Set(t *testing.T) { b.Select = []string{"my_job"} diags := bundle.Apply(t.Context(), b, mutator.MarkSelectUsed()) require.NoError(t, diags.Error()) - assert.Contains(t, b.Metrics.BoolValues, protos.BoolMapEntry{Key: metrics.SelectUsed, Value: true}) + assert.True(t, b.Telemetry.SelectUsed) } // TestMarkSelectUsed_Unset checks that the flag is left unset when --select is not @@ -36,5 +34,5 @@ func TestMarkSelectUsed_Unset(t *testing.T) { b := &bundle.Bundle{} diags := bundle.Apply(t.Context(), b, mutator.MarkSelectUsed()) require.NoError(t, diags.Error()) - assert.Empty(t, b.Metrics.BoolValues) + assert.False(t, b.Telemetry.SelectUsed) } diff --git a/bundle/config/mutator/python/python_mutator.go b/bundle/config/mutator/python/python_mutator.go index 757ab50e0fb..b74cd90d5af 100644 --- a/bundle/config/mutator/python/python_mutator.go +++ b/bundle/config/mutator/python/python_mutator.go @@ -266,6 +266,7 @@ func (m *pythonMutator) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagno // mutateDiags is used because Mutate returns 'error' instead of 'diag.Diagnostics' var mutateDiags diag.Diagnostics var result applyPythonOutputResult + var before, after dyn.Value mutateDiagsHasError := errors.New("unexpected error") err = b.Config.Mutate(func(leftRoot dyn.Value) (dyn.Value, error) { @@ -297,6 +298,7 @@ func (m *pythonMutator) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagno if err != nil { return dyn.InvalidValue, fmt.Errorf("internal error when merging output of Python mutator: %w", err) } + before, after = leftRoot, newRoot for _, resourceKey := range result.AddedResources.ToArray() { log.Debugf(ctx, "added resource at 'resources.%s.%s'", resourceKey.Type, resourceKey.Name) @@ -351,6 +353,12 @@ func (m *pythonMutator) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagno return mutateDiags } + // Record what Python wrote, before the defaults of added resources are filled in: + // the resources it added, and the fields it set or changed on existing ones. + // Comparing with the config Python received leaves out the defaults those + // already had. + b.Bitmap.MarkChanges(before, after) + resourcemutator.NormalizeAndInitializeResources(ctx, b, result.AddedResources) if logdiag.HasError(ctx) { return mutateDiags diff --git a/bundle/config/mutator/python/python_mutator_test.go b/bundle/config/mutator/python/python_mutator_test.go index 8e1b3369897..d4b086c71a3 100644 --- a/bundle/config/mutator/python/python_mutator_test.go +++ b/bundle/config/mutator/python/python_mutator_test.go @@ -20,6 +20,7 @@ import ( "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/bitmap" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/internal/testutil" "github.com/databricks/cli/libs/cmdio" @@ -121,8 +122,15 @@ workspace: { current_user: { userName: test }}`) assert.Equal(t, "pipeline_0", pipeline0.Name) } + // The added resources are recorded in the bitmap; pipelines exist only in Python. + bits, err := b.Bitmap.Bits(struct{}{}) + require.NoError(t, err) + text := bitmap.Text(bitmap.EmbeddedSchema(), bits, bitmap.ContextFullBundle) + assert.Contains(t, text, "P resources.pipelines") + assert.Contains(t, text, "P resources.pipelines.*.name") + // output of locations.json should be applied to underlying dyn.Value - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { + err = b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { // location is databricks.yml, because output contains resource as-is jobName0, err := dyn.GetByPath(v, dyn.MustPathFromString("resources.jobs.job0.name")) require.NoError(t, err) @@ -235,6 +243,11 @@ resources: return v, nil }) assert.NoError(t, err) + + // The field the mutator set on a YAML resource is recorded in the bitmap. + bits, err := b.Bitmap.Bits(struct{}{}) + require.NoError(t, err) + assert.Contains(t, bitmap.Text(bitmap.EmbeddedSchema(), bits, bitmap.ContextFullBundle), "P resources.jobs.*.description") } func TestPythonConfigSection(t *testing.T) { @@ -601,9 +614,13 @@ func loadYaml(name, content string) *bundle.Bundle { panic(diag.Error()) } - return &bundle.Bundle{ + // Initialize records the loaded config in the bitmap before the Python mutator runs. + b := &bundle.Bundle{ Config: *v, + Bitmap: bitmap.NewBuilder(bitmap.EmbeddedSchema()), } + b.Bitmap.Mark(v.Value()) + return b } func withFakeVEnv(t *testing.T, venvPath string) { diff --git a/bundle/config/mutator/resolve_variable_references.go b/bundle/config/mutator/resolve_variable_references.go index b8245569981..816baae41de 100644 --- a/bundle/config/mutator/resolve_variable_references.go +++ b/bundle/config/mutator/resolve_variable_references.go @@ -205,7 +205,7 @@ func (m *resolveVariableReferences) Apply(ctx context.Context, b *bundle.Bundle) } if m.artifactsReferenceUsed { - b.Metrics.SetBoolValue("artifacts_reference_used", true) + b.Telemetry.ArtifactsReferenceUsed = true } return diags diff --git a/bundle/config/mutator/resourcemutator/apply_presets.go b/bundle/config/mutator/resourcemutator/apply_presets.go index d808daf82fb..e589bc3a17f 100644 --- a/bundle/config/mutator/resourcemutator/apply_presets.go +++ b/bundle/config/mutator/resourcemutator/apply_presets.go @@ -9,7 +9,6 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/bundle/metrics" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/textutil" "github.com/databricks/databricks-sdk-go/service/catalog" @@ -47,7 +46,7 @@ func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos t := b.Config.Presets prefix := t.NamePrefix - b.Metrics.AddBoolValue(metrics.PresetsNamePrefixIsSet, prefix != "") + b.Telemetry.PresetsNamePrefixIsSet = new(prefix != "") tags := toTagArray(t.Tags) diff --git a/bundle/config/mutator/resourcemutator/run_as.go b/bundle/config/mutator/resourcemutator/run_as.go index 0a6943cf3f1..e4d8dcb7587 100644 --- a/bundle/config/mutator/resourcemutator/run_as.go +++ b/bundle/config/mutator/resourcemutator/run_as.go @@ -227,10 +227,10 @@ func setPipelineOwnersToRunAsIdentity(b *bundle.Bundle) { func (m *setRunAs) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { // Track the use of the legacy run_as mode. - b.Metrics.AddBoolValue("experimental.use_legacy_run_as", b.Config.Experimental != nil && config.IsExplicitlyEnabled(b.Config.Experimental.UseLegacyRunAs)) + b.Telemetry.UseLegacyRunAs = new(b.Config.Experimental != nil && config.IsExplicitlyEnabled(b.Config.Experimental.UseLegacyRunAs)) // Track whether top level run_as is set. - b.Metrics.AddBoolValue("run_as_set", b.Config.Value().Get("run_as").Kind() != dyn.KindInvalid) + b.Telemetry.RunAsSet = new(b.Config.Value().Get("run_as").Kind() != dyn.KindInvalid) // Mutator is a no-op if run_as is not specified in the bundle if b.Config.Value().Get("run_as").Kind() == dyn.KindInvalid { diff --git a/bundle/config/validate/tf_only_references.go b/bundle/config/validate/tf_only_references.go index 3603b6f4a7f..a9e27e1d69c 100644 --- a/bundle/config/validate/tf_only_references.go +++ b/bundle/config/validate/tf_only_references.go @@ -47,7 +47,7 @@ func (m *tfOnlyReferences) Apply(_ context.Context, b *bundle.Bundle) diag.Diagn }) if len(diags) > 0 { - b.Metrics.AddBoolValue("has_tf_only_references", true) + b.Telemetry.HasTfOnlyReferences = true } return diags diff --git a/bundle/metrics/metrics.go b/bundle/metrics/metrics.go index f5fcb82fb1e..efa62ffe820 100644 --- a/bundle/metrics/metrics.go +++ b/bundle/metrics/metrics.go @@ -1,127 +1,11 @@ package metrics -const ( - ExperimentalPythonWheelWrapperIsSet = "python_wheel_wrapper_is_set" - ArtifactDynamicVersionIsSet = "artifact_dynamic_version_is_set" - ArtifactBuildCommandIsSet = "artifact_build_command_is_set" - ArtifactFilesIsSet = "artifact_files_is_set" - PresetsNamePrefixIsSet = "presets_name_prefix_is_set" - AppLifecycleStarted = "app_lifecycle_started" - ClusterLifecycleStarted = "cluster_lifecycle_started" - DeploymentHistoryEnabled = "deployment_history_enabled" - SqlWarehouseLifecycleStarted = "sql_warehouse_lifecycle_started" - SelectUsed = "select_used" - - // Whether the deploy registered an immutable snapshot. Derived from the snapshot - // resource, not from experimental.immutable_folder: the flag may change, and the - // resource is what the deploy actually uploads. - ImmutableSnapshotUsed = "immutable_snapshot_used" - - // Key under which the snapshot upload duration is reported in - // bundle_mutator_execution_time_ms, next to files.(upload) so the two can be - // compared. Covers creating the snapshot resource (the upload plus recording it in - // state), not building the zip, which is snapshot.(snapshotUpload). Absent when no - // snapshot was uploaded. It lacks the parentheses of a mutator name because it is - // not a mutator. - SnapshotUploadDuration = "snapshot.upload" - - // Outcome of an automatic migration of terraform state to the direct engine. - // - migrate_error: state conversion itself errored. - // - migrate_commit_error: the state was converted, but committing it - // (renaming files / pushing to workspace) failed. - // - migrate_warnings: the conversion emitted warnings (e.g. resources the direct engine can't represent). - DirectMigrateError = "direct_migrate_error" - DirectMigrateCommitError = "direct_migrate_commit_error" - DirectMigrateWarnings = "direct_migrate_warnings" - // True when the post-convert plan check failed; the migration was not committed. - DirectMigratePlanError = "direct_migrate_plan_error" - // True when the migrated state's first plan would recreate (destroy + create) an - // existing resource. The migration still commits; the recreate is a genuine pending - // change gated by the deploy's --auto-approve, and this only records how often it happens. - DirectMigrateRecreatePlanned = "direct_migrate_recreate_planned" - - // Recorded when an automatic post-deploy migration to the direct engine - // actually ran (state was rewritten). Exactly one of the three keys is true; - // all are absent when auto-migration did not run. If both config and env - // set direct, bundle.engine wins per ValidateEngineSetting, so via_config - // covers the "durable opt-in" population and via_env covers the - // "env-var only" population. - // - via_config: bundle.engine = "direct" was set in the bundle config. - // - via_env: only DATABRICKS_BUNDLE_ENGINE=direct was set. - // - via_default: neither was set, so the migration came from the default. - // This is the population that did not ask for anything. - DirectAutoMigrateViaConfig = "direct_migrated_via_config" - DirectAutoMigrateViaEnv = "direct_migrated_via_env" - DirectAutoMigrateViaDefault = "direct_migrated_via_default" - - // Whether the (deprecated) terraform engine was explicitly opted into, and via - // which source. Independent: both can be present when config and env both - // request terraform. Emitted only when true; absence means "not opted in via - // this source", so the population still on terraform can be sliced by source. - // - engine_terraform_config: bundle.engine = "terraform" in the bundle config. - // - engine_terraform_env: DATABRICKS_BUNDLE_ENGINE=terraform in the environment. - EngineTerraformConfig = "engine_terraform_config" - EngineTerraformEnv = "engine_terraform_env" - - // Whether workspace.state_path is under /Workspace/Shared. - StatePathIsShared = "state_path_is_shared" - - // Whether this deploy is compatible with an automatic migration of the deployment - // state to a dedicated state storage service (DMS). Deploying a bundle requires - // write access (CAN_EDIT or higher) to the state folder; after migration that is - // governed by the permissions on the deployment object instead. - // - // When the bundle has no permissions section, the migration can mirror the state - // folder's ACLs onto the deployment (CAN_EDIT -> CAN_EDIT, CAN_MANAGE -> - // CAN_MANAGE), preserving everyone's access wherever the state lives. When a - // permissions section is set, the migration applies exactly those permissions, so - // anyone with write access to the state folder who is not declared with - // CAN_MANAGE would lose the ability to deploy. - // - // Exactly one of the three keys below is recorded per deploy: - // - auto: no permissions section (folder ACLs are mirrored), or every principal - // with write access to the state folder is declared. - // - only_self_undeclared: a permissions section is set and the only principal - // with undeclared write access is the deploying user. The migration - // grants the deploying user CAN_MANAGE on the deployment object, so this - // is auto-migratable if we choose to preserve that grant on future - // deploys. Recorded separately to measure how common this case is. - // - not: a permissions section is set and the state folder has undeclared write - // access from a principal other than the deploying user. - DMSCompatAuto = "dms_compat_auto" - DMSCompatOnlySelfUndeclared = "dms_compat_only_self_undeclared" - DMSCompatNot = "dms_compat_not" - - // Breakdown dimensions recorded on every deploy alongside the verdict above, so the - // DMS auto-migration population can be sliced without inferring it from the verdict. - // Each is an independent boolean. - - // Whether a top-level permissions section is set. The no-permissions case is always - // auto-migration compatible (folder ACLs are mirrored), so this separates the two - // populations that both land on dms_compat_auto. - PermissionsSectionSet = "permissions_section_set" - - // Where the deployment state folder lives. Exactly one of StatePathIsShared, - // StatePathInDeployerHome, StatePathInOtherUserHome, and StatePathOther is true per - // deploy. StatePathOther is any other /Workspace folder (not a user home or shared). - StatePathInDeployerHome = "state_path_in_deployer_home" - StatePathInOtherUserHome = "state_path_in_other_user_home" - StatePathOther = "state_path_other" - - // Which principal types have undeclared write access to the state folder — the - // access an auto-migration governed by the permissions section would drop. These can - // co-occur; all false when the deploy is auto-migration compatible. - DMSUndeclaredDeployingUser = "dms_undeclared_deploying_user" - DMSUndeclaredOtherUser = "dms_undeclared_other_user" - DMSUndeclaredServicePrincipal = "dms_undeclared_service_principal" - DMSUndeclaredGroup = "dms_undeclared_group" - - // Task-type usage is recorded generically at deploy as one "has_" key - // per task type present in the bundle (e.g. has_notebook_task, has_ai_runtime_task); - // see collectTaskTypes in bundle/phases. The two keys below are extra - // ai_runtime_task-specific dimensions, emitted only when the bundle declares an - // ai_runtime_task, so a false value means "has an ai_runtime_task, but it is not - // scheduled / not multi-task". GPU type and count are intentionally not recorded here. - AiRuntimeTaskScheduled = "ai_runtime_task_scheduled" - AiRuntimeTaskMultitask = "ai_runtime_task_multitask" -) +// Boolean telemetry keys are declared as `metric` tags on bundle.Telemetry. + +// Key under which the snapshot upload duration is reported in +// bundle_mutator_execution_time_ms, next to files.(upload) so the two can be +// compared. Covers creating the snapshot resource (the upload plus recording it in +// state), not building the zip, which is snapshot.(snapshotUpload). Absent when no +// snapshot was uploaded. It lacks the parentheses of a mutator name because it is +// not a mutator. +const SnapshotUploadDuration = "snapshot.upload" diff --git a/bundle/metrics/track_used_compute.go b/bundle/metrics/track_used_compute.go index 1f3bf051dc0..d62aa68e3db 100644 --- a/bundle/metrics/track_used_compute.go +++ b/bundle/metrics/track_used_compute.go @@ -47,9 +47,9 @@ func (c *trackUsedCompute) Apply(ctx context.Context, b *bundle.Bundle) diag.Dia } } - b.Metrics.AddBoolValue("has_serverless_compute", hasServerlessCompute) - b.Metrics.AddBoolValue("has_classic_job_compute", hasClassicJobCompute) - b.Metrics.AddBoolValue("has_classic_interactive_compute", hasClassicInteractiveCompute) + b.Telemetry.HasServerlessCompute = new(hasServerlessCompute) + b.Telemetry.HasClassicJobCompute = new(hasClassicJobCompute) + b.Telemetry.HasClassicInteractiveCompute = new(hasClassicInteractiveCompute) return nil } diff --git a/bundle/permissions/workspace_root.go b/bundle/permissions/workspace_root.go index d4f3ee173c7..c9a6b0b4146 100644 --- a/bundle/permissions/workspace_root.go +++ b/bundle/permissions/workspace_root.go @@ -11,7 +11,6 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/libraries" - "github.com/databricks/cli/bundle/metrics" "github.com/databricks/cli/bundle/paths" "github.com/databricks/cli/libs/diag" "github.com/databricks/databricks-sdk-go/service/workspace" @@ -163,16 +162,16 @@ func recordPermissionMetrics(b *bundle.Bundle, stateFolderPerms *WorkspacePathPe isShared := libraries.IsWorkspaceSharedPath(statePath) owner, underUserHome := userHomeOwner(statePath) - b.Metrics.SetBoolValue(metrics.StatePathIsShared, isShared) - b.Metrics.SetBoolValue(metrics.PermissionsSectionSet, len(b.Config.Permissions) > 0) + b.Telemetry.StatePathIsShared = new(isShared) + b.Telemetry.PermissionsSectionSet = new(len(b.Config.Permissions) > 0) // userHomeOwner yields a non-empty owner whenever underUserHome is true, so the two // home flags are exact complements: an unresolved deployer ("") never equals the // owner and falls into the other-user bucket. StatePathOther covers any /Workspace // folder that is neither shared nor a user home. - b.Metrics.SetBoolValue(metrics.StatePathInDeployerHome, underUserHome && owner == deployer) - b.Metrics.SetBoolValue(metrics.StatePathInOtherUserHome, underUserHome && owner != deployer) - b.Metrics.SetBoolValue(metrics.StatePathOther, !isShared && !underUserHome) + b.Telemetry.StatePathInDeployerHome = new(underUserHome && owner == deployer) + b.Telemetry.StatePathInOtherUserHome = new(underUserHome && owner != deployer) + b.Telemetry.StatePathOther = new(!isShared && !underUserHome) // stateFolderPerms is nil when no permissions are declared, in which case there are // no undeclared writers (the migration mirrors the folder's ACLs). @@ -181,25 +180,25 @@ func recordPermissionMetrics(b *bundle.Bundle, stateFolderPerms *WorkspacePathPe undeclared = stateFolderPerms.UndeclaredWriters(b.Config.Permissions) } self, otherUser, servicePrincipal, group := undeclaredWriterTypes(undeclared, deployer) - b.Metrics.SetBoolValue(metrics.DMSUndeclaredDeployingUser, self) - b.Metrics.SetBoolValue(metrics.DMSUndeclaredOtherUser, otherUser) - b.Metrics.SetBoolValue(metrics.DMSUndeclaredServicePrincipal, servicePrincipal) - b.Metrics.SetBoolValue(metrics.DMSUndeclaredGroup, group) + b.Telemetry.DMSUndeclaredDeployingUser = new(self) + b.Telemetry.DMSUndeclaredOtherUser = new(otherUser) + b.Telemetry.DMSUndeclaredServicePrincipal = new(servicePrincipal) + b.Telemetry.DMSUndeclaredGroup = new(group) - // Emit exactly one of the auto-migration verdict keys. - b.Metrics.SetBoolValue(autoMigrationVerdict(b, stateFolderPerms, undeclared), true) + // Set exactly one of the auto-migration verdict flags. + *autoMigrationVerdict(b, stateFolderPerms, undeclared) = true } -// autoMigrationVerdict returns the metric key describing whether this deploy is +// autoMigrationVerdict returns the telemetry flag describing whether this deploy is // compatible with an automatic migration of the deployment state to a dedicated // state storage service. undeclared is the state folder's undeclared writers (empty -// when no permissions are declared). See metrics.DMSCompatAuto. -func autoMigrationVerdict(b *bundle.Bundle, stateFolderPerms *WorkspacePathPermissions, undeclared []resources.Permission) string { +// when no permissions are declared). See bundle.Telemetry.DMSCompatAuto. +func autoMigrationVerdict(b *bundle.Bundle, stateFolderPerms *WorkspacePathPermissions, undeclared []resources.Permission) *bool { // No permissions section: the migration mirrors the state folder's ACLs onto the // deployment (CAN_EDIT -> CAN_EDIT, CAN_MANAGE -> CAN_MANAGE), preserving // everyone's access wherever the state lives. if len(b.Config.Permissions) == 0 { - return metrics.DMSCompatAuto + return &b.Telemetry.DMSCompatAuto } // A permissions section is set. The state folder is always one of the synced bundle @@ -207,7 +206,7 @@ func autoMigrationVerdict(b *bundle.Bundle, stateFolderPerms *WorkspacePathPermi // stateFolderPerms is non-nil here. Guard against nil regardless so a telemetry // computation can never panic the deploy. if stateFolderPerms == nil { - return metrics.DMSCompatNot + return &b.Telemetry.DMSCompatNot } // The migration applies exactly the declared permissions to the deployment, so @@ -215,14 +214,14 @@ func autoMigrationVerdict(b *bundle.Bundle, stateFolderPerms *WorkspacePathPermi // ability to deploy. switch { case len(undeclared) == 0: - return metrics.DMSCompatAuto + return &b.Telemetry.DMSCompatAuto case len(undeclared) == 1 && isDeployingUser(b, undeclared[0]): // The deploying user is the only undeclared writer. The migration grants the // deploying user CAN_MANAGE on the deployment object, so this deploy is // auto-migratable if we choose to preserve that grant on future deploys. - return metrics.DMSCompatOnlySelfUndeclared + return &b.Telemetry.DMSCompatOnlySelfUndeclared default: - return metrics.DMSCompatNot + return &b.Telemetry.DMSCompatNot } } diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index 8c5a68d3130..660175ff307 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -279,4 +279,8 @@ func Initialize(ctx context.Context, b *bundle.Bundle) { // Executes the post_init script hook defined in the bundle configuration scripts.Execute(config.ScriptPostInit), ) + + // Variables have their final values now; record them for the fields that + // were set through a ${var.*} reference. + b.Bitmap.ResolveVariables(b.Config.Value()) } diff --git a/bundle/phases/load.go b/bundle/phases/load.go index 6fa1106b3bf..cf5baa6a5a2 100644 --- a/bundle/phases/load.go +++ b/bundle/phases/load.go @@ -4,6 +4,7 @@ import ( "context" "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/bitmap" "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/logdiag" @@ -26,6 +27,7 @@ func LoadDefaultTarget(ctx context.Context, b *bundle.Bundle) { } bundle.ApplyContext(ctx, b, mutator.SelectDefaultTarget()) + recordConfig(b) } func LoadNamedTarget(ctx context.Context, b *bundle.Bundle, target string) { @@ -37,4 +39,13 @@ func LoadNamedTarget(ctx context.Context, b *bundle.Bundle, target string) { } bundle.ApplyContext(ctx, b, mutator.SelectTarget(target)) + recordConfig(b) +} + +// recordConfig starts the bundle bitmap from the config as the user wrote it: the +// files with the selected target merged in. It runs before command-line flags +// (--var, --profile, --force-lock, ...) and the initialize phase set any field. +func recordConfig(b *bundle.Bundle) { + b.Bitmap = bitmap.NewBuilder(bitmap.EmbeddedSchema()) + b.Bitmap.Mark(b.Config.Value()) } diff --git a/bundle/phases/telemetry.go b/bundle/phases/telemetry.go index 723bcd9346f..276efe42b91 100644 --- a/bundle/phases/telemetry.go +++ b/bundle/phases/telemetry.go @@ -11,6 +11,7 @@ import ( "time" "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/bitmap" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/engine" "github.com/databricks/cli/bundle/config/resources" @@ -159,16 +160,16 @@ func collectTaskTypes(jobs map[string]*resources.Job) []string { return keys } -// aiRuntimeTaskMetrics computes the ai_runtime_task-specific deploy dimensions. -// present is true when any job declares an ai_runtime_task (including one nested in -// a for_each_task); scheduled and multitask describe those jobs and are meaningful -// only when present is true. Task-type presence itself (has_ai_runtime_task and the -// other has_*_task keys) is recorded generically by collectTaskTypes. +// aiRuntimeTaskMetrics records the ai_runtime_task-specific deploy dimensions: whether +// each job that declares an ai_runtime_task (including one nested in a for_each_task) +// is scheduled and has more than one task. Task-type presence itself +// (has_ai_runtime_task and the other has_*_task keys) is recorded generically by +// collectTaskTypes. // // code_source_path is deliberately not inspected: it is rewritten to its uploaded // remote path before this runs, so it would always read as remote. GPU type and // count are likewise not recorded here. -func aiRuntimeTaskMetrics(jobs map[string]*resources.Job) (present, scheduled, multitask bool) { +func aiRuntimeTaskMetrics(jobs map[string]*resources.Job, t *bundle.Telemetry) { for _, job := range jobs { if job == nil { continue @@ -187,15 +188,9 @@ func aiRuntimeTaskMetrics(jobs map[string]*resources.Job) (present, scheduled, m if !jobHasAiRuntimeTask { continue } - present = true - if job.Schedule != nil || job.Trigger != nil || job.Continuous != nil { - scheduled = true - } - if len(job.Tasks) > 1 { - multitask = true - } + t.AiRuntimeTaskScheduled.SetBool(job.Schedule != nil || job.Trigger != nil || job.Continuous != nil) + t.AiRuntimeTaskMultitask.SetBool(len(job.Tasks) > 1) } - return present, scheduled, multitask } // pydabsMetrics reports usage of Python support. Loaders and mutators are read from @@ -257,8 +252,8 @@ func fromWorkspaceFolder(ctx context.Context, b *bundle.Bundle) bool { // LogDeployTelemetry logs a telemetry event for a bundle deploy command. func LogDeployTelemetry(ctx context.Context, b *bundle.Bundle, errMsg string) { errMsg = telemetry.ScrubErrorMessage(errMsg) - b.Metrics.SetBoolValue(metrics.DeploymentHistoryEnabled, b.DeploymentBundle.StateDB.RecordsHistory()) - b.Metrics.SetBoolValue(metrics.ImmutableSnapshotUsed, len(b.Config.Resources.Snapshots) > 0) + b.Telemetry.DeploymentHistoryEnabled = new(b.DeploymentBundle.StateDB.RecordsHistory()) + b.Telemetry.ImmutableSnapshotUsed = new(len(b.Config.Resources.Snapshots) > 0) if d, ok := b.DeploymentBundle.DeployDurations.Load(resources.SnapshotKey); ok { b.Metrics.ExecutionTimes = append(b.Metrics.ExecutionTimes, protos.IntMapEntry{ Key: metrics.SnapshotUploadDuration, @@ -337,34 +332,30 @@ func LogDeployTelemetry(ctx context.Context, b *bundle.Bundle, errMsg string) { for _, app := range b.Config.Resources.Apps { if app != nil && app.Lifecycle != nil && app.Lifecycle.Started != nil { - b.Metrics.SetBoolValue(metrics.AppLifecycleStarted, *app.Lifecycle.Started) - break + b.Telemetry.AppLifecycleStarted.SetBool(*app.Lifecycle.Started) } } for _, cluster := range b.Config.Resources.Clusters { if cluster != nil && cluster.Lifecycle != nil && cluster.Lifecycle.Started != nil { - b.Metrics.SetBoolValue(metrics.ClusterLifecycleStarted, *cluster.Lifecycle.Started) - break + b.Telemetry.ClusterLifecycleStarted.SetBool(*cluster.Lifecycle.Started) } } for _, warehouse := range b.Config.Resources.SqlWarehouses { if warehouse != nil && warehouse.Lifecycle != nil && warehouse.Lifecycle.Started != nil { - b.Metrics.SetBoolValue(metrics.SqlWarehouseLifecycleStarted, *warehouse.Lifecycle.Started) - break + b.Telemetry.SqlWarehouseLifecycleStarted.SetBool(*warehouse.Lifecycle.Started) } } // Record which task types the bundle uses (has_ per type present), // plus the ai_runtime_task-specific scheduling / multi-task dimensions. + // The has_ keys are dynamic, so they bypass b.Telemetry; the bitmap + // records each task type as a config field (resources.jobs.*.tasks[*]._task). for _, key := range collectTaskTypes(b.Config.Resources.Jobs) { b.Metrics.SetBoolValue(key, true) } - if airPresent, airScheduled, airMultitask := aiRuntimeTaskMetrics(b.Config.Resources.Jobs); airPresent { - b.Metrics.SetBoolValue(metrics.AiRuntimeTaskScheduled, airScheduled) - b.Metrics.SetBoolValue(metrics.AiRuntimeTaskMultitask, airMultitask) - } + aiRuntimeTaskMetrics(b.Config.Resources.Jobs, &b.Telemetry) // Record whether the removed terraform engine was still requested (the command // then fails with the removal error), separately per source. Only emitted when @@ -372,10 +363,10 @@ func LogDeployTelemetry(ctx context.Context, b *bundle.Bundle, errMsg string) { // already failed the command upstream via ValidateEngineSetting, so treating the // FromEnv error as "not terraform" here cannot mask a real terraform request. if b.Config.Bundle.Engine == engine.EngineTerraform { - b.Metrics.SetBoolValue(metrics.EngineTerraformConfig, true) + b.Telemetry.EngineTerraformConfig = true } if envEngine, _ := engine.FromEnv(ctx); envEngine == engine.EngineTerraform { - b.Metrics.SetBoolValue(metrics.EngineTerraformEnv, true) + b.Telemetry.EngineTerraformEnv = true } // If the bundle UUID is not set, we use a default 0 value. @@ -475,6 +466,7 @@ func LogDeployTelemetry(ctx context.Context, b *bundle.Bundle, errMsg string) { Git: bundleGitInfo(b), FromWorkspaceFolder: fromWorkspaceFolder(ctx, b), InGitRepo: b.InGitRepo, + BundleBitmap: bundleBitmap(ctx, b), Experimental: &protos.BundleDeployExperimental{ BundleMode: mode, @@ -483,7 +475,7 @@ func LogDeployTelemetry(ctx context.Context, b *bundle.Bundle, errMsg string) { UploadFileSizes: uploadFileSizeHistogram(uploadFileSizers), TargetCount: b.Metrics.TargetCount, WorkspaceArtifactPathType: artifactPathType, - BoolValues: b.Metrics.BoolValues, + BoolValues: append(b.Telemetry.BoolValues(), b.Metrics.BoolValues...), LocalCacheMeasurementsMs: b.Metrics.LocalCacheMeasurementsMs, VariableCount: int64(variableCount), ComplexVariableCount: complexVariableCount, @@ -493,3 +485,23 @@ func LogDeployTelemetry(ctx context.Context, b *bundle.Bundle, errMsg string) { }, }) } + +// bundleBitmap encodes the bundle bitmap with the telemetry flags measured so +// far. Loading starts the bitmap once the target is selected, so it is nil when +// loading failed before that; deploy telemetry is logged on that path too. +func bundleBitmap(ctx context.Context, b *bundle.Bundle) string { + if b.Bitmap == nil { + return "" + } + bits, err := b.Bitmap.Bits(b.Telemetry) + if err != nil { + log.Debugf(ctx, "failed to compute the bundle bitmap: %s", err) + return "" + } + encoded, err := bitmap.Encode(bits, bitmap.ContextFullBundle) + if err != nil { + log.Debugf(ctx, "failed to encode the bundle bitmap: %s", err) + return "" + } + return encoded +} diff --git a/bundle/phases/telemetry_test.go b/bundle/phases/telemetry_test.go index 1436ecb7e84..613f0f6de10 100644 --- a/bundle/phases/telemetry_test.go +++ b/bundle/phases/telemetry_test.go @@ -129,9 +129,9 @@ func TestAiRuntimeTaskMetrics(t *testing.T) { }}} tests := []struct { - name string - jobs map[string]*resources.Job - present, scheduled, multitask bool + name string + jobs map[string]*resources.Job + scheduled, multitask []bool }{ { name: "no jobs", @@ -142,38 +142,43 @@ func TestAiRuntimeTaskMetrics(t *testing.T) { jobs: map[string]*resources.Job{"j": {JobSettings: jobs.JobSettings{Tasks: []jobs.Task{{TaskKey: "notebook"}}}}}, }, { - name: "single task", - jobs: map[string]*resources.Job{"j": aiRuntimeJob(false)}, - present: true, + name: "single task", + jobs: map[string]*resources.Job{"j": aiRuntimeJob(false)}, + scheduled: []bool{false}, multitask: []bool{false}, }, { - name: "multi-task", - jobs: map[string]*resources.Job{"j": aiRuntimeJob(true)}, - present: true, multitask: true, + name: "multi-task", + jobs: map[string]*resources.Job{"j": aiRuntimeJob(true)}, + scheduled: []bool{false}, multitask: []bool{true}, }, { - name: "scheduled single task", - jobs: map[string]*resources.Job{"j": scheduledJob}, - present: true, scheduled: true, + name: "scheduled single task", + jobs: map[string]*resources.Job{"j": scheduledJob}, + scheduled: []bool{true}, multitask: []bool{false}, }, { - name: "for_each nested ai_runtime_task", - jobs: map[string]*resources.Job{"j": forEachJob}, - present: true, + name: "for_each nested ai_runtime_task", + jobs: map[string]*resources.Job{"j": forEachJob}, + scheduled: []bool{false}, multitask: []bool{false}, }, { - name: "nil job is skipped", - jobs: map[string]*resources.Job{"j": nil, "k": aiRuntimeJob(false)}, - present: true, + name: "nil job is skipped", + jobs: map[string]*resources.Job{"j": nil, "k": aiRuntimeJob(false)}, + scheduled: []bool{false}, multitask: []bool{false}, + }, + { + name: "jobs that differ record both values", + jobs: map[string]*resources.Job{"j": scheduledJob, "k": aiRuntimeJob(true)}, + scheduled: []bool{true, false}, multitask: []bool{true, false}, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - present, scheduled, multitask := aiRuntimeTaskMetrics(tc.jobs) - assert.Equal(t, tc.present, present, "present") - assert.Equal(t, tc.scheduled, scheduled, "scheduled") - assert.Equal(t, tc.multitask, multitask, "multitask") + var telemetry bundle.Telemetry + aiRuntimeTaskMetrics(tc.jobs, &telemetry) + assert.Equal(t, tc.scheduled, telemetry.AiRuntimeTaskScheduled.Values(), "scheduled") + assert.Equal(t, tc.multitask, telemetry.AiRuntimeTaskMultitask.Values(), "multitask") }) } } diff --git a/bundle/statemgmt/direct_migration.go b/bundle/statemgmt/direct_migration.go index 11a0fc452c1..028dd08203d 100644 --- a/bundle/statemgmt/direct_migration.go +++ b/bundle/statemgmt/direct_migration.go @@ -18,7 +18,6 @@ import ( "github.com/databricks/cli/bundle/direct" "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/bundle/direct/dstate" - "github.com/databricks/cli/bundle/metrics" "github.com/databricks/cli/bundle/migrate" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/dyn" @@ -54,7 +53,7 @@ func Migrate(ctx context.Context, b *bundle.Bundle) (bool, error) { _, localTerraformPath := b.StateFilenameTerraform(ctx) tfState, err := migrate.ParseTFStateFull(ctx, localTerraformPath) if err != nil { - b.Metrics.SetBoolValue(metrics.DirectMigrateError, true) + b.Telemetry.DirectMigrateError = true return false, fmt.Errorf("parsing terraform state: %w%s", err, migrationFailedHint) } if tfState == nil { @@ -73,10 +72,10 @@ func Migrate(ctx context.Context, b *bundle.Bundle) (bool, error) { }() } if hasWarnings { - b.Metrics.SetBoolValue(metrics.DirectMigrateWarnings, true) + b.Telemetry.DirectMigrateWarnings = true } if err != nil { - b.Metrics.SetBoolValue(metrics.DirectMigrateError, true) + b.Telemetry.DirectMigrateError = true return false, fmt.Errorf("converting terraform state to the direct engine: %w%s", err, migrationFailedHint) } @@ -84,7 +83,7 @@ func Migrate(ctx context.Context, b *bundle.Bundle) (bool, error) { // record the recreate metric. plan, err := checkPlanOnTempState(ctx, b, tempStatePath, cfg) if err != nil { - b.Metrics.SetBoolValue(metrics.DirectMigratePlanError, true) + b.Telemetry.DirectMigratePlanError = true return false, fmt.Errorf("the migrated state failed its plan check: %w%s", err, migrationFailedHint) } @@ -92,7 +91,7 @@ func Migrate(ctx context.Context, b *bundle.Bundle) (bool, error) { // such a recreate is a genuine pending config change (the same one terraform would apply), and // the command's approval flow gates it behind --auto-approve exactly like any other recreate. if recreated := recreatedResources(plan); len(recreated) > 0 { - b.Metrics.SetBoolValue(metrics.DirectMigrateRecreatePlanned, true) + b.Telemetry.DirectMigrateRecreatePlanned = true log.Infof(ctx, "migration to the direct engine will recreate %v; the command's approval gates this", recreated) } @@ -168,11 +167,11 @@ func CommitMigration(ctx context.Context, b *bundle.Bundle) { func recordAutoMigrateSource(ctx context.Context, b *bundle.Bundle) { switch { case b.Config.Bundle.Engine != engine.EngineNotSet: - b.Metrics.SetBoolValue(metrics.DirectAutoMigrateViaConfig, true) + b.Telemetry.DirectAutoMigrateViaConfig = true case env.Get(ctx, engine.EnvVar) != "": - b.Metrics.SetBoolValue(metrics.DirectAutoMigrateViaEnv, true) + b.Telemetry.DirectAutoMigrateViaEnv = true default: - b.Metrics.SetBoolValue(metrics.DirectAutoMigrateViaDefault, true) + b.Telemetry.DirectAutoMigrateViaDefault = true } } diff --git a/bundle/telemetry.go b/bundle/telemetry.go new file mode 100644 index 00000000000..b2744b6c3aa --- /dev/null +++ b/bundle/telemetry.go @@ -0,0 +1,207 @@ +package bundle + +import ( + "reflect" + + "github.com/databricks/cli/libs/telemetry/protos" +) + +// Telemetry holds the boolean flags reported with a deploy. It is the single +// source of the deploy event's bool_values (see BoolValues) and of the +// telemetry.* bits of the bundle bitmap (see bundle/bitmap). It lives on Bundle, +// not on the config, so users cannot set it. +// +// The field type says what was measured: +// - bool: a flag reported only when true; absence means "not seen". +// - *bool: a measured value; nil when the measurement did not run. +// - BoolSet: a measurement taken several times (e.g. once per artifact); +// each value it took is reported once. +// +// The `metric` tag is the key in bool_values. Fields have no json tags: the +// bitmap uses the Go field names (e.g. "telemetry.SelectUsed"). +// +// Not here: the has_ keys, which the bitmap already records as config +// fields (resources.jobs.*.tasks[*]._task), and the local.cache.* keys, +// which describe the local user cache rather than the bundle. Both are written +// to Metrics.BoolValues directly. +type Telemetry struct { + PythonWheelWrapperIsSet *bool `metric:"python_wheel_wrapper_is_set"` + ArtifactDynamicVersionIsSet BoolSet `metric:"artifact_dynamic_version_is_set"` + ArtifactBuildCommandIsSet BoolSet `metric:"artifact_build_command_is_set"` + ArtifactFilesIsSet BoolSet `metric:"artifact_files_is_set"` + PresetsNamePrefixIsSet *bool `metric:"presets_name_prefix_is_set"` + // The lifecycle.started values set across the resources of each type. + AppLifecycleStarted BoolSet `metric:"app_lifecycle_started"` + ClusterLifecycleStarted BoolSet `metric:"cluster_lifecycle_started"` + DeploymentHistoryEnabled *bool `metric:"deployment_history_enabled"` + SqlWarehouseLifecycleStarted BoolSet `metric:"sql_warehouse_lifecycle_started"` + SelectUsed bool `metric:"select_used"` + + // Whether the deploy registered an immutable snapshot. Derived from the snapshot + // resource, not from experimental.immutable_folder: the flag may change, and the + // resource is what the deploy actually uploads. + ImmutableSnapshotUsed *bool `metric:"immutable_snapshot_used"` + + // Outcome of an automatic migration of terraform state to the direct engine. + // - migrate_error: state conversion itself errored. + // - migrate_warnings: the conversion emitted warnings (e.g. resources the direct engine can't represent). + DirectMigrateError bool `metric:"direct_migrate_error"` + DirectMigrateWarnings bool `metric:"direct_migrate_warnings"` + // True when the post-convert plan check failed; the migration was not committed. + DirectMigratePlanError bool `metric:"direct_migrate_plan_error"` + // True when the migrated state's first plan would recreate (destroy + create) an + // existing resource. The migration still commits; the recreate is a genuine pending + // change gated by the deploy's --auto-approve, and this only records how often it happens. + DirectMigrateRecreatePlanned bool `metric:"direct_migrate_recreate_planned"` + + // Recorded when an automatic post-deploy migration to the direct engine + // actually ran (state was rewritten). Exactly one of the three keys is true; + // all are absent when auto-migration did not run. If both config and env + // set direct, bundle.engine wins per ValidateEngineSetting, so via_config + // covers the "durable opt-in" population and via_env covers the + // "env-var only" population. + // - via_config: bundle.engine = "direct" was set in the bundle config. + // - via_env: only DATABRICKS_BUNDLE_ENGINE=direct was set. + // - via_default: neither was set, so the migration came from the default. + // This is the population that did not ask for anything. + DirectAutoMigrateViaConfig bool `metric:"direct_migrated_via_config"` + DirectAutoMigrateViaEnv bool `metric:"direct_migrated_via_env"` + DirectAutoMigrateViaDefault bool `metric:"direct_migrated_via_default"` + + // Whether the (deprecated) terraform engine was explicitly opted into, and via + // which source. Independent: both can be present when config and env both + // request terraform. Emitted only when true; absence means "not opted in via + // this source", so the population still on terraform can be sliced by source. + // - engine_terraform_config: bundle.engine = "terraform" in the bundle config. + // - engine_terraform_env: DATABRICKS_BUNDLE_ENGINE=terraform in the environment. + EngineTerraformConfig bool `metric:"engine_terraform_config"` + EngineTerraformEnv bool `metric:"engine_terraform_env"` + + // Whether workspace.state_path is under /Workspace/Shared. + StatePathIsShared *bool `metric:"state_path_is_shared"` + + // Whether this deploy is compatible with an automatic migration of the deployment + // state to a dedicated state storage service (DMS). Deploying a bundle requires + // write access (CAN_EDIT or higher) to the state folder; after migration that is + // governed by the permissions on the deployment object instead. + // + // When the bundle has no permissions section, the migration can mirror the state + // folder's ACLs onto the deployment (CAN_EDIT -> CAN_EDIT, CAN_MANAGE -> + // CAN_MANAGE), preserving everyone's access wherever the state lives. When a + // permissions section is set, the migration applies exactly those permissions, so + // anyone with write access to the state folder who is not declared with + // CAN_MANAGE would lose the ability to deploy. + // + // Exactly one of the three keys below is recorded per deploy: + // - auto: no permissions section (folder ACLs are mirrored), or every principal + // with write access to the state folder is declared. + // - only_self_undeclared: a permissions section is set and the only principal + // with undeclared write access is the deploying user. The migration + // grants the deploying user CAN_MANAGE on the deployment object, so this + // is auto-migratable if we choose to preserve that grant on future + // deploys. Recorded separately to measure how common this case is. + // - not: a permissions section is set and the state folder has undeclared write + // access from a principal other than the deploying user. + DMSCompatAuto bool `metric:"dms_compat_auto"` + DMSCompatOnlySelfUndeclared bool `metric:"dms_compat_only_self_undeclared"` + DMSCompatNot bool `metric:"dms_compat_not"` + + // Breakdown dimensions recorded on every deploy alongside the verdict above, so the + // DMS auto-migration population can be sliced without inferring it from the verdict. + // Each is an independent boolean. + + // Whether a top-level permissions section is set. The no-permissions case is always + // auto-migration compatible (folder ACLs are mirrored), so this separates the two + // populations that both land on dms_compat_auto. + PermissionsSectionSet *bool `metric:"permissions_section_set"` + + // Where the deployment state folder lives. Exactly one of StatePathIsShared, + // StatePathInDeployerHome, StatePathInOtherUserHome, and StatePathOther is true per + // deploy. StatePathOther is any other /Workspace folder (not a user home or shared). + StatePathInDeployerHome *bool `metric:"state_path_in_deployer_home"` + StatePathInOtherUserHome *bool `metric:"state_path_in_other_user_home"` + StatePathOther *bool `metric:"state_path_other"` + + // Which principal types have undeclared write access to the state folder — the + // access an auto-migration governed by the permissions section would drop. These can + // co-occur; all false when the deploy is auto-migration compatible. + DMSUndeclaredDeployingUser *bool `metric:"dms_undeclared_deploying_user"` + DMSUndeclaredOtherUser *bool `metric:"dms_undeclared_other_user"` + DMSUndeclaredServicePrincipal *bool `metric:"dms_undeclared_service_principal"` + DMSUndeclaredGroup *bool `metric:"dms_undeclared_group"` + + // Task-type usage is recorded generically at deploy as one "has_" key + // per task type present in the bundle (e.g. has_notebook_task, has_ai_runtime_task); + // see collectTaskTypes in bundle/phases. The two keys below are extra + // ai_runtime_task-specific dimensions, recorded for each job that declares an + // ai_runtime_task, so a false value means "has a job with an ai_runtime_task that is + // not scheduled / not multi-task". GPU type and count are intentionally not recorded here. + AiRuntimeTaskScheduled BoolSet `metric:"ai_runtime_task_scheduled"` + AiRuntimeTaskMultitask BoolSet `metric:"ai_runtime_task_multitask"` + + SkipArtifactCleanup *bool `metric:"skip_artifact_cleanup"` + SourceLinkedSetForNonDevelopment *bool `metric:"source_linked_set_for_non_development"` + ConfigHasDoubleDollarBrace bool `metric:"config_has_double_dollar_brace"` + ConfigHasDoubleDollar bool `metric:"config_has_double_dollar"` + ConfigHasBackslashDollarBrace bool `metric:"config_has_backslash_dollar_brace"` + ConfigHasBackslashDollar bool `metric:"config_has_backslash_dollar"` + ArtifactsReferenceUsed bool `metric:"artifacts_reference_used"` + UseLegacyRunAs *bool `metric:"experimental.use_legacy_run_as"` + RunAsSet *bool `metric:"run_as_set"` + HasTfOnlyReferences bool `metric:"has_tf_only_references"` + HasServerlessCompute *bool `metric:"has_serverless_compute"` + HasClassicJobCompute *bool `metric:"has_classic_job_compute"` + HasClassicInteractiveCompute *bool `metric:"has_classic_interactive_compute"` +} + +// BoolValues returns the flags as deploy-event entries, in field order: a true +// bool, the value of a non-nil *bool, and each value a BoolSet took. +func (t *Telemetry) BoolValues() []protos.BoolMapEntry { + var out []protos.BoolMapEntry + v := reflect.ValueOf(t).Elem() + for i := range v.NumField() { + key := v.Type().Field(i).Tag.Get("metric") + switch f := v.Field(i).Interface().(type) { + case bool: + if f { + out = append(out, protos.BoolMapEntry{Key: key, Value: true}) + } + case *bool: + if f != nil { + out = append(out, protos.BoolMapEntry{Key: key, Value: *f}) + } + case BoolSet: + for _, value := range f.Values() { + out = append(out, protos.BoolMapEntry{Key: key, Value: value}) + } + } + } + return out +} + +// BoolSet is a measurement that can be taken several times, e.g. once per +// artifact. It records which values it took: true, false, or both. +type BoolSet struct { //nolint:recvcheck // pointer receiver for SetBool, value for Values: reflection reads Telemetry by value + seenTrue, seenFalse bool +} + +// SetBool records that the measurement took value v. +func (s *BoolSet) SetBool(v bool) { + if v { + s.seenTrue = true + } else { + s.seenFalse = true + } +} + +// Values returns the values the measurement took, true before false. +func (s BoolSet) Values() []bool { + var values []bool + if s.seenTrue { + values = append(values, true) + } + if s.seenFalse { + values = append(values, false) + } + return values +} diff --git a/bundle/telemetry_test.go b/bundle/telemetry_test.go new file mode 100644 index 00000000000..fa564ab6b7a --- /dev/null +++ b/bundle/telemetry_test.go @@ -0,0 +1,130 @@ +package bundle + +import ( + "reflect" + "testing" + + "github.com/databricks/cli/libs/telemetry/protos" + "github.com/stretchr/testify/assert" +) + +func TestTelemetryBoolValues(t *testing.T) { + for _, tc := range []struct { + name string + tm Telemetry + want []protos.BoolMapEntry + }{ + { + name: "zero value reports nothing", + tm: Telemetry{}, + want: nil, + }, + { + name: "bool false is not reported", + tm: Telemetry{SelectUsed: false}, + want: nil, + }, + { + name: "bool true", + tm: Telemetry{SelectUsed: true}, + want: []protos.BoolMapEntry{{Key: "select_used", Value: true}}, + }, + { + name: "*bool nil is not reported", + tm: Telemetry{RunAsSet: nil}, + want: nil, + }, + { + name: "*bool false", + tm: Telemetry{RunAsSet: new(false)}, + want: []protos.BoolMapEntry{{Key: "run_as_set", Value: false}}, + }, + { + name: "*bool true", + tm: Telemetry{RunAsSet: new(true)}, + want: []protos.BoolMapEntry{{Key: "run_as_set", Value: true}}, + }, + { + name: "BoolSet unset is not reported", + tm: Telemetry{ArtifactFilesIsSet: boolSet()}, + want: nil, + }, + { + name: "BoolSet reports each value once, true first", + tm: Telemetry{ArtifactFilesIsSet: boolSet(false, true, false)}, + want: []protos.BoolMapEntry{ + {Key: "artifact_files_is_set", Value: true}, + {Key: "artifact_files_is_set", Value: false}, + }, + }, + { + name: "fields are reported in declaration order", + tm: Telemetry{RunAsSet: new(true), SelectUsed: true}, + want: []protos.BoolMapEntry{ + {Key: "select_used", Value: true}, + {Key: "run_as_set", Value: true}, + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, tc.tm.BoolValues()) + }) + } +} + +// BoolValues and the bitmap skip fields of any other type, and an empty or +// duplicate key would be indistinguishable on the wire. +func TestTelemetryFields(t *testing.T) { + supported := []reflect.Type{reflect.TypeFor[bool](), reflect.TypeFor[*bool](), reflect.TypeFor[BoolSet]()} + keys := map[string]string{} + for _, f := range reflect.VisibleFields(reflect.TypeFor[Telemetry]()) { + assert.Contains(t, supported, f.Type, "field %s", f.Name) + key := f.Tag.Get("metric") + assert.NotEmpty(t, key, "field %s has no metric tag", f.Name) + if other, ok := keys[key]; ok { + assert.Fail(t, "duplicate metric key", "%q is used by %s and %s", key, other, f.Name) + } + keys[key] = f.Name + } +} + +func TestBoolSet(t *testing.T) { + for _, tc := range []struct { + name string + set []bool + want []bool + }{ + { + name: "never set", + set: nil, + want: nil, + }, + { + name: "true", + set: []bool{true, true}, + want: []bool{true}, + }, + { + name: "false", + set: []bool{false}, + want: []bool{false}, + }, + { + name: "both, true first", + set: []bool{false, true}, + want: []bool{true, false}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, boolSet(tc.set...).Values()) + }) + } +} + +func boolSet(values ...bool) BoolSet { + var s BoolSet + for _, v := range values { + s.SetBool(v) + } + return s +} diff --git a/bundle/trampoline/python_wheel.go b/bundle/trampoline/python_wheel.go index e8138612dde..089771aeeda 100644 --- a/bundle/trampoline/python_wheel.go +++ b/bundle/trampoline/python_wheel.go @@ -10,7 +10,6 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/libraries" - "github.com/databricks/cli/bundle/metrics" "github.com/databricks/cli/libs/diag" "github.com/databricks/databricks-sdk-go/service/compute" "github.com/databricks/databricks-sdk-go/service/jobs" @@ -78,7 +77,7 @@ func TransformWheelTask() bundle.Mutator { func (transformWheelTask) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { isEnabled := b.Config.Experimental != nil && config.IsExplicitlyEnabled(b.Config.Experimental.PythonWheelWrapper) - b.Metrics.AddBoolValue(metrics.ExperimentalPythonWheelWrapperIsSet, isEnabled) + b.Telemetry.PythonWheelWrapperIsSet = new(isEnabled) if !isEnabled { return nil } diff --git a/cmd/bundle/bitmap/bitmap.go b/cmd/bundle/bitmap/bitmap.go new file mode 100644 index 00000000000..26a0f51f468 --- /dev/null +++ b/cmd/bundle/bitmap/bitmap.go @@ -0,0 +1,203 @@ +package bitmap + +import ( + "encoding/json" + "fmt" + "io" + "reflect" + "strings" + + "github.com/databricks/cli/bundle" + bundlebitmap "github.com/databricks/cli/bundle/bitmap" + "github.com/databricks/cli/cmd/bundle/utils" + "github.com/databricks/cli/cmd/root" + "github.com/spf13/cobra" +) + +// NewBitmapCommand returns the hidden "bitmap" command group. It exposes the +// bundle bitmap: a per-field presence map used for telemetry. See +// bundle/bitmap for the format and schema. +func NewBitmapCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "bitmap", + Short: "Inspect the bundle field bitmap used for telemetry", + Long: `The bundle bitmap encodes the presence of every bundle configuration field +as a single bit. Its schema is an ordered list of field paths derived from the +bundle configuration type and embedded into the CLI binary.`, + Hidden: true, + } + cmd.AddCommand(newSchemaCommand()) + cmd.AddCommand(newUpdateSchemaCommand()) + cmd.AddCommand(newBitmapCommand()) + cmd.AddCommand(newDecodeCommand()) + return cmd +} + +func newSchemaCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "schema", + Short: "Print the embedded bitmap schema, one field path per line", + Args: root.NoArgs, + } + cmd.RunE = func(cmd *cobra.Command, args []string) error { + return printLines(cmd.OutOrStdout(), bundlebitmap.EmbeddedSchema()) + } + return cmd +} + +func newUpdateSchemaCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "update-schema", + Short: "Append newly added fields to the embedded schema", + Long: `Walks the bundle configuration type and appends any fields not already in the +embedded schema, printing the merged schema. The schema is append-only: removed +fields are kept so that bit positions stay stable.`, + Args: root.NoArgs, + } + var validate bool + cmd.Flags().BoolVar(&validate, "validate", false, "Fail if the embedded schema is missing fields instead of printing the merged schema") + cmd.RunE = func(cmd *cobra.Command, args []string) error { + fresh, err := bundlebitmap.WalkSchema(reflect.TypeFor[bundle.Telemetry]()) + if err != nil { + return err + } + merged, added := bundlebitmap.Merge(bundlebitmap.EmbeddedSchema(), fresh) + if validate { + if len(added) > 0 { + return fmt.Errorf("embedded schema is out of date, %d field(s) missing:\n%s\nrun 'task generate-bitmap-schema' to update", len(added), strings.Join(added, "\n")) + } + return nil + } + return printLines(cmd.OutOrStdout(), merged) + } + return cmd +} + +// Values of the -o flag. +const ( + outputCompressed = "compressed" + outputText = "text" + outputJSON = "json" +) + +// outputFormat is the -o flag of the bitmap and decode commands. It shadows the +// global --output flag, which only accepts text and json. +type outputFormat string + +func (f *outputFormat) String() string { + return string(*f) +} + +func (f *outputFormat) Set(s string) error { + switch s { + case outputCompressed, outputText, outputJSON: + *f = outputFormat(s) + return nil + } + return fmt.Errorf("accepted values are %s, %s and %s", outputCompressed, outputText, outputJSON) +} + +func (f *outputFormat) Type() string { + return "format" +} + +const outputHelp = `output format: + compressed: base64 of the DEFLATE-compressed wire format + text: a "size=N context=C" header, then one "PTF " line per set + field: "P" if present, "T" and "F" if a boolean field held true + and false, a space for each that is not; unset fields are omitted + json: size, context, and trees of the fields present ("presence") and of + the boolean fields seen holding "true" and "false"` + +func addOutputFlag(cmd *cobra.Command, output *outputFormat, def string) { + *output = outputFormat(def) + cmd.Flags().VarP(output, "output", "o", outputHelp) +} + +func newBitmapCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "bitmap", + Short: "Print the bitmap of the current bundle", + Args: root.NoArgs, + } + var output outputFormat + addOutputFlag(cmd, &output, outputCompressed) + cmd.RunE = func(cmd *cobra.Command, args []string) error { + bits, schema, err := loadBits(cmd) + if err != nil { + return err + } + return render(cmd.OutOrStdout(), output, schema, bits, bundlebitmap.ContextFullBundle) + } + return cmd +} + +func newDecodeCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "decode", + Short: "Decode a compressed bitmap read from stdin", + Long: "The input is the output of 'bundle bitmap bitmap'.", + Args: root.NoArgs, + } + var output outputFormat + addOutputFlag(cmd, &output, outputText) + cmd.RunE = func(cmd *cobra.Command, args []string) error { + encoded, err := io.ReadAll(cmd.InOrStdin()) + if err != nil { + return err + } + bits, context, err := bundlebitmap.Decode(strings.TrimSpace(string(encoded))) + if err != nil { + return err + } + if context != bundlebitmap.ContextFullBundle { + return fmt.Errorf("unsupported bitmap context %d", context) + } + // The schema is append-only, so an older bitmap decodes against a prefix of + // it; a newer one has bits this binary cannot name, shown as placeholders. + return render(cmd.OutOrStdout(), output, bundlebitmap.EmbeddedSchema(), bits, context) + } + return cmd +} + +func render(out io.Writer, output outputFormat, schema []string, bits []bool, context uint16) error { + switch output { + case outputCompressed: + encoded, err := bundlebitmap.Encode(bits, context) + if err != nil { + return err + } + return printLines(out, []string{encoded}) + case outputText: + return printLines(out, bundlebitmap.Text(schema, bits, context)) + case outputJSON: + data, err := json.MarshalIndent(bundlebitmap.ToJSON(schema, bits, context), "", " ") + if err != nil { + return err + } + return printLines(out, []string{string(data)}) + } + return fmt.Errorf("unknown output format %q", output) +} + +// loadBits loads the bundle and returns its bitmap against the embedded schema. +func loadBits(cmd *cobra.Command) ([]bool, []string, error) { + b, err := utils.ProcessBundle(cmd, utils.ProcessOptions{Validate: true}) + if err != nil { + return nil, nil, err + } + bits, err := b.Bitmap.Bits(b.Telemetry) + if err != nil { + return nil, nil, err + } + return bits, bundlebitmap.EmbeddedSchema(), nil +} + +func printLines(out io.Writer, lines []string) error { + for _, l := range lines { + if _, err := fmt.Fprintln(out, l); err != nil { + return err + } + } + return nil +} diff --git a/cmd/bundle/bundle.go b/cmd/bundle/bundle.go index ad384c8d148..2d5e1854689 100644 --- a/cmd/bundle/bundle.go +++ b/cmd/bundle/bundle.go @@ -3,6 +3,7 @@ package bundle import ( "context" + "github.com/databricks/cli/cmd/bundle/bitmap" "github.com/databricks/cli/cmd/bundle/deployment" "github.com/databricks/cli/cmd/root" "github.com/spf13/cobra" @@ -46,4 +47,5 @@ Online documentation: https://docs.databricks.com/en/dev-tools/bundles/index.htm cmd.AddCommand(newOpenCommand()) cmd.AddCommand(newPlanCommand()) cmd.AddCommand(newConfigRemoteSyncCommand()) + cmd.AddCommand(bitmap.NewBitmapCommand()) } diff --git a/libs/telemetry/protos/bundle_deploy.go b/libs/telemetry/protos/bundle_deploy.go index 3200c3860c7..38c4db28df2 100644 --- a/libs/telemetry/protos/bundle_deploy.go +++ b/libs/telemetry/protos/bundle_deploy.go @@ -59,6 +59,14 @@ type BundleDeployEvent struct { // the field is omitted only when git detection did not run or failed. InGitRepo *bool `json:"in_git_repo,omitempty"` + // The bundle bitmap: one bit per field of the append-only schema in + // bundle/bitmap, set for each config field the bundle sets and each boolean + // telemetry flag measured, with the values of boolean fields. Base64 of raw + // DEFLATE; decode with 'databricks bundle bitmap decode' or the decoder in + // databricks-bundles. Field names, not values: map keys such as resource + // names are recorded as "*". Empty when the configuration did not load. + BundleBitmap string `json:"bundle_bitmap,omitempty"` + Experimental *BundleDeployExperimental `json:"experimental,omitempty"` } diff --git a/python/Taskfile.yml b/python/Taskfile.yml index 28595ae9cf5..6e524f8c606 100644 --- a/python/Taskfile.yml +++ b/python/Taskfile.yml @@ -82,7 +82,10 @@ tasks: - | find databricks/bundles -type d -mindepth 1 -maxdepth 1 \ ! -path databricks/bundles/core \ + ! -path databricks/bundles/_bitmap \ -exec rm -rf {} \; + # _bitmap/ is the bundle bitmap decoder; its schema.deflate comes from + # generate-bitmap-schema in the root Taskfile, not from this codegen. # core/ is hand-written except for the generated wiring under _generated/. - rm -rf databricks/bundles/core/_generated # test_resources.py is hand-written except for the generated TestCase data. diff --git a/python/codegen/codegen/generated_docs.py b/python/codegen/codegen/generated_docs.py index 3ae866a09d3..9c54a800759 100644 --- a/python/codegen/codegen/generated_docs.py +++ b/python/codegen/codegen/generated_docs.py @@ -35,11 +35,14 @@ def write_docs(output: str): docs = Path(output) / "docs" bundles = Path(output) / "databricks" / "bundles" - # core is hand-written; every other package dir gets a generated page. + # core is hand-written; every other public package dir gets a generated page. + # Private packages (e.g. _bitmap) are not part of the documented API. namespaces = sorted( p.name for p in bundles.iterdir() - if p.name != "core" and (p / "__init__.py").exists() + if p.name != "core" + and not p.name.startswith("_") + and (p / "__init__.py").exists() ) # Drop stale pages so a removed resource loses its page; keep core.rst. diff --git a/python/databricks/bundles/_bitmap/__init__.py b/python/databricks/bundles/_bitmap/__init__.py new file mode 100644 index 00000000000..fc9be8c2974 --- /dev/null +++ b/python/databricks/bundles/_bitmap/__init__.py @@ -0,0 +1,187 @@ +"""Decoder for the bundle bitmap that ``databricks bundle bitmap bitmap`` prints. + +Self-contained: the package ships the schema (``schema.deflate``, a copy of the +CLI's), so decoding does not need the CLI. :func:`decode` returns the same lines as +``databricks bundle bitmap decode`` and :func:`decode_json` the same object as +``databricks bundle bitmap decode -o json``. + +Wire format (``bundle/bitmap/bitmap.go`` in the CLI): base64 of raw DEFLATE of +magic ``DBTB`` (4 bytes) | size (uint32, big-endian) | context (uint16, +big-endian) | ceil(size / 8) bitmap bytes, with bit ``i`` (MSB first within each +byte) standing for schema line ``i``. +""" + +import base64 +import struct +import zlib +from importlib.resources import files +from typing import Any + +__all__ = [ + "CONTEXT_FULL_BUNDLE", + "SCHEMA", + "decode", + "decode_bits", + "decode_json", + "text", + "to_json", +] + +CONTEXT_FULL_BUNDLE = 0 + +_MAGIC = b"DBTB" +_HEADER_SIZE = 10 +_TRUE_SUFFIX = "=true" +_FALSE_SUFFIX = "=false" +_LIST_SUFFIX = "[*]" + +# The schema, one field path per line, compressed with raw DEFLATE. Generated by +# 'task generate-bitmap-schema' in the CLI repo, byte for byte the copy the CLI embeds. +SCHEMA: list[str] = ( + zlib.decompress(files(__name__).joinpath("schema.deflate").read_bytes(), -15) + .decode() + .split() +) + + +def decode(encoded: str) -> list[str]: + """Decode an encoded bitmap against the embedded schema into :func:`text` lines.""" + bits, context = _decode_bundle(encoded) + return text(SCHEMA, bits, context) + + +def decode_json(encoded: str) -> dict[str, Any]: + """Decode an encoded bitmap against the embedded schema into :func:`to_json` output.""" + bits, context = _decode_bundle(encoded) + return to_json(SCHEMA, bits, context) + + +def _decode_bundle(encoded: str) -> tuple[list[bool], int]: + bits, context = decode_bits(encoded) + if context != CONTEXT_FULL_BUNDLE: + raise ValueError(f"unsupported bitmap context {context}") + # The schema is append-only, so an older bitmap decodes against a prefix of + # it; a newer one has bits this package cannot name, shown as placeholders. + return bits, context + + +def decode_bits(encoded: str) -> tuple[list[bool], int]: + """Decode an encoded bitmap into its bits and context. + + Raises ``ValueError`` (including ``binascii.Error``) or ``zlib.error`` on input + that is not an encoded bitmap. + """ + raw = zlib.decompress(base64.b64decode(encoded.strip(), validate=True), -15) + if len(raw) < _HEADER_SIZE: + raise ValueError(f"payload too short: {len(raw)} bytes") + if raw[:4] != _MAGIC: + raise ValueError(f"bad magic {raw[:4]!r}") + size, context = struct.unpack(">IH", raw[4:_HEADER_SIZE]) + if len(raw) - _HEADER_SIZE != (size + 7) // 8: + raise ValueError(f"payload size mismatch: {size} bits, {len(raw)} bytes") + bitmap = raw[_HEADER_SIZE:] + bits = [(bitmap[i // 8] >> (7 - i % 8)) & 1 == 1 for i in range(size)] + return bits, context + + +def text(schema: list[str], bits: list[bool], context: int) -> list[str]: + """Render a ``"size=N context=C"`` header and then one ``"PTF "`` line + per set field in schema order. + + ``P`` is a present field, ``T`` and ``F`` a bool field seen holding true and + false, and a space stands for each that is not set. A bool with ``P`` alone + holds an unresolved variable reference; one with both ``T`` and ``F`` has both + values under a wildcard. Unset fields are omitted. ``bits`` may be shorter + than ``schema`` (an older bitmap) or longer (a newer one): a set bit past the + schema is a ``P`` line named by :func:`_future_field`. + """ + schema = schema[: len(bits)] + index = {path: i for i, path in enumerate(schema)} + lines = [f"size={len(bits)} context={context}"] + for i, path in enumerate(schema): + if path.endswith((_TRUE_SUFFIX, _FALSE_SUFFIX)): + continue + flags = ( + ("P" if bits[i] else " ") + + ("T" if _is_set(index, bits, path + _TRUE_SUFFIX) else " ") + + ("F" if _is_set(index, bits, path + _FALSE_SUFFIX) else " ") + ) + if flags != " ": + lines.append(f"{flags} {path}") + for i in range(len(schema), len(bits)): + if bits[i]: + lines.append("P " + _future_field(i)) + return lines + + +def _future_field(bit: int) -> str: + # A set bit past the end of the schema, a field only a newer CLI can name. + # Whether it is a field or the value of a bool is unknown. + return f"future #{bit}" + + +def _is_set(index: dict[str, int], bits: list[bool], path: str) -> bool: + i = index.get(path) + return i is not None and bits[i] + + +def to_json(schema: list[str], bits: list[bool], context: int) -> dict[str, Any]: + """Render the set bits as ``size``, ``context`` and three trees of set fields: + ``presence``, ``true`` and ``false``. + + The trees are keyed by path segment (``*`` for a map key, and a list field's + name plus ``[*]`` for its elements) with ``True`` leaves; ``tasks[*]`` replaces + the leaf of ``tasks``. A bool field is in ``presence`` and in the tree of each + value it holds. Keys are in the CLI's order, so ``json.dumps(..., + indent=2)`` prints what ``databricks bundle bitmap decode -o json`` prints. + ``bits`` may be shorter than ``schema`` or longer: a set bit past the schema + is a top-level ``presence`` entry named by :func:`_future_field`. + """ + presence: dict[str, Any] = {} + true_tree: dict[str, Any] = {} + false_tree: dict[str, Any] = {} + known = min(len(bits), len(schema)) + for path, bit in zip(schema[:known], bits[:known], strict=True): + if not bit: + continue + tree = presence + if path.endswith(_TRUE_SUFFIX): + tree, path = true_tree, path[: -len(_TRUE_SUFFIX)] + elif path.endswith(_FALSE_SUFFIX): + tree, path = false_tree, path[: -len(_FALSE_SUFFIX)] + _insert(tree, path.split(".")) + for i in range(known, len(bits)): + if bits[i]: + presence[_future_field(i)] = True + return { + "size": len(bits), + "context": context, + "presence": _sorted(presence), + "true": _sorted(true_tree), + "false": _sorted(false_tree), + } + + +def _insert(tree: dict[str, Any], keys: list[str]) -> None: + # A field with set children is a subtree: a child replaces the field's True + # leaf, and a leaf never overwrites a subtree. Likewise "tasks[*]" replaces + # the "tasks" leaf, and a "tasks" leaf is not added next to "tasks[*]". + for i, key in enumerate(keys): + if key.endswith(_LIST_SUFFIX): + tree.pop(key[: -len(_LIST_SUFFIX)], None) + if i == len(keys) - 1: + if key not in tree and key + _LIST_SUFFIX not in tree: + tree[key] = True + return + child = tree.get(key) + if not isinstance(child, dict): + child = tree[key] = {} + tree = child + + +def _sorted(tree: dict[str, Any]) -> dict[str, Any]: + # The CLI marshals Go maps, which encoding/json writes with sorted keys. + return { + key: _sorted(value) if isinstance(value, dict) else value + for key, value in sorted(tree.items()) + } diff --git a/python/databricks/bundles/_bitmap/__main__.py b/python/databricks/bundles/_bitmap/__main__.py new file mode 100644 index 00000000000..e782dcdcb47 --- /dev/null +++ b/python/databricks/bundles/_bitmap/__main__.py @@ -0,0 +1,43 @@ +"""Decode a bundle bitmap read from stdin, like ``databricks bundle bitmap decode``. + +Usage: ``python -m databricks.bundles._bitmap [-o text|json] < bitmap.b64`` +""" + +import argparse +import json +import sys +import zlib + +from databricks.bundles._bitmap import decode, decode_json + + +def main() -> int: + parser = argparse.ArgumentParser( + prog="python -m databricks.bundles._bitmap", + description="Decode a bundle bitmap read from stdin.", + ) + parser.add_argument( + "-o", + "--output", + choices=["text", "json"], + default="text", + help="text: one line per set field; json: trees of set fields", + ) + args = parser.parse_args() + try: + encoded = sys.stdin.read() + if args.output == "json": + lines = [json.dumps(decode_json(encoded), indent=2)] + else: + lines = decode(encoded) + except (ValueError, zlib.error) as e: + print(f"Error: {e}", file=sys.stderr) + return 1 + # Write bytes so Windows does not translate "\n" to "\r\n": the output must + # match the CLI's byte for byte. + sys.stdout.buffer.write("".join(line + "\n" for line in lines).encode()) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/python/databricks/bundles/_bitmap/schema.deflate b/python/databricks/bundles/_bitmap/schema.deflate new file mode 100644 index 00000000000..21b84b572f3 Binary files /dev/null and b/python/databricks/bundles/_bitmap/schema.deflate differ diff --git a/python/databricks_tests/bundles/test_bitmap.py b/python/databricks_tests/bundles/test_bitmap.py new file mode 100644 index 00000000000..34e749fa341 --- /dev/null +++ b/python/databricks_tests/bundles/test_bitmap.py @@ -0,0 +1,234 @@ +import base64 +import struct +import zlib + +import pytest + +from databricks.bundles._bitmap import ( + CONTEXT_FULL_BUNDLE, + SCHEMA, + decode, + decode_bits, + decode_json, + text, + to_json, +) + + +def _encode(raw: bytes) -> str: + compressor = zlib.compressobj(wbits=-15) + return base64.b64encode(compressor.compress(raw) + compressor.flush()).decode() + + +def _payload(bits: list[bool], context: int = 0) -> bytes: + bitmap = bytearray((len(bits) + 7) // 8) + for i, bit in enumerate(bits): + if bit: + bitmap[i // 8] |= 1 << (7 - i % 8) + return b"DBTB" + struct.pack(">IH", len(bits), context) + bytes(bitmap) + + +def test_embedded_schema(): + assert "bundle.name" in SCHEMA + assert "telemetry.SelectUsed=true" in SCHEMA + assert len(SCHEMA) == len(set(SCHEMA)) + + +@pytest.mark.parametrize("n", [0, 1, 7, 8, 9, 16, 17]) +def test_decode_bits_round_trip(n: int): + bits = [i % 3 == 0 for i in range(n)] + assert decode_bits(_encode(_payload(bits, context=0xBEEF))) == (bits, 0xBEEF) + + +def test_decode_bits_reads_cli_payload(): + # The raw payload that TestEncodeBitOrder pins for the CLI's Encode of bits + # 1,0,0,0,0,0,0,1,1, so both implementations read the same bit order. + encoded = _encode( + bytes([0x44, 0x42, 0x54, 0x42, 0, 0, 0, 9, 0, 0, 0b1000_0001, 0b1000_0000]) + ) + assert decode_bits(encoded) == ( + [True, False, False, False, False, False, False, True, True], + 0, + ) + + +@pytest.mark.parametrize( + ("encoded", "error"), + [ + ("not-valid-base64!!!", "base64"), + (base64.b64encode(b"\xff\xff").decode(), "decompressing"), + (_encode(b"DBTB"), "payload too short"), + (_encode(b"XXXX" + struct.pack(">IH", 1, 0) + b"\x80"), "bad magic"), + (_encode(b"DBTB" + struct.pack(">IH", 9, 0)), "payload size mismatch"), + (_encode(b"DBTB" + struct.pack(">IH", 0xFFFFFFFF, 0)), "payload size mismatch"), + ], +) +def test_decode_bits_errors(encoded: str, error: str): + with pytest.raises((ValueError, zlib.error), match=error): + decode_bits(encoded) + + +SCHEMA_FOR_TEXT = [ + "bundle", + "bundle.name", + "flag", + "flag=true", + "flag=false", + "jobs.*.enabled", + "jobs.*.enabled=true", + "jobs.*.enabled=false", + "ref", + "ref=true", + "ref=false", +] + + +@pytest.mark.parametrize( + ("set_paths", "lines"), + [ + ([], []), + (["bundle", "bundle.name"], ["P bundle", "P bundle.name"]), + (["flag", "flag=true"], ["PT flag"]), + (["flag", "flag=false"], ["P F flag"]), + ( + ["jobs.*.enabled", "jobs.*.enabled=true", "jobs.*.enabled=false"], + ["PTF jobs.*.enabled"], + ), + (["ref"], ["P ref"]), + (["flag=true"], [" T flag"]), + ], +) +def test_text(set_paths: list[str], lines: list[str]): + bits = [path in set_paths for path in SCHEMA_FOR_TEXT] + header = f"size={len(SCHEMA_FOR_TEXT)} context=0" + assert text(SCHEMA_FOR_TEXT, bits, CONTEXT_FULL_BUNDLE) == [header, *lines] + + +def test_text_older_bitmap(): + assert text(SCHEMA_FOR_TEXT, [True], 7) == ["size=1 context=7", "P bundle"] + + +def test_text_newer_bitmap(): + bits = [path == "bundle" for path in SCHEMA_FOR_TEXT] + [False, True] + n = len(SCHEMA_FOR_TEXT) + assert text(SCHEMA_FOR_TEXT, bits, 0) == [ + f"size={n + 2} context=0", + "P bundle", + f"P future #{n + 1}", + ] + + +def test_decode(): + bits = [path in ("bundle", "bundle.name") for path in SCHEMA] + assert decode(_encode(_payload(bits))) == [ + f"size={len(SCHEMA)} context=0", + "P bundle", + "P bundle.name", + ] + + +@pytest.mark.parametrize("decoder", [decode, decode_json]) +def test_decode_rejects_unknown_context(decoder): + with pytest.raises(ValueError, match="unsupported bitmap context 1"): + decoder(_encode(_payload([], context=1))) + + +def test_decode_newer_bitmap(): + # A newer CLI's bitmap has fields past the embedded schema; a set one is a placeholder. + bits = [path == "bundle" for path in SCHEMA] + [False, True] + encoded = _encode(_payload(bits)) + future = f"future #{len(SCHEMA) + 1}" + assert decode(encoded) == [ + f"size={len(SCHEMA) + 2} context=0", + "P bundle", + f"P {future}", + ] + assert decode_json(encoded) == { + "size": len(SCHEMA) + 2, + "context": 0, + "presence": {"bundle": True, future: True}, + "true": {}, + "false": {}, + } + + +SCHEMA_FOR_JSON = [ + "bundle", + "bundle.name", + "jobs", + "jobs.*", + "jobs.*.tasks", + "jobs.*.tasks[*]", + "jobs.*.tasks[*].enabled", + "jobs.*.tasks[*].enabled=true", + "jobs.*.tasks[*].enabled=false", + "later", +] +TASKS = ["jobs", "jobs.*", "jobs.*.tasks"] +ENABLED = [*TASKS, "jobs.*.tasks[*]", "jobs.*.tasks[*].enabled"] +ENABLED_TREE = {"jobs": {"*": {"tasks[*]": {"enabled": True}}}} + + +@pytest.mark.parametrize( + ("set_paths", "presence", "true", "false"), + [ + ([], {}, {}, {}), + # A parent with set children is a subtree. + (["bundle", "bundle.name"], {"bundle": {"name": True}}, {}, {}), + # A set field without set children is a leaf. + (TASKS, {"jobs": {"*": {"tasks": True}}}, {}, {}), + # A list's elements replace its leaf. + ([*TASKS, "jobs.*.tasks[*]"], {"jobs": {"*": {"tasks[*]": True}}}, {}, {}), + ([*ENABLED, "jobs.*.tasks[*].enabled=false"], ENABLED_TREE, {}, ENABLED_TREE), + # Both values under a wildcard. + ( + [*ENABLED, "jobs.*.tasks[*].enabled=true", "jobs.*.tasks[*].enabled=false"], + ENABLED_TREE, + ENABLED_TREE, + ENABLED_TREE, + ), + ], +) +def test_to_json(set_paths: list[str], presence: dict, true: dict, false: dict): + bits = [path in set_paths for path in SCHEMA_FOR_JSON] + assert to_json(SCHEMA_FOR_JSON, bits, CONTEXT_FULL_BUNDLE) == { + "size": len(SCHEMA_FOR_JSON), + "context": 0, + "presence": presence, + "true": true, + "false": false, + } + + +def test_to_json_older_bitmap(): + assert to_json(SCHEMA_FOR_JSON, [True], 7) == { + "size": 1, + "context": 7, + "presence": {"bundle": True}, + "true": {}, + "false": {}, + } + + +def test_to_json_list_leaf_in_any_order(): + out = to_json(["x[*]", "x"], [True, True], CONTEXT_FULL_BUNDLE) + assert out["presence"] == {"x[*]": True} + + +def test_to_json_key_order(): + # Same order as the CLI, which sorts map keys, so the printed JSON matches. + out = to_json(["b", "a", "a.z", "a.y"], [True] * 4, CONTEXT_FULL_BUNDLE) + assert list(out) == ["size", "context", "presence", "true", "false"] + assert list(out["presence"]) == ["a", "b"] + assert list(out["presence"]["a"]) == ["y", "z"] + + +def test_decode_json(): + bits = [path in ("bundle", "bundle.name") for path in SCHEMA] + assert decode_json(_encode(_payload(bits))) == { + "size": len(SCHEMA), + "context": 0, + "presence": {"bundle": {"name": True}}, + "true": {}, + "false": {}, + }