From e95ddeb302f73c0a6e7128e98254cabfc93d398f Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 4 Jun 2026 18:30:34 +0200 Subject: [PATCH 1/2] Remove resref/resreferr telemetry Co-authored-by: Isaac --- .../config/mutator/log_resource_references.go | 148 ------------------ .../mutator/log_resource_references_test.go | 110 ------------- .../process_static_resources.go | 2 - 3 files changed, 260 deletions(-) delete mode 100644 bundle/config/mutator/log_resource_references.go delete mode 100644 bundle/config/mutator/log_resource_references_test.go diff --git a/bundle/config/mutator/log_resource_references.go b/bundle/config/mutator/log_resource_references.go deleted file mode 100644 index a18b3aaa249..00000000000 --- a/bundle/config/mutator/log_resource_references.go +++ /dev/null @@ -1,148 +0,0 @@ -package mutator - -import ( - "context" - "reflect" - "strings" - - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/dynvar" - "github.com/databricks/cli/libs/log" - "github.com/databricks/cli/libs/structs/structaccess" - "github.com/databricks/cli/libs/structs/structpath" -) - -// Longest field name: -// >>> len('hard_deletion_sync_min_interval_in_seconds') == 42 -const maxFieldLength = 42 - -const maxMetricLength = 200 - -// logResourceReferences scans resources for ${resources.*} references and logs them. -func LogResourceReferences() bundle.Mutator { - return &logResourceReferences{} -} - -type logResourceReferences struct{} - -func (m *logResourceReferences) Name() string { - return "LogResourceReferences" -} - -func (m *logResourceReferences) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - used := map[string]struct{}{} - resources := b.Config.Value().Get("resources") - if !resources.IsValid() { - // No resources section, nothing to do - return nil - } - - _ = dyn.WalkReadOnly(resources, func(path dyn.Path, val dyn.Value) error { - ref, ok := dynvar.NewRef(val) - if !ok { - return nil - } - for _, r := range ref.References() { - // Only track ${resources.*} references. - if !strings.HasPrefix(r, "resources.") { - continue - } - - key := convertReferenceToMetric(ctx, b.Config, r) - if key != "" { - used[key] = struct{}{} - } - } - return nil - }) - - maxRefsLogged := 50 - - // map iteration is randomized, which works for this case - for key := range used { - b.Metrics.AddBoolValue(key, true) - maxRefsLogged-- - if maxRefsLogged <= 0 { - break - } - } - - return nil -} - -// convertReferenceToMetric converts a reference like "resources.jobs.foo.id" to -// a telemetry key like "resref__jobs__id" -func convertReferenceToMetric(ctx context.Context, cfg any, ref string) string { - p, err := dyn.NewPathFromString(ref) - if err != nil || len(p) < 3 || p[0].Key() != "resources" { - return "" - } - - group := truncate(p[1].Key(), maxFieldLength, "") - kept := []string{"resref_" + group} - - for i := 3; i < len(p); i++ { - c := p[i] - if c.Key() != "" { - item := c.Key() - - repl, err := censorValue(ctx, cfg, p[:i]) - if err != nil { - kept[0] = "resreferr_" + group - break - } - - if repl != "" { - item = repl - } - - item = truncate(item, maxFieldLength, "") - kept = append(kept, item) - continue - } - } - - result := strings.Join(kept, ".") - return truncate(result, maxMetricLength, "__cut") -} - -func truncate(s string, n int, suffix string) string { - if len(s) <= n { - return s - } - return s[:n] + suffix -} - -func censorValue(ctx context.Context, v any, path dyn.Path) (string, error) { - pathString := path.String() - pathNode, err := structpath.ParsePath(pathString) - if err != nil { - log.Warnf(ctx, "internal error: parsing %q: %s", pathString, err) - return "err", err - } - - v, err = structaccess.Get(v, pathNode) - if err != nil { - log.Infof(ctx, "internal error: path=%s: %s", path, err) - return "err", err - } - - rv := reflect.ValueOf(v) - for rv.IsValid() { - switch rv.Kind() { - case reflect.Pointer, reflect.Interface: - if rv.IsNil() { - return "", nil - } - rv = rv.Elem() - default: - if rv.Kind() == reflect.Map { - return "*", nil - } - return "", nil - } - } - return "", nil -} diff --git a/bundle/config/mutator/log_resource_references_test.go b/bundle/config/mutator/log_resource_references_test.go deleted file mode 100644 index 837a3dde1bb..00000000000 --- a/bundle/config/mutator/log_resource_references_test.go +++ /dev/null @@ -1,110 +0,0 @@ -package mutator - -import ( - "testing" - - "github.com/databricks/cli/bundle/config" - cres "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/databricks-sdk-go/service/catalog" - "github.com/databricks/databricks-sdk-go/service/jobs" - "github.com/stretchr/testify/assert" -) - -func TestConvertReferenceToMetric_Table(t *testing.T) { - ctx := t.Context() - cfg := config.Root{ - Resources: config.Resources{ - Jobs: map[string]*cres.Job{ - "foo": { - JobSettings: jobs.JobSettings{ - Tasks: []jobs.Task{ - {TaskKey: "alpha"}, - }, - }, - }, - "джоб": { - JobSettings: jobs.JobSettings{}, - }, - - "niljob": nil, - }, - QualityMonitors: map[string]*cres.QualityMonitor{ - "foo": { - CreateMonitor: catalog.CreateMonitor{ - TableName: "catalog.schema.table", - }, - }, - }, - }, - } - - tests := []struct { - name string - ref string - want string - }{ - { - name: "basic job id", - ref: "resources.jobs.foo.id", - want: "resref_jobs.id", - }, - { - name: "basic job id with non-ascii key", - ref: "resources.jobs.джоб.id", - want: "resref_jobs.id", - }, - { - name: "invalid empty", - ref: "", - want: "", - }, - { - name: "invalid variables", - ref: "variables.foo", - want: "", - }, - { - name: "invalid short resources", - ref: "resources", - want: "", - }, - { - name: "invalid short group", - ref: "resources.jobs", - want: "", - }, - { - name: "nil job pointer yields plain id", - ref: "resources.jobs.niljob.id", - want: "resref_jobs.id", - }, - { - name: "err censor on missing job key", - ref: "resources.jobs.missing.id", - want: "resreferr_jobs", - }, - { - // id is TF-only for quality_monitors but the reference should still resolve cleanly. - name: "quality monitor id", - ref: "resources.quality_monitors.foo.id", - want: "resref_quality_monitors.id", - }, - { - name: "array index tasks key", - ref: "resources.jobs.foo.tasks[0].task_key", - want: "resref_jobs.tasks.task_key", - }, - { - name: "array index task key", - ref: "resources.jobs.foo.task[0].task_key", - want: "resreferr_jobs.task", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := convertReferenceToMetric(ctx, cfg, tt.ref) - assert.Equal(t, tt.want, got) - }) - } -} diff --git a/bundle/config/mutator/resourcemutator/process_static_resources.go b/bundle/config/mutator/resourcemutator/process_static_resources.go index 3f859fd11cd..7d3ad742e4b 100644 --- a/bundle/config/mutator/resourcemutator/process_static_resources.go +++ b/bundle/config/mutator/resourcemutator/process_static_resources.go @@ -45,8 +45,6 @@ func (p processStaticResources) Apply(ctx context.Context, b *bundle.Bundle) dia // Updates (dynamic): resources.* (strings) (resolves variable references to their actual values) // Resolves variable references in 'resources' using bundle, workspace, and variables prefixes mutator.ResolveVariableReferencesOnlyResources(), - // After normal variable resolution, log all ${resources.*} references - mutator.LogResourceReferences(), mutator.NormalizePaths(), // Translate dashboard paths into paths in the workspace file system From 3a4f5462f7bbb6d6c6f28c566c96353cd72416d0 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 4 Jun 2026 20:24:42 +0200 Subject: [PATCH 2/2] Update acceptance test outputs after removing resref telemetry Co-authored-by: Isaac --- .../bundle/resource_deps/job_tasks/out.telemetry.direct.txt | 2 -- .../bundle/resource_deps/job_tasks/out.telemetry.terraform.txt | 2 -- acceptance/bundle/resource_deps/resources_var/output.txt | 3 --- .../resource_deps/tf_path_only_error/out.telemetry.direct.txt | 1 - .../tf_path_only_error/out.telemetry.terraform.txt | 1 - 5 files changed, 9 deletions(-) diff --git a/acceptance/bundle/resource_deps/job_tasks/out.telemetry.direct.txt b/acceptance/bundle/resource_deps/job_tasks/out.telemetry.direct.txt index 50371a06442..3e6a825eb5e 100644 --- a/acceptance/bundle/resource_deps/job_tasks/out.telemetry.direct.txt +++ b/acceptance/bundle/resource_deps/job_tasks/out.telemetry.direct.txt @@ -6,7 +6,5 @@ local.cache.attempt true local.cache.miss true presets_name_prefix_is_set false python_wheel_wrapper_is_set false -resref_jobs.tags.* true -resreferr_jobs.task true run_as_set false skip_artifact_cleanup false diff --git a/acceptance/bundle/resource_deps/job_tasks/out.telemetry.terraform.txt b/acceptance/bundle/resource_deps/job_tasks/out.telemetry.terraform.txt index 50371a06442..3e6a825eb5e 100644 --- a/acceptance/bundle/resource_deps/job_tasks/out.telemetry.terraform.txt +++ b/acceptance/bundle/resource_deps/job_tasks/out.telemetry.terraform.txt @@ -6,7 +6,5 @@ local.cache.attempt true local.cache.miss true presets_name_prefix_is_set false python_wheel_wrapper_is_set false -resref_jobs.tags.* true -resreferr_jobs.task true run_as_set false skip_artifact_cleanup false diff --git a/acceptance/bundle/resource_deps/resources_var/output.txt b/acceptance/bundle/resource_deps/resources_var/output.txt index cd34790c2ca..59ee3bc4616 100644 --- a/acceptance/bundle/resource_deps/resources_var/output.txt +++ b/acceptance/bundle/resource_deps/resources_var/output.txt @@ -44,8 +44,5 @@ local.cache.attempt true local.cache.hit true presets_name_prefix_is_set true python_wheel_wrapper_is_set false -resref_volumes.catalog_name true -resref_volumes.name true -resref_volumes.schema_name true run_as_set false skip_artifact_cleanup false diff --git a/acceptance/bundle/resource_deps/tf_path_only_error/out.telemetry.direct.txt b/acceptance/bundle/resource_deps/tf_path_only_error/out.telemetry.direct.txt index 1371f471078..f9aa960d00f 100644 --- a/acceptance/bundle/resource_deps/tf_path_only_error/out.telemetry.direct.txt +++ b/acceptance/bundle/resource_deps/tf_path_only_error/out.telemetry.direct.txt @@ -3,5 +3,4 @@ has_tf_only_references true local.cache.attempt true local.cache.hit true presets_name_prefix_is_set false -resref_jobs.always_running true run_as_set false diff --git a/acceptance/bundle/resource_deps/tf_path_only_error/out.telemetry.terraform.txt b/acceptance/bundle/resource_deps/tf_path_only_error/out.telemetry.terraform.txt index 82d3fbed127..fbb1d343107 100644 --- a/acceptance/bundle/resource_deps/tf_path_only_error/out.telemetry.terraform.txt +++ b/acceptance/bundle/resource_deps/tf_path_only_error/out.telemetry.terraform.txt @@ -7,6 +7,5 @@ local.cache.attempt true local.cache.hit true presets_name_prefix_is_set false python_wheel_wrapper_is_set false -resref_jobs.always_running true run_as_set false skip_artifact_cleanup false