From 1763d5d852dc8cfbb6b9eb6167ff336f4b2fd867 Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Wed, 18 Nov 2026 05:22:49 +0000 Subject: [PATCH 1/2] acc: disable DMS heartbeats in bundle/dms recording tests The heartbeat fires after 30s without a state write, so slow cloud deploys recorded an extra POST .../heartbeat at a random point. Add a test-only DATABRICKS_BUNDLE_DMS_HEARTBEAT_INTERVAL_MS override and set it to 1h for bundle/dms. cloud: TestAccept/bundle/dms --- acceptance/bundle/dms/test.toml | 4 ++++ libs/dms/heartbeat.go | 25 ++++++++++++++++++++----- libs/dms/heartbeat_test.go | 25 +++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 5 deletions(-) diff --git a/acceptance/bundle/dms/test.toml b/acceptance/bundle/dms/test.toml index f91ae5c29ab..62a2f2482a8 100644 --- a/acceptance/bundle/dms/test.toml +++ b/acceptance/bundle/dms/test.toml @@ -15,3 +15,7 @@ Ignore = [ # rejection path. Pinned to "true" (not the suite's ["", "true"]) since the field already # records - the empty variant would just duplicate it. EnvMatrix.DMS = ["true"] + +# The CLI sends a heartbeat after 30s without a state write, so a slow cloud deploy would +# record one at a random point. 1h never fires within a test. +Env.DATABRICKS_BUNDLE_DMS_HEARTBEAT_INTERVAL_MS = "3600000" diff --git a/libs/dms/heartbeat.go b/libs/dms/heartbeat.go index 4c239bcc647..8e33aad6bf3 100644 --- a/libs/dms/heartbeat.go +++ b/libs/dms/heartbeat.go @@ -2,9 +2,11 @@ package dms import ( "context" + "strconv" "time" "github.com/databricks/cli/libs/cmdctx" + "github.com/databricks/cli/libs/env" "github.com/databricks/cli/libs/log" "github.com/databricks/databricks-sdk-go/service/bundledeployments" ) @@ -12,6 +14,19 @@ import ( // heartbeatInterval is the maximum time between consecutive updates. const heartbeatInterval = 30 * time.Second +// HeartbeatIntervalMsVariable overrides heartbeatInterval in milliseconds. Acceptance tests that +// record requests set it high so a slow cloud deploy does not add a heartbeat at a random point. +const HeartbeatIntervalMsVariable = "DATABRICKS_BUNDLE_DMS_HEARTBEAT_INTERVAL_MS" + +// interval returns heartbeatInterval, or the override if it is set to a positive number. +func interval(ctx context.Context) time.Duration { + ms, err := strconv.Atoi(env.Get(ctx, HeartbeatIntervalMsVariable)) + if err != nil || ms <= 0 { + return heartbeatInterval + } + return time.Duration(ms) * time.Millisecond +} + // Heartbeat updates the update_time of an open deployment version. This is also updated by // CreateVersion and every successful write. type Heartbeat struct { @@ -32,12 +47,12 @@ func startVersionHeartbeat(ctx context.Context, deploymentID string, versionNum func startHeartbeat(ctx context.Context, send func(context.Context) error) *Heartbeat { ctx, cancel := context.WithCancel(ctx) h := &Heartbeat{renewed: make(chan struct{}, 1), cancel: cancel} - go h.run(ctx, send) + go h.run(ctx, send, interval(ctx)) return h } -func (h *Heartbeat) run(ctx context.Context, send func(context.Context) error) { - timer := time.NewTimer(heartbeatInterval) +func (h *Heartbeat) run(ctx context.Context, send func(context.Context) error, interval time.Duration) { + timer := time.NewTimer(interval) for { select { case <-ctx.Done(): @@ -52,9 +67,9 @@ func (h *Heartbeat) run(ctx context.Context, send func(context.Context) error) { if ctx.Err() != nil { return } - log.Infof(ctx, "Failed to send deployment heartbeat, so the deployment may show as interrupted; retrying in %s: %s", heartbeatInterval, err) + log.Infof(ctx, "Failed to send deployment heartbeat, so the deployment may show as interrupted; retrying in %s: %s", interval, err) } - timer.Reset(heartbeatInterval) + timer.Reset(interval) } } diff --git a/libs/dms/heartbeat_test.go b/libs/dms/heartbeat_test.go index 3b6dc320567..f29f11685d4 100644 --- a/libs/dms/heartbeat_test.go +++ b/libs/dms/heartbeat_test.go @@ -8,6 +8,7 @@ import ( "testing/synctest" "time" + "github.com/databricks/cli/libs/env" "github.com/stretchr/testify/assert" ) @@ -92,3 +93,27 @@ func TestHeartbeatStop(t *testing.T) { assert.Equal(t, int32(0), f.calls.Load()) }) } + +func TestHeartbeatIntervalOverride(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + f := &fakeSend{} + ctx := env.Set(t.Context(), HeartbeatIntervalMsVariable, "100000") + h := startHeartbeat(ctx, f.send) + defer h.Stop() + + time.Sleep(99 * time.Second) + synctest.Wait() + assert.Equal(t, int32(0), f.calls.Load()) + + time.Sleep(2 * time.Second) + synctest.Wait() + assert.Equal(t, int32(1), f.calls.Load()) + }) +} + +func TestHeartbeatIntervalInvalidOverride(t *testing.T) { + for _, v := range []string{"", "abc", "0", "-5"} { + ctx := env.Set(t.Context(), HeartbeatIntervalMsVariable, v) + assert.Equal(t, heartbeatInterval, interval(ctx), v) + } +} From dbfb998725bfbf9083d157f13115a5e32eae08d0 Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Wed, 18 Nov 2026 07:21:32 +0000 Subject: [PATCH 2/2] acc: show the DMS heartbeat interval override reaching the CLI bundle/dms_fault/heartbeat sets the interval to 1ms, so the faulted heartbeat endpoint is actually called (DMS API called: yes) and deploys still succeed while every heartbeat fails. cloud: TestAccept/bundle/dms Co-authored-by: Isaac --- acceptance/bundle/dms_fault/heartbeat/output.txt | 2 +- acceptance/bundle/dms_fault/heartbeat/test.toml | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 acceptance/bundle/dms_fault/heartbeat/test.toml diff --git a/acceptance/bundle/dms_fault/heartbeat/output.txt b/acceptance/bundle/dms_fault/heartbeat/output.txt index b512e794a75..8b50d26db05 100644 --- a/acceptance/bundle/dms_fault/heartbeat/output.txt +++ b/acceptance/bundle/dms_fault/heartbeat/output.txt @@ -141,4 +141,4 @@ Destroy: 2 deleted -DMS API called: no +DMS API called: yes diff --git a/acceptance/bundle/dms_fault/heartbeat/test.toml b/acceptance/bundle/dms_fault/heartbeat/test.toml new file mode 100644 index 00000000000..6dc2cc286c7 --- /dev/null +++ b/acceptance/bundle/dms_fault/heartbeat/test.toml @@ -0,0 +1,3 @@ +# Beat as often as possible so the faulted heartbeat endpoint is actually called: at the +# default 30s interval a local deploy never sends one. Also checks that the override reaches the CLI. +Env.DATABRICKS_BUNDLE_DMS_HEARTBEAT_INTERVAL_MS = "1"