Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions acceptance/bundle/dms/test.toml
Original file line number Diff line number Diff line change
Expand Up @@ -15,3 +15,7 @@ Ignore = [
# rejection path. Pinned to "true" (not the suite's ["", "true"]) since the field already
# records - the empty variant would just duplicate it.
EnvMatrix.DMS = ["true"]

# The CLI sends a heartbeat after 30s without a state write, so a slow cloud deploy would
# record one at a random point. 1h never fires within a test.
Env.DATABRICKS_BUNDLE_DMS_HEARTBEAT_INTERVAL_MS = "3600000"
Comment thread
denik marked this conversation as resolved.
Resolved
2 changes: 1 addition & 1 deletion acceptance/bundle/dms_fault/heartbeat/output.txt
Original file line number Diff line number Diff line change
Expand Up @@ -141,4 +141,4 @@

Destroy: 2 deleted

DMS API called: no
DMS API called: yes
3 changes: 3 additions & 0 deletions acceptance/bundle/dms_fault/heartbeat/test.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
# Beat as often as possible so the faulted heartbeat endpoint is actually called: at the
# default 30s interval a local deploy never sends one. Also checks that the override reaches the CLI.
Env.DATABRICKS_BUNDLE_DMS_HEARTBEAT_INTERVAL_MS = "1"
25 changes: 20 additions & 5 deletions libs/dms/heartbeat.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,16 +2,31 @@ package dms

import (
"context"
"strconv"
"time"

"github.com/databricks/cli/libs/cmdctx"
"github.com/databricks/cli/libs/env"
"github.com/databricks/cli/libs/log"
"github.com/databricks/databricks-sdk-go/service/bundledeployments"
)

// heartbeatInterval is the maximum time between consecutive updates.
const heartbeatInterval = 30 * time.Second

// HeartbeatIntervalMsVariable overrides heartbeatInterval in milliseconds. Acceptance tests that
// record requests set it high so a slow cloud deploy does not add a heartbeat at a random point.
const HeartbeatIntervalMsVariable = "DATABRICKS_BUNDLE_DMS_HEARTBEAT_INTERVAL_MS"

// interval returns heartbeatInterval, or the override if it is set to a positive number.
func interval(ctx context.Context) time.Duration {
ms, err := strconv.Atoi(env.Get(ctx, HeartbeatIntervalMsVariable))
if err != nil || ms <= 0 {
return heartbeatInterval
}
return time.Duration(ms) * time.Millisecond
}

// Heartbeat updates the update_time of an open deployment version. This is also updated by
// CreateVersion and every successful write.
type Heartbeat struct {
Expand All @@ -32,12 +47,12 @@ func startVersionHeartbeat(ctx context.Context, deploymentID string, versionNum
func startHeartbeat(ctx context.Context, send func(context.Context) error) *Heartbeat {
ctx, cancel := context.WithCancel(ctx)
h := &Heartbeat{renewed: make(chan struct{}, 1), cancel: cancel}
go h.run(ctx, send)
go h.run(ctx, send, interval(ctx))
return h
}

func (h *Heartbeat) run(ctx context.Context, send func(context.Context) error) {
timer := time.NewTimer(heartbeatInterval)
func (h *Heartbeat) run(ctx context.Context, send func(context.Context) error, interval time.Duration) {
timer := time.NewTimer(interval)
for {
select {
case <-ctx.Done():
Expand All @@ -52,9 +67,9 @@ func (h *Heartbeat) run(ctx context.Context, send func(context.Context) error) {
if ctx.Err() != nil {
return
}
log.Infof(ctx, "Failed to send deployment heartbeat, so the deployment may show as interrupted; retrying in %s: %s", heartbeatInterval, err)
log.Infof(ctx, "Failed to send deployment heartbeat, so the deployment may show as interrupted; retrying in %s: %s", interval, err)
}
timer.Reset(heartbeatInterval)
timer.Reset(interval)
}
}

Expand Down
25 changes: 25 additions & 0 deletions libs/dms/heartbeat_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"testing/synctest"
"time"

"github.com/databricks/cli/libs/env"
"github.com/stretchr/testify/assert"
)

Expand Down Expand Up @@ -92,3 +93,27 @@ func TestHeartbeatStop(t *testing.T) {
assert.Equal(t, int32(0), f.calls.Load())
})
}

func TestHeartbeatIntervalOverride(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
f := &fakeSend{}
ctx := env.Set(t.Context(), HeartbeatIntervalMsVariable, "100000")
h := startHeartbeat(ctx, f.send)
defer h.Stop()

time.Sleep(99 * time.Second)
synctest.Wait()
assert.Equal(t, int32(0), f.calls.Load())

time.Sleep(2 * time.Second)
synctest.Wait()
assert.Equal(t, int32(1), f.calls.Load())
})
}

func TestHeartbeatIntervalInvalidOverride(t *testing.T) {
for _, v := range []string{"", "abc", "0", "-5"} {
ctx := env.Set(t.Context(), HeartbeatIntervalMsVariable, v)
assert.Equal(t, heartbeatInterval, interval(ctx), v)
}
}
Loading