Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions acceptance/bundle/dms/multiple-resources/output.txt
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,14 @@ Resources: 5 created, 0 changed, 0 deleted, 0 unchanged
"queue_size_in_range": true
}

=== Drain telemetry measures foreground wait, including zero milliseconds

>>> cat [OUTPUT_DIR]/out.requests.txt
{
"measurement_count": 1,
"duration_is_nonnegative": true
}

>>> print_requests.py --dms //versions/1/operations --sort --del-body state --oneline
{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/1/operations/jobs.five", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[FIVE_ID]", "sequence_id": 0, "status": "OPERATION_STATUS_SUCCEEDED"}}
{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/1/operations/jobs.four", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[FOUR_ID]", "sequence_id": 0, "status": "OPERATION_STATUS_SUCCEEDED"}}
Expand Down Expand Up @@ -196,6 +204,11 @@ Resources: 0 created, 0 changed, 0 deleted, 5 unchanged
>>> cat [OUTPUT_DIR]/out.requests.txt
false

=== A deploy with no buffer omits drain timing

>>> cat [OUTPUT_DIR]/out.requests.txt
false

>>> print_requests.py --dms //api/2.0/bundle --sort --oneline

=== The plan records the deployment_history feature
Expand Down
4 changes: 4 additions & 0 deletions acceptance/bundle/dms/multiple-resources/script
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ trace $CLI bundle deploy
title "History telemetry survives version completion and counts all five staged operations"
trace cat "$OUT_REQUESTS" | jq 'select(.path? == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.bundle_deploy_event.deployment_history | {staged_operation_count, operation_recording_success_count, operation_recording_failure_count, operation_recording_pending_count, queued_update_count_at_timeout, recording_attempt_failure_count, recording_attempt_success_count, queue_size_in_range: (.peak_pending_update_count >= 1 and .peak_pending_update_count <= 5)}'
replace_ids.py
title "Drain telemetry measures foreground wait, including zero milliseconds"
trace cat "$OUT_REQUESTS" | jq 'select(.path? == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.bundle_deploy_event.experimental.bundle_mutator_execution_time_ms | map(select(.key == "dms.operation_buffer_drain")) | {measurement_count: length, duration_is_nonnegative: (.[0].value >= 0)}'
trace print_requests.py --dms //versions/1/operations --sort --del-body state --oneline

title "The plan records the deployment_history feature"
Expand All @@ -22,6 +24,8 @@ title "Redeploy with no changes: nothing is applied, so no version is created at
trace $CLI bundle deploy
title "A deploy with no version omits operation buffer telemetry"
trace cat "$OUT_REQUESTS" | jq 'select(.path? == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.bundle_deploy_event | has("deployment_history")'
title "A deploy with no buffer omits drain timing"
trace cat "$OUT_REQUESTS" | jq 'select(.path? == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.bundle_deploy_event.experimental.bundle_mutator_execution_time_ms | any(.key == "dms.operation_buffer_drain")'
trace print_requests.py --dms //api/2.0/bundle --sort --oneline

title "The plan records the deployment_history feature"
Expand Down
4 changes: 4 additions & 0 deletions bundle/metrics/metrics.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,10 @@ const (
// not a mutator.
SnapshotUploadDuration = "snapshot.upload"

// Foreground wait in the deployment history buffer's Drain calls, in
// bundle_mutator_execution_time_ms. Background upload time is excluded.
DeploymentHistoryDrainDuration = "dms.operation_buffer_drain"

// Outcome of an automatic migration of terraform state to the direct engine.
// - migrate_error: state conversion itself errored.
// - migrate_commit_error: the state was converted, but committing it
Expand Down
11 changes: 10 additions & 1 deletion bundle/phases/telemetry.go
Original file line number Diff line number Diff line change
Expand Up @@ -428,6 +428,7 @@ func LogDeployTelemetry(ctx context.Context, b *bundle.Bundle, errMsg string) {
uploadFileSizers[i] = f
}

executionTimes := getExecutionTimes(b)
var deploymentHistory *protos.BundleDeploymentHistory
if stats := b.DeploymentBundle.StateDB.OperationBufferStats(); stats != nil {
deploymentHistory = &protos.BundleDeploymentHistory{
Expand All @@ -440,6 +441,14 @@ func LogDeployTelemetry(ctx context.Context, b *bundle.Bundle, errMsg string) {
OperationRecordingFailureCount: int64(stats.OperationRecordingFailureCount),
OperationRecordingPendingCount: int64(stats.OperationRecordingPendingCount),
}
if stats.DrainCount > 0 {
// Preserve zero waits and keep this measurement outside the top-250
// mutator limit, so a missing entry means Drain was not called.
executionTimes = append(executionTimes, protos.IntMapEntry{
Key: metrics.DeploymentHistoryDrainDuration,
Value: stats.DrainDuration.Milliseconds(),
})
}
}

telemetry.Log(ctx, protos.DatabricksCliLog{
Expand Down Expand Up @@ -488,7 +497,7 @@ func LogDeployTelemetry(ctx context.Context, b *bundle.Bundle, errMsg string) {
VariableCount: int64(variableCount),
ComplexVariableCount: complexVariableCount,
LookupVariableCount: lookupVariableCount,
BundleMutatorExecutionTimeMs: getExecutionTimes(b),
BundleMutatorExecutionTimeMs: executionTimes,
},
},
})
Expand Down
12 changes: 12 additions & 0 deletions libs/dms/operation_buffer.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,9 @@ type BufferOptions struct {

// OperationBufferStats describes one deployment version's history uploads.
type OperationBufferStats struct {
// Foreground time spent waiting in Drain, summed across calls. Background uploads are excluded.
DrainDuration time.Duration
DrainCount int
StagedOperationCount int
PeakPendingUpdateCount int
QueuedUpdateCountAtTimeout int
Expand Down Expand Up @@ -278,6 +281,15 @@ func (b *OperationBuffer) write(ctx context.Context, key string, entry *buffered
// Shadow history cancels outstanding uploads when the timeout or ctx expires.
// It returns the first recording error and is safe to call more than once.
func (b *OperationBuffer) Drain(ctx context.Context) error {
start := time.Now()
defer func() {
elapsed := time.Since(start)
b.mu.Lock()
defer b.mu.Unlock()
b.stats.DrainDuration += elapsed
b.stats.DrainCount++
}()

b.stopQueue()
if !b.bestEffort {
// DMS is authoritative: wait for all queued writes and return any recording error.
Expand Down
31 changes: 31 additions & 0 deletions libs/dms/operation_buffer_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,35 @@ func TestDrainIsIdempotent(t *testing.T) {
require.NoError(t, b.Drain(t.Context()))
}

func TestDrainMeasuresForegroundWait(t *testing.T) {
for _, bestEffort := range []bool{false, true} {
t.Run(fmt.Sprintf("best_effort=%t", bestEffort), func(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
b := queued()
b.bestEffort = bestEffort
b.drainTimeout = 10 * time.Second
b.done = make(chan struct{})
b.stopQueue = sync.OnceFunc(func() { close(b.queue) })
go func() {
time.Sleep(3 * time.Second)
close(b.done)
}()

// Uploads overlap one second of deployment before Drain starts.
time.Sleep(time.Second)
require.NoError(t, b.Drain(t.Context()))
assert.Equal(t, 2*time.Second, b.Stats().DrainDuration)
assert.Equal(t, 1, b.Stats().DrainCount)

// Draining an already-finished worker adds an explicit zero wait.
require.NoError(t, b.Drain(t.Context()))
assert.Equal(t, 2*time.Second, b.Stats().DrainDuration)
assert.Equal(t, 2, b.Stats().DrainCount)
})
})
}
}

func TestRecordFailureKeepsTheStateTheLastWriteRecorded(t *testing.T) {
s := queued()

Expand Down Expand Up @@ -165,6 +194,8 @@ func TestHistoryDrainCancelsStalledWriter(t *testing.T) {
start := time.Now()
assert.ErrorIs(t, b.Drain(t.Context()), context.DeadlineExceeded)
assert.Equal(t, time.Second, time.Since(start))
assert.Equal(t, time.Second, b.Stats().DrainDuration)
assert.Equal(t, 1, b.Stats().DrainCount)
// Drain must also call cancel(), stopping any upload the writer would be doing.
assert.ErrorIs(t, ctx.Err(), context.Canceled)
})
Expand Down
Loading