diff --git a/acceptance/bundle/dms/multiple-resources/output.txt b/acceptance/bundle/dms/multiple-resources/output.txt index 80487e6f218..ade8ec21057 100644 --- a/acceptance/bundle/dms/multiple-resources/output.txt +++ b/acceptance/bundle/dms/multiple-resources/output.txt @@ -25,6 +25,14 @@ Resources: 5 created, 0 changed, 0 deleted, 0 unchanged "queue_size_in_range": true } +=== Drain telemetry measures foreground wait, including zero milliseconds + +>>> cat [OUTPUT_DIR]/out.requests.txt +{ + "measurement_count": 1, + "duration_is_nonnegative": true +} + >>> print_requests.py --dms //versions/1/operations --sort --del-body state --oneline {"method": "PATCH", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/1/operations/jobs.five", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[FIVE_ID]", "sequence_id": 0, "status": "OPERATION_STATUS_SUCCEEDED"}} {"method": "PATCH", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/1/operations/jobs.four", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[FOUR_ID]", "sequence_id": 0, "status": "OPERATION_STATUS_SUCCEEDED"}} @@ -196,6 +204,11 @@ Resources: 0 created, 0 changed, 0 deleted, 5 unchanged >>> cat [OUTPUT_DIR]/out.requests.txt false +=== A deploy with no buffer omits drain timing + +>>> cat [OUTPUT_DIR]/out.requests.txt +false + >>> print_requests.py --dms //api/2.0/bundle --sort --oneline === The plan records the deployment_history feature diff --git a/acceptance/bundle/dms/multiple-resources/script b/acceptance/bundle/dms/multiple-resources/script index 55e21494154..172d495eb3c 100644 --- a/acceptance/bundle/dms/multiple-resources/script +++ b/acceptance/bundle/dms/multiple-resources/script @@ -5,6 +5,8 @@ trace $CLI bundle deploy title "History telemetry survives version completion and counts all five staged operations" trace cat "$OUT_REQUESTS" | jq 'select(.path? == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.bundle_deploy_event.deployment_history | {staged_operation_count, operation_recording_success_count, operation_recording_failure_count, operation_recording_pending_count, queued_update_count_at_timeout, recording_attempt_failure_count, recording_attempt_success_count, queue_size_in_range: (.peak_pending_update_count >= 1 and .peak_pending_update_count <= 5)}' replace_ids.py +title "Drain telemetry measures foreground wait, including zero milliseconds" +trace cat "$OUT_REQUESTS" | jq 'select(.path? == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.bundle_deploy_event.experimental.bundle_mutator_execution_time_ms | map(select(.key == "dms.operation_buffer_drain")) | {measurement_count: length, duration_is_nonnegative: (.[0].value >= 0)}' trace print_requests.py --dms //versions/1/operations --sort --del-body state --oneline title "The plan records the deployment_history feature" @@ -22,6 +24,8 @@ title "Redeploy with no changes: nothing is applied, so no version is created at trace $CLI bundle deploy title "A deploy with no version omits operation buffer telemetry" trace cat "$OUT_REQUESTS" | jq 'select(.path? == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.bundle_deploy_event | has("deployment_history")' +title "A deploy with no buffer omits drain timing" +trace cat "$OUT_REQUESTS" | jq 'select(.path? == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.bundle_deploy_event.experimental.bundle_mutator_execution_time_ms | any(.key == "dms.operation_buffer_drain")' trace print_requests.py --dms //api/2.0/bundle --sort --oneline title "The plan records the deployment_history feature" diff --git a/bundle/metrics/metrics.go b/bundle/metrics/metrics.go index b77e795a232..21ed4db2b77 100644 --- a/bundle/metrics/metrics.go +++ b/bundle/metrics/metrics.go @@ -25,6 +25,10 @@ const ( // not a mutator. SnapshotUploadDuration = "snapshot.upload" + // Foreground wait in the deployment history buffer's Drain calls, in + // bundle_mutator_execution_time_ms. Background upload time is excluded. + DeploymentHistoryDrainDuration = "dms.operation_buffer_drain" + // Outcome of an automatic migration of terraform state to the direct engine. // - migrate_error: state conversion itself errored. // - migrate_commit_error: the state was converted, but committing it diff --git a/bundle/phases/telemetry.go b/bundle/phases/telemetry.go index 723bcd9346f..3d7fde1e379 100644 --- a/bundle/phases/telemetry.go +++ b/bundle/phases/telemetry.go @@ -428,6 +428,7 @@ func LogDeployTelemetry(ctx context.Context, b *bundle.Bundle, errMsg string) { uploadFileSizers[i] = f } + executionTimes := getExecutionTimes(b) var deploymentHistory *protos.BundleDeploymentHistory if stats := b.DeploymentBundle.StateDB.OperationBufferStats(); stats != nil { deploymentHistory = &protos.BundleDeploymentHistory{ @@ -440,6 +441,14 @@ func LogDeployTelemetry(ctx context.Context, b *bundle.Bundle, errMsg string) { OperationRecordingFailureCount: int64(stats.OperationRecordingFailureCount), OperationRecordingPendingCount: int64(stats.OperationRecordingPendingCount), } + if stats.DrainCount > 0 { + // Preserve zero waits and keep this measurement outside the top-250 + // mutator limit, so a missing entry means Drain was not called. + executionTimes = append(executionTimes, protos.IntMapEntry{ + Key: metrics.DeploymentHistoryDrainDuration, + Value: stats.DrainDuration.Milliseconds(), + }) + } } telemetry.Log(ctx, protos.DatabricksCliLog{ @@ -488,7 +497,7 @@ func LogDeployTelemetry(ctx context.Context, b *bundle.Bundle, errMsg string) { VariableCount: int64(variableCount), ComplexVariableCount: complexVariableCount, LookupVariableCount: lookupVariableCount, - BundleMutatorExecutionTimeMs: getExecutionTimes(b), + BundleMutatorExecutionTimeMs: executionTimes, }, }, }) diff --git a/libs/dms/operation_buffer.go b/libs/dms/operation_buffer.go index 9c13f543908..a1c8b34025e 100644 --- a/libs/dms/operation_buffer.go +++ b/libs/dms/operation_buffer.go @@ -40,6 +40,9 @@ type BufferOptions struct { // OperationBufferStats describes one deployment version's history uploads. type OperationBufferStats struct { + // Foreground time spent waiting in Drain, summed across calls. Background uploads are excluded. + DrainDuration time.Duration + DrainCount int StagedOperationCount int PeakPendingUpdateCount int QueuedUpdateCountAtTimeout int @@ -278,6 +281,15 @@ func (b *OperationBuffer) write(ctx context.Context, key string, entry *buffered // Shadow history cancels outstanding uploads when the timeout or ctx expires. // It returns the first recording error and is safe to call more than once. func (b *OperationBuffer) Drain(ctx context.Context) error { + start := time.Now() + defer func() { + elapsed := time.Since(start) + b.mu.Lock() + defer b.mu.Unlock() + b.stats.DrainDuration += elapsed + b.stats.DrainCount++ + }() + b.stopQueue() if !b.bestEffort { // DMS is authoritative: wait for all queued writes and return any recording error. diff --git a/libs/dms/operation_buffer_test.go b/libs/dms/operation_buffer_test.go index 6aa6aea4943..47989f9984b 100644 --- a/libs/dms/operation_buffer_test.go +++ b/libs/dms/operation_buffer_test.go @@ -86,6 +86,35 @@ func TestDrainIsIdempotent(t *testing.T) { require.NoError(t, b.Drain(t.Context())) } +func TestDrainMeasuresForegroundWait(t *testing.T) { + for _, bestEffort := range []bool{false, true} { + t.Run(fmt.Sprintf("best_effort=%t", bestEffort), func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + b := queued() + b.bestEffort = bestEffort + b.drainTimeout = 10 * time.Second + b.done = make(chan struct{}) + b.stopQueue = sync.OnceFunc(func() { close(b.queue) }) + go func() { + time.Sleep(3 * time.Second) + close(b.done) + }() + + // Uploads overlap one second of deployment before Drain starts. + time.Sleep(time.Second) + require.NoError(t, b.Drain(t.Context())) + assert.Equal(t, 2*time.Second, b.Stats().DrainDuration) + assert.Equal(t, 1, b.Stats().DrainCount) + + // Draining an already-finished worker adds an explicit zero wait. + require.NoError(t, b.Drain(t.Context())) + assert.Equal(t, 2*time.Second, b.Stats().DrainDuration) + assert.Equal(t, 2, b.Stats().DrainCount) + }) + }) + } +} + func TestRecordFailureKeepsTheStateTheLastWriteRecorded(t *testing.T) { s := queued() @@ -165,6 +194,8 @@ func TestHistoryDrainCancelsStalledWriter(t *testing.T) { start := time.Now() assert.ErrorIs(t, b.Drain(t.Context()), context.DeadlineExceeded) assert.Equal(t, time.Second, time.Since(start)) + assert.Equal(t, time.Second, b.Stats().DrainDuration) + assert.Equal(t, 1, b.Stats().DrainCount) // Drain must also call cancel(), stopping any upload the writer would be doing. assert.ErrorIs(t, ctx.Err(), context.Canceled) })