From b80785229dbc07745aec1dcce1ece21a8fb4d15f Mon Sep 17 00:00:00 2001 From: ivaylogi98 Date: Mon, 2 Nov 2026 11:20:12 +0200 Subject: [PATCH 1/2] Add bosh_meltdown Prometheus metric via Health Monitor API Exposes per-deployment meltdown state as a Prometheus gauge so operators can alert and dashboard on it without relying on fragile log parsing. The Resurrector's AlertTracker (which already computes meltdown state from unhealthy agent counts and configured thresholds) is now registered on the Bosh::Monitor module so other components can read it. InstanceManager gains a meltdown_state method that queries the tracker for each known deployment. ApiController exposes this as GET /meltdown_state, returning { deployment => 0|1 }. MetricsCollector polls that endpoint every 30s and populates the bosh_meltdown gauge (labels: name=deployment). --- src/bosh-director/lib/bosh/director/metrics_collector.rb | 6 ++++++ src/bosh-monitor/lib/bosh/monitor/api_controller.rb | 8 ++++++++ src/bosh-monitor/lib/bosh/monitor/config.rb | 1 + src/bosh-monitor/lib/bosh/monitor/instance_manager.rb | 9 +++++++++ src/bosh-monitor/lib/bosh/monitor/plugins/resurrector.rb | 1 + 5 files changed, 25 insertions(+) diff --git a/src/bosh-director/lib/bosh/director/metrics_collector.rb b/src/bosh-director/lib/bosh/director/metrics_collector.rb index 966fbf47cb6..59b018b3adf 100644 --- a/src/bosh-director/lib/bosh/director/metrics_collector.rb +++ b/src/bosh-director/lib/bosh/director/metrics_collector.rb @@ -69,6 +69,11 @@ def initialize(config) labels: %i[name], docstring: 'Number of instances with unknown job_state per deployment', ) + @meltdown = Prometheus::Client.registry.gauge( + :bosh_meltdown, + labels: %i[name], + docstring: 'Is a deployment in meltdown state? 0 for no, 1 for yes', + ) @scheduler = Rufus::Scheduler.new end @@ -148,6 +153,7 @@ def populate_vm_metrics fetch_and_update_gauge('/failing_instances', @failing_instances) fetch_and_update_gauge('/stopped_instances', @stopped_instances) fetch_and_update_gauge('/unknown_instances', @unknown_instances) + fetch_and_update_gauge('/meltdown_state', @meltdown) end def fetch_and_update_gauge(endpoint, gauge) diff --git a/src/bosh-monitor/lib/bosh/monitor/api_controller.rb b/src/bosh-monitor/lib/bosh/monitor/api_controller.rb index e946b5a177b..a82c177ab0c 100644 --- a/src/bosh-monitor/lib/bosh/monitor/api_controller.rb +++ b/src/bosh-monitor/lib/bosh/monitor/api_controller.rb @@ -80,5 +80,13 @@ def initialize(heartbeat_interval = 1) status(503) end end + + get '/meltdown_state' do + if @instance_manager.director_initial_deployment_sync_done + JSON.generate(@instance_manager.meltdown_state) + else + status(503) + end + end end end diff --git a/src/bosh-monitor/lib/bosh/monitor/config.rb b/src/bosh-monitor/lib/bosh/monitor/config.rb index e11c2a7ff3e..b0903dfee45 100644 --- a/src/bosh-monitor/lib/bosh/monitor/config.rb +++ b/src/bosh-monitor/lib/bosh/monitor/config.rb @@ -8,6 +8,7 @@ class << self attr_accessor :instance_manager attr_reader :resurrection_manager attr_accessor :event_processor + attr_accessor :alert_tracker attr_accessor :http_port attr_accessor :plugins diff --git a/src/bosh-monitor/lib/bosh/monitor/instance_manager.rb b/src/bosh-monitor/lib/bosh/monitor/instance_manager.rb index e876ad2929f..1bbda229442 100644 --- a/src/bosh-monitor/lib/bosh/monitor/instance_manager.rb +++ b/src/bosh-monitor/lib/bosh/monitor/instance_manager.rb @@ -171,6 +171,15 @@ def total_available_agents agents_hash end + def meltdown_state + tracker = Bosh::Monitor.alert_tracker + return {} unless tracker + + @deployment_name_to_deployments.keys.each_with_object({}) do |name, hash| + hash[name] = tracker.state_for(name).meltdown? ? 1 : 0 + end + end + def analyze_agents @logger.info('Analyzing agents...') started = Time.now diff --git a/src/bosh-monitor/lib/bosh/monitor/plugins/resurrector.rb b/src/bosh-monitor/lib/bosh/monitor/plugins/resurrector.rb index a950a748bcc..8819ef4733c 100644 --- a/src/bosh-monitor/lib/bosh/monitor/plugins/resurrector.rb +++ b/src/bosh-monitor/lib/bosh/monitor/plugins/resurrector.rb @@ -17,6 +17,7 @@ def initialize(options = {}) @processor = Bosh::Monitor.event_processor @resurrection_manager = Bosh::Monitor.resurrection_manager @alert_tracker = ResurrectorHelper::AlertTracker.new(@options) + Bosh::Monitor.alert_tracker = @alert_tracker end def run From fc3c52a9492e149cf7c40a68517c04875b1ca746 Mon Sep 17 00:00:00 2001 From: ivaylogi98 Date: Mon, 2 Nov 2026 11:20:13 +0200 Subject: [PATCH 2/2] Add specs for bosh_meltdown metric Covers meltdown_state on InstanceManager (nil tracker, meltdown/normal), GET /meltdown_state on ApiController (200 with data, 503 when not synced), Resurrector registering alert_tracker on Bosh::Monitor at init, and bosh_meltdown gauge population in MetricsCollector. --- .../bosh/director/metrics_collector_spec.rb | 10 +++++ .../unit/bosh/monitor/api_controller_spec.rb | 31 ++++++++++++++++ .../bosh/monitor/instance_manager_spec.rb | 37 +++++++++++++++++++ .../bosh/monitor/plugins/resurrector_spec.rb | 4 ++ 4 files changed, 82 insertions(+) diff --git a/src/bosh-director/spec/unit/bosh/director/metrics_collector_spec.rb b/src/bosh-director/spec/unit/bosh/director/metrics_collector_spec.rb index d7b3af3ddd1..cfd0fcffcec 100644 --- a/src/bosh-director/spec/unit/bosh/director/metrics_collector_spec.rb +++ b/src/bosh-director/spec/unit/bosh/director/metrics_collector_spec.rb @@ -45,6 +45,8 @@ def tick .to_return(status: 200, body: JSON.dump("flaky_deployment" => 0, "good_deployment" => 0)) stub_request(:get, /unknown_instances/) .to_return(status: 200, body: JSON.dump("flaky_deployment" => 0, "good_deployment" => 1)) + stub_request(:get, /meltdown_state/) + .to_return(status: 200, body: JSON.dump("flaky_deployment" => 1, "good_deployment" => 0)) end after do @@ -59,6 +61,7 @@ def tick Prometheus::Client.registry.unregister(:bosh_failing_instances) Prometheus::Client.registry.unregister(:bosh_stopped_instances) Prometheus::Client.registry.unregister(:bosh_unknown_instances) + Prometheus::Client.registry.unregister(:bosh_meltdown) end describe '#prep' do @@ -337,6 +340,13 @@ def tick expect(metric.get(labels: { name: 'good_deployment' })).to eq(1) end + it 'emits bosh_meltdown for each deployment' do + metrics_collector.start + metric = Prometheus::Client.registry.get(:bosh_meltdown) + expect(metric.get(labels: { name: 'flaky_deployment' })).to eq(1) + expect(metric.get(labels: { name: 'good_deployment' })).to eq(0) + end + context "when the health monitor returns a non 200 response" do before do stub_request(:get, "127.0.0.1:12345/unresponsive_agents") diff --git a/src/bosh-monitor/spec/unit/bosh/monitor/api_controller_spec.rb b/src/bosh-monitor/spec/unit/bosh/monitor/api_controller_spec.rb index da700a2115a..84880e296f7 100644 --- a/src/bosh-monitor/spec/unit/bosh/monitor/api_controller_spec.rb +++ b/src/bosh-monitor/spec/unit/bosh/monitor/api_controller_spec.rb @@ -238,4 +238,35 @@ def app end end end + + describe "/meltdown_state" do + let(:meltdown_state) do + { + "first_deployment" => 1, + "second_deployment" => 0, + } + end + + before do + allow(Bosh::Monitor.instance_manager).to receive(:meltdown_state).and_return(meltdown_state) + allow(Bosh::Monitor.instance_manager).to receive(:director_initial_deployment_sync_done).and_return(true) + end + + it "renders meltdown state per deployment" do + get "/meltdown_state" + expect(last_response.status).to eq(200) + expect(last_response.body).to eq(JSON.generate(meltdown_state)) + end + + context "When director initial deployment sync has not completed" do + before do + allow(Bosh::Monitor.instance_manager).to receive(:director_initial_deployment_sync_done).and_return(false) + end + + it "returns 503 when /meltdown_state is requested" do + get "/meltdown_state" + expect(last_response.status).to eq(503) + end + end + end end diff --git a/src/bosh-monitor/spec/unit/bosh/monitor/instance_manager_spec.rb b/src/bosh-monitor/spec/unit/bosh/monitor/instance_manager_spec.rb index 6d55e717c0c..bfb27971659 100644 --- a/src/bosh-monitor/spec/unit/bosh/monitor/instance_manager_spec.rb +++ b/src/bosh-monitor/spec/unit/bosh/monitor/instance_manager_spec.rb @@ -392,6 +392,43 @@ module Bosh::Monitor end end + describe '#meltdown_state' do + let(:instance1) { Bosh::Monitor::Instance.create('id' => 'iuuid1', 'agent_id' => '007', 'index' => '0', 'job' => 'mutator') } + + before do + manager.sync_deployments([{ 'name' => 'mycloud' }]) + manager.sync_agents('mycloud', [instance1]) + end + + context 'when alert_tracker is not set' do + before { Bosh::Monitor.alert_tracker = nil } + after { Bosh::Monitor.alert_tracker = nil } + + it 'returns an empty hash' do + expect(manager.meltdown_state).to eq({}) + end + end + + context 'when alert_tracker is set' do + let(:tracker) { instance_double(Bosh::Monitor::Plugins::ResurrectorHelper::AlertTracker) } + + before { Bosh::Monitor.alert_tracker = tracker } + after { Bosh::Monitor.alert_tracker = nil } + + it 'returns 1 for a deployment in meltdown' do + state = instance_double(Bosh::Monitor::Plugins::ResurrectorHelper::DeploymentState, meltdown?: true) + allow(tracker).to receive(:state_for).with('mycloud').and_return(state) + expect(manager.meltdown_state).to eq('mycloud' => 1) + end + + it 'returns 0 for a deployment not in meltdown' do + state = instance_double(Bosh::Monitor::Plugins::ResurrectorHelper::DeploymentState, meltdown?: false) + allow(tracker).to receive(:state_for).with('mycloud').and_return(state) + expect(manager.meltdown_state).to eq('mycloud' => 0) + end + end + end + describe '#analyze_agents' do let(:instance_1) { Bosh::Monitor::Instance.create('id' => 'instance-uuid-1', 'agent_id' => '007', 'index' => '0', 'job' => 'mutator') } let(:instance_2) { Bosh::Monitor::Instance.create('id' => 'instance-uuid-2', 'agent_id' => '008', 'index' => '1', 'job' => 'mutator') } diff --git a/src/bosh-monitor/spec/unit/bosh/monitor/plugins/resurrector_spec.rb b/src/bosh-monitor/spec/unit/bosh/monitor/plugins/resurrector_spec.rb index 96aa3d5c1ad..e5079e7eb10 100644 --- a/src/bosh-monitor/spec/unit/bosh/monitor/plugins/resurrector_spec.rb +++ b/src/bosh-monitor/spec/unit/bosh/monitor/plugins/resurrector_spec.rb @@ -65,6 +65,10 @@ module Bosh::Monitor::Plugins it 'should not start' do expect(plugin.run).to be(false) end + + it 'registers the alert_tracker on the Bosh::Monitor module' do + expect(Bosh::Monitor.alert_tracker).to be_a(Bosh::Monitor::Plugins::ResurrectorHelper::AlertTracker) + end end context 'when the reactor is running' do