Expose bosh_meltdown Prometheus metric via Health Monitor API

src/bosh-director/lib/bosh/director/metrics_collector.rb

@@ -69,6 +69,11 @@ def initialize(config)
           labels: %i[name],
           docstring: 'Number of instances with unknown job_state per deployment',
         )
+        @meltdown = Prometheus::Client.registry.gauge(
+          :bosh_meltdown,
+          labels: %i[name],
+          docstring: 'Is a deployment in meltdown state? 0 for no, 1 for yes',
+        )
         @scheduler = Rufus::Scheduler.new
       end
 
@@ -148,6 +153,7 @@ def populate_vm_metrics
         fetch_and_update_gauge('/failing_instances', @failing_instances)
         fetch_and_update_gauge('/stopped_instances', @stopped_instances)
         fetch_and_update_gauge('/unknown_instances', @unknown_instances)
+        fetch_and_update_gauge('/meltdown_state', @meltdown)
       end
 
       def fetch_and_update_gauge(endpoint, gauge)

src/bosh-director/spec/unit/bosh/director/metrics_collector_spec.rb

@@ -45,6 +45,8 @@ def tick
         .to_return(status: 200, body: JSON.dump("flaky_deployment" => 0, "good_deployment" => 0))
       stub_request(:get, /unknown_instances/)
         .to_return(status: 200, body: JSON.dump("flaky_deployment" => 0, "good_deployment" => 1))
+      stub_request(:get, /meltdown_state/)
+        .to_return(status: 200, body: JSON.dump("flaky_deployment" => 1, "good_deployment" => 0))
     end
 
     after do
@@ -59,6 +61,7 @@ def tick
       Prometheus::Client.registry.unregister(:bosh_failing_instances)
       Prometheus::Client.registry.unregister(:bosh_stopped_instances)
       Prometheus::Client.registry.unregister(:bosh_unknown_instances)
+      Prometheus::Client.registry.unregister(:bosh_meltdown)
     end
 
     describe '#prep' do
@@ -337,6 +340,13 @@ def tick
           expect(metric.get(labels: { name: 'good_deployment' })).to eq(1)
         end
 
+        it 'emits bosh_meltdown for each deployment' do
+          metrics_collector.start
+          metric = Prometheus::Client.registry.get(:bosh_meltdown)
+          expect(metric.get(labels: { name: 'flaky_deployment' })).to eq(1)
+          expect(metric.get(labels: { name: 'good_deployment' })).to eq(0)
+        end
+
         context "when the health monitor returns a non 200 response" do
           before do
             stub_request(:get, "127.0.0.1:12345/unresponsive_agents")

src/bosh-monitor/lib/bosh/monitor/api_controller.rb

@@ -80,5 +80,13 @@ def initialize(heartbeat_interval = 1)
         status(503)
       end
     end
+
+    get '/meltdown_state' do
+      if @instance_manager.director_initial_deployment_sync_done
+        JSON.generate(@instance_manager.meltdown_state)
+      else
+        status(503)
+      end
+    end
   end
 end

src/bosh-monitor/lib/bosh/monitor/config.rb

@@ -8,6 +8,7 @@ class << self
     attr_accessor :instance_manager
     attr_reader :resurrection_manager
     attr_accessor :event_processor
+    attr_accessor :alert_tracker
 
     attr_accessor :http_port
     attr_accessor :plugins

src/bosh-monitor/lib/bosh/monitor/instance_manager.rb

@@ -171,6 +171,15 @@ def total_available_agents
       agents_hash
     end
 
+    def meltdown_state
+      tracker = Bosh::Monitor.alert_tracker
+      return {} unless tracker
+
+      @deployment_name_to_deployments.keys.each_with_object({}) do |name, hash|
+        hash[name] = tracker.state_for(name).meltdown? ? 1 : 0
+      end
+    end
+
     def analyze_agents
       @logger.info('Analyzing agents...')
       started = Time.now

src/bosh-monitor/lib/bosh/monitor/plugins/resurrector.rb

@@ -17,6 +17,7 @@ def initialize(options = {})
         @processor            = Bosh::Monitor.event_processor
         @resurrection_manager = Bosh::Monitor.resurrection_manager
         @alert_tracker        = ResurrectorHelper::AlertTracker.new(@options)
+        Bosh::Monitor.alert_tracker = @alert_tracker
       end
 
       def run

src/bosh-monitor/spec/unit/bosh/monitor/api_controller_spec.rb

@@ -238,4 +238,35 @@ def app
       end
     end
   end
+
+  describe "/meltdown_state" do
+    let(:meltdown_state) do
+      {
+        "first_deployment" => 1,
+        "second_deployment" => 0,
+      }
+    end
+
+    before do
+      allow(Bosh::Monitor.instance_manager).to receive(:meltdown_state).and_return(meltdown_state)
+      allow(Bosh::Monitor.instance_manager).to receive(:director_initial_deployment_sync_done).and_return(true)
+    end
+
+    it "renders meltdown state per deployment" do
+      get "/meltdown_state"
+      expect(last_response.status).to eq(200)
+      expect(last_response.body).to eq(JSON.generate(meltdown_state))
+    end
+
+    context "When director initial deployment sync has not completed" do
+      before do
+        allow(Bosh::Monitor.instance_manager).to receive(:director_initial_deployment_sync_done).and_return(false)
+      end
+
+      it "returns 503 when /meltdown_state is requested" do
+        get "/meltdown_state"
+        expect(last_response.status).to eq(503)
+      end
+    end
+  end
 end

src/bosh-monitor/spec/unit/bosh/monitor/instance_manager_spec.rb

@@ -392,6 +392,43 @@ module Bosh::Monitor
         end
       end
 
+      describe '#meltdown_state' do
+        let(:instance1) { Bosh::Monitor::Instance.create('id' => 'iuuid1', 'agent_id' => '007', 'index' => '0', 'job' => 'mutator') }
+
+        before do
+          manager.sync_deployments([{ 'name' => 'mycloud' }])
+          manager.sync_agents('mycloud', [instance1])
+        end
+
+        context 'when alert_tracker is not set' do
+          before { Bosh::Monitor.alert_tracker = nil }
+          after  { Bosh::Monitor.alert_tracker = nil }
+
+          it 'returns an empty hash' do
+            expect(manager.meltdown_state).to eq({})
+          end
+        end
+
+        context 'when alert_tracker is set' do
+          let(:tracker) { instance_double(Bosh::Monitor::Plugins::ResurrectorHelper::AlertTracker) }
+
+          before { Bosh::Monitor.alert_tracker = tracker }
+          after  { Bosh::Monitor.alert_tracker = nil }
+
+          it 'returns 1 for a deployment in meltdown' do
+            state = instance_double(Bosh::Monitor::Plugins::ResurrectorHelper::DeploymentState, meltdown?: true)
+            allow(tracker).to receive(:state_for).with('mycloud').and_return(state)
+            expect(manager.meltdown_state).to eq('mycloud' => 1)
+          end
+
+          it 'returns 0 for a deployment not in meltdown' do
+            state = instance_double(Bosh::Monitor::Plugins::ResurrectorHelper::DeploymentState, meltdown?: false)
+            allow(tracker).to receive(:state_for).with('mycloud').and_return(state)
+            expect(manager.meltdown_state).to eq('mycloud' => 0)
+          end
+        end
+      end
+
       describe '#analyze_agents' do
         let(:instance_1) { Bosh::Monitor::Instance.create('id' => 'instance-uuid-1', 'agent_id' => '007', 'index' => '0', 'job' => 'mutator') }
         let(:instance_2) { Bosh::Monitor::Instance.create('id' => 'instance-uuid-2', 'agent_id' => '008', 'index' => '1', 'job' => 'mutator') }

src/bosh-monitor/spec/unit/bosh/monitor/plugins/resurrector_spec.rb

@@ -65,6 +65,10 @@ module Bosh::Monitor::Plugins
       it 'should not start' do
         expect(plugin.run).to be(false)
       end
+
+      it 'registers the alert_tracker on the Bosh::Monitor module' do
+        expect(Bosh::Monitor.alert_tracker).to be_a(Bosh::Monitor::Plugins::ResurrectorHelper::AlertTracker)
+      end
     end
 
     context 'when the reactor is running' do