diff --git a/src/bosh-director/lib/bosh/director/metrics_collector.rb b/src/bosh-director/lib/bosh/director/metrics_collector.rb index 966fbf47cb..59b018b3ad 100644 --- a/src/bosh-director/lib/bosh/director/metrics_collector.rb +++ b/src/bosh-director/lib/bosh/director/metrics_collector.rb @@ -69,6 +69,11 @@ def initialize(config) labels: %i[name], docstring: 'Number of instances with unknown job_state per deployment', ) + @meltdown = Prometheus::Client.registry.gauge( + :bosh_meltdown, + labels: %i[name], + docstring: 'Is a deployment in meltdown state? 0 for no, 1 for yes', + ) @scheduler = Rufus::Scheduler.new end @@ -148,6 +153,7 @@ def populate_vm_metrics fetch_and_update_gauge('/failing_instances', @failing_instances) fetch_and_update_gauge('/stopped_instances', @stopped_instances) fetch_and_update_gauge('/unknown_instances', @unknown_instances) + fetch_and_update_gauge('/meltdown_state', @meltdown) end def fetch_and_update_gauge(endpoint, gauge) diff --git a/src/bosh-director/spec/unit/bosh/director/metrics_collector_spec.rb b/src/bosh-director/spec/unit/bosh/director/metrics_collector_spec.rb index d7b3af3ddd..cfd0fcffce 100644 --- a/src/bosh-director/spec/unit/bosh/director/metrics_collector_spec.rb +++ b/src/bosh-director/spec/unit/bosh/director/metrics_collector_spec.rb @@ -45,6 +45,8 @@ def tick .to_return(status: 200, body: JSON.dump("flaky_deployment" => 0, "good_deployment" => 0)) stub_request(:get, /unknown_instances/) .to_return(status: 200, body: JSON.dump("flaky_deployment" => 0, "good_deployment" => 1)) + stub_request(:get, /meltdown_state/) + .to_return(status: 200, body: JSON.dump("flaky_deployment" => 1, "good_deployment" => 0)) end after do @@ -59,6 +61,7 @@ def tick Prometheus::Client.registry.unregister(:bosh_failing_instances) Prometheus::Client.registry.unregister(:bosh_stopped_instances) Prometheus::Client.registry.unregister(:bosh_unknown_instances) + Prometheus::Client.registry.unregister(:bosh_meltdown) end describe '#prep' do @@ -337,6 +340,13 @@ def tick expect(metric.get(labels: { name: 'good_deployment' })).to eq(1) end + it 'emits bosh_meltdown for each deployment' do + metrics_collector.start + metric = Prometheus::Client.registry.get(:bosh_meltdown) + expect(metric.get(labels: { name: 'flaky_deployment' })).to eq(1) + expect(metric.get(labels: { name: 'good_deployment' })).to eq(0) + end + context "when the health monitor returns a non 200 response" do before do stub_request(:get, "127.0.0.1:12345/unresponsive_agents") diff --git a/src/bosh-monitor/lib/bosh/monitor/api_controller.rb b/src/bosh-monitor/lib/bosh/monitor/api_controller.rb index e946b5a177..a82c177ab0 100644 --- a/src/bosh-monitor/lib/bosh/monitor/api_controller.rb +++ b/src/bosh-monitor/lib/bosh/monitor/api_controller.rb @@ -80,5 +80,13 @@ def initialize(heartbeat_interval = 1) status(503) end end + + get '/meltdown_state' do + if @instance_manager.director_initial_deployment_sync_done + JSON.generate(@instance_manager.meltdown_state) + else + status(503) + end + end end end diff --git a/src/bosh-monitor/lib/bosh/monitor/config.rb b/src/bosh-monitor/lib/bosh/monitor/config.rb index e11c2a7ff3..b0903dfee4 100644 --- a/src/bosh-monitor/lib/bosh/monitor/config.rb +++ b/src/bosh-monitor/lib/bosh/monitor/config.rb @@ -8,6 +8,7 @@ class << self attr_accessor :instance_manager attr_reader :resurrection_manager attr_accessor :event_processor + attr_accessor :alert_tracker attr_accessor :http_port attr_accessor :plugins diff --git a/src/bosh-monitor/lib/bosh/monitor/instance_manager.rb b/src/bosh-monitor/lib/bosh/monitor/instance_manager.rb index e876ad2929..1bbda22944 100644 --- a/src/bosh-monitor/lib/bosh/monitor/instance_manager.rb +++ b/src/bosh-monitor/lib/bosh/monitor/instance_manager.rb @@ -171,6 +171,15 @@ def total_available_agents agents_hash end + def meltdown_state + tracker = Bosh::Monitor.alert_tracker + return {} unless tracker + + @deployment_name_to_deployments.keys.each_with_object({}) do |name, hash| + hash[name] = tracker.state_for(name).meltdown? ? 1 : 0 + end + end + def analyze_agents @logger.info('Analyzing agents...') started = Time.now diff --git a/src/bosh-monitor/lib/bosh/monitor/plugins/resurrector.rb b/src/bosh-monitor/lib/bosh/monitor/plugins/resurrector.rb index a950a748bc..8819ef4733 100644 --- a/src/bosh-monitor/lib/bosh/monitor/plugins/resurrector.rb +++ b/src/bosh-monitor/lib/bosh/monitor/plugins/resurrector.rb @@ -17,6 +17,7 @@ def initialize(options = {}) @processor = Bosh::Monitor.event_processor @resurrection_manager = Bosh::Monitor.resurrection_manager @alert_tracker = ResurrectorHelper::AlertTracker.new(@options) + Bosh::Monitor.alert_tracker = @alert_tracker end def run diff --git a/src/bosh-monitor/spec/unit/bosh/monitor/api_controller_spec.rb b/src/bosh-monitor/spec/unit/bosh/monitor/api_controller_spec.rb index da700a2115..84880e296f 100644 --- a/src/bosh-monitor/spec/unit/bosh/monitor/api_controller_spec.rb +++ b/src/bosh-monitor/spec/unit/bosh/monitor/api_controller_spec.rb @@ -238,4 +238,35 @@ def app end end end + + describe "/meltdown_state" do + let(:meltdown_state) do + { + "first_deployment" => 1, + "second_deployment" => 0, + } + end + + before do + allow(Bosh::Monitor.instance_manager).to receive(:meltdown_state).and_return(meltdown_state) + allow(Bosh::Monitor.instance_manager).to receive(:director_initial_deployment_sync_done).and_return(true) + end + + it "renders meltdown state per deployment" do + get "/meltdown_state" + expect(last_response.status).to eq(200) + expect(last_response.body).to eq(JSON.generate(meltdown_state)) + end + + context "When director initial deployment sync has not completed" do + before do + allow(Bosh::Monitor.instance_manager).to receive(:director_initial_deployment_sync_done).and_return(false) + end + + it "returns 503 when /meltdown_state is requested" do + get "/meltdown_state" + expect(last_response.status).to eq(503) + end + end + end end diff --git a/src/bosh-monitor/spec/unit/bosh/monitor/instance_manager_spec.rb b/src/bosh-monitor/spec/unit/bosh/monitor/instance_manager_spec.rb index 6d55e717c0..bfb2797165 100644 --- a/src/bosh-monitor/spec/unit/bosh/monitor/instance_manager_spec.rb +++ b/src/bosh-monitor/spec/unit/bosh/monitor/instance_manager_spec.rb @@ -392,6 +392,43 @@ module Bosh::Monitor end end + describe '#meltdown_state' do + let(:instance1) { Bosh::Monitor::Instance.create('id' => 'iuuid1', 'agent_id' => '007', 'index' => '0', 'job' => 'mutator') } + + before do + manager.sync_deployments([{ 'name' => 'mycloud' }]) + manager.sync_agents('mycloud', [instance1]) + end + + context 'when alert_tracker is not set' do + before { Bosh::Monitor.alert_tracker = nil } + after { Bosh::Monitor.alert_tracker = nil } + + it 'returns an empty hash' do + expect(manager.meltdown_state).to eq({}) + end + end + + context 'when alert_tracker is set' do + let(:tracker) { instance_double(Bosh::Monitor::Plugins::ResurrectorHelper::AlertTracker) } + + before { Bosh::Monitor.alert_tracker = tracker } + after { Bosh::Monitor.alert_tracker = nil } + + it 'returns 1 for a deployment in meltdown' do + state = instance_double(Bosh::Monitor::Plugins::ResurrectorHelper::DeploymentState, meltdown?: true) + allow(tracker).to receive(:state_for).with('mycloud').and_return(state) + expect(manager.meltdown_state).to eq('mycloud' => 1) + end + + it 'returns 0 for a deployment not in meltdown' do + state = instance_double(Bosh::Monitor::Plugins::ResurrectorHelper::DeploymentState, meltdown?: false) + allow(tracker).to receive(:state_for).with('mycloud').and_return(state) + expect(manager.meltdown_state).to eq('mycloud' => 0) + end + end + end + describe '#analyze_agents' do let(:instance_1) { Bosh::Monitor::Instance.create('id' => 'instance-uuid-1', 'agent_id' => '007', 'index' => '0', 'job' => 'mutator') } let(:instance_2) { Bosh::Monitor::Instance.create('id' => 'instance-uuid-2', 'agent_id' => '008', 'index' => '1', 'job' => 'mutator') } diff --git a/src/bosh-monitor/spec/unit/bosh/monitor/plugins/resurrector_spec.rb b/src/bosh-monitor/spec/unit/bosh/monitor/plugins/resurrector_spec.rb index 96aa3d5c1a..e5079e7eb1 100644 --- a/src/bosh-monitor/spec/unit/bosh/monitor/plugins/resurrector_spec.rb +++ b/src/bosh-monitor/spec/unit/bosh/monitor/plugins/resurrector_spec.rb @@ -65,6 +65,10 @@ module Bosh::Monitor::Plugins it 'should not start' do expect(plugin.run).to be(false) end + + it 'registers the alert_tracker on the Bosh::Monitor module' do + expect(Bosh::Monitor.alert_tracker).to be_a(Bosh::Monitor::Plugins::ResurrectorHelper::AlertTracker) + end end context 'when the reactor is running' do