FEATURE: Add support for resync with non-grouped alerts payload

FEATURE: Add support for resync with non-grouped alerts payload

This is supported on the old /prometheus/receiver/grouped/alerts/:token endpoint, as well as a new /prometheus/receiver/resync/:token endpoint

diff --git a/app/jobs/regular/process_grouped_alerts.rb b/app/jobs/regular/process_grouped_alerts.rb
index 9f78733..b49d813 100644
--- a/app/jobs/regular/process_grouped_alerts.rb
+++ b/app/jobs/regular/process_grouped_alerts.rb
@@ -18,8 +18,16 @@ module Jobs
         token
       )
 
-      mark_stale_alerts(receiver, data, graph_url)
-      process_silenced_alerts(receiver, data)
+      if data[0]&.key?("blocks")
+        # Data from {{alertmanager}}/api/v1/alerts/grouped (removed in alertmanager v0.16.0)
+        current_alerts = current_alerts(data)
+      else
+        # Data from {{alertmanager}}/api/v1/alerts
+        current_alerts = data
+      end
+
+      mark_stale_alerts(receiver, current_alerts, graph_url)
+      process_silenced_alerts(receiver, current_alerts)
     end
 
     private
@@ -42,7 +50,7 @@ module Jobs
       end
     end
 
-    def mark_stale_alerts(receiver, data, graph_url)
+    def mark_stale_alerts(receiver, current_alerts, graph_url)
       Topic.firing_alerts.each do |topic|
         DistributedMutex.synchronize("prom_alert_receiver_topic_#{topic.id}") do
           alerts = topic.custom_fields.dig(alert_history_key, 'alerts')
@@ -50,7 +58,7 @@ module Jobs
 
           alerts&.each do |alert|
             if alert['graph_url'].include?(graph_url) && is_firing?(alert['status'])
-              is_stale = !current_alerts(data).any? do |current_alert|
+              is_stale = !current_alerts.any? do |current_alert|
                 current_alert['labels']['id'] == alert['id']
               end
 
@@ -94,44 +102,43 @@ module Jobs
       end
     end
 
-    def process_silenced_alerts(receiver, data)
-      data.each do |group|
-        topic = Topic.find_by(id: receiver[:topic_map][group["labels"]["alertname"]])
+    def process_silenced_alerts(receiver, current_alerts)
+      grouped_alerts = current_alerts.group_by { |a| a["labels"]["alertname"] }
+
+      grouped_alerts.each do |alertname, active_alerts|
+        topic = Topic.find_by(id: receiver[:topic_map][alertname])
 
         if topic
           DistributedMutex.synchronize("prom_alert_receiver_topic_#{topic.id}") do
-            group["blocks"].each do |block|
-              active_alerts = block["alerts"]
-              annotations = active_alerts.first["annotations"]
+            annotations = active_alerts.first["annotations"]
 
-              stored_alerts = begin
-                topic.custom_fields[alert_history_key]&.dig('alerts') || []
-              end
+            stored_alerts = begin
+              topic.custom_fields[alert_history_key]&.dig('alerts') || []
+            end
 
-              silenced = silence_alerts(stored_alerts, active_alerts,
-                datacenter: group["labels"]["datacenter"]
+            silenced = silence_alerts(stored_alerts, active_alerts,
+              datacenter: active_alerts[0]["labels"]["datacenter"]
+            )
+
+            if silenced
+              topic.save_custom_fields(true)
+
+              raw = first_post_body(
+                receiver: receiver,
+                topic_body: annotations["topic_body"],
+                alert_history: stored_alerts,
+                prev_topic_id: topic.custom_fields[::DiscoursePrometheusAlertReceiver::PREVIOUS_TOPIC_CUSTOM_FIELD]
               )
 
-              if silenced
-                topic.save_custom_fields(true)
-
-                raw = first_post_body(
-                  receiver: receiver,
-                  topic_body: annotations["topic_body"],
-                  alert_history: stored_alerts,
-                  prev_topic_id: topic.custom_fields[::DiscoursePrometheusAlertReceiver::PREVIOUS_TOPIC_CUSTOM_FIELD]
-                )
-
-                revise_topic(
-                  topic: topic,
-                  title: generate_title(annotations["topic_title"], stored_alerts),
-                  raw: raw,
-                  datacenters: datacenters(stored_alerts),
-                  firing: stored_alerts.any? { |alert| is_firing?(alert["status"]) }
-                )
-
-                publish_alert_counts
-              end
+              revise_topic(
+                topic: topic,
+                title: generate_title(annotations["topic_title"], stored_alerts),
+                raw: raw,
+                datacenters: datacenters(stored_alerts),
+                firing: stored_alerts.any? { |alert| is_firing?(alert["status"]) }
+              )
+
+              publish_alert_counts
             end
           end
         end
diff --git a/plugin.rb b/plugin.rb
index 409b4b7..79b6810 100644
--- a/plugin.rb
+++ b/plugin.rb
@@ -121,6 +121,7 @@ after_initialize do
   ::DiscoursePrometheusAlertReceiver::Engine.routes.draw do
     token_format = /[a-f0-9]{64}/
     post "/receiver/:token" => "receiver#receive", token: token_format, as: :receive
+    post "/receiver/resync/:token" => "receiver#receive_grouped_alerts", token: token_format
     post "/receiver/grouped/alerts/:token" => "receiver#receive_grouped_alerts", token: token_format, as: :receive_grouped_alerts
     post "/receiver/generate" => "receiver#generate_receiver_url", constraints: AdminConstraint.new
   end
diff --git a/spec/integration/discourse_prometheus_alert_receiver/receiver_controller_spec.rb b/spec/integration/discourse_prometheus_alert_receiver/receiver_controller_spec.rb
index e0595d8..220b19b 100644
--- a/spec/integration/discourse_prometheus_alert_receiver/receiver_controller_spec.rb
+++ b/spec/integration/discourse_prometheus_alert_receiver/receiver_controller_spec.rb
@@ -330,6 +330,132 @@ RSpec.describe DiscoursePrometheusAlertReceiver::ReceiverController do
           end
         end
 
+        describe 'when payload includes silenced alerts in the new format' do
+          let(:payload) do
+            {
+              "status" => "success",
+              "externalURL" => "supposed.to.be.a.url",
+              "graphURL" => "to.be.a.url",
+              "data" => [
+                {
+                  "labels" => {
+                    "alertname" => alert_name,
+                    "datacenter" => "somedatacenter",
+                    'id' => 'somethingnotfunny',
+                    "instance" => "someinstance",
+                    "job" => "somejob",
+                    "notify" => "live"
+                  },
+                  "annotations" => {
+                    "description" => "some description",
+                    "topic_body" => "some body",
+                    "topic_title" => "some title"
+                  },
+                  "startsAt" => "2018-07-24T23:25:31.363742334Z",
+                  "endsAt" => "0001-01-01T00:00:00Z",
+                  "generatorURL" => "http://supposed.to.be.a.url/graph?g0.expr=lolrus",
+                  "status" => {
+                    "state" => "suppressed",
+                    "silencedBy" => [
+                      "1de62db1-ac72-48d8-b2d0-7ce0e8acdcb1"
+                    ],
+                    "inhibitedBy" => nil
+                  },
+                  "receivers" => nil,
+                  "fingerprint" => "09aae3ea59ed5a65"
+                },
+                {
+                  "labels" => {
+                    "alertname" => alert_name,
+                    "datacenter" => "somedatacenter",
+                    'id' => 'somethingfunny',
+                    "instance" => "someinstance",
+                    "job" => "somejob",
+                    "notify" => "live"
+                  },
+                  "annotations" => {
+                    "description" => "some description",
+                    "topic_body" => "some body",
+                    "topic_title" => "some title"
+                  },
+                  "startsAt" => "2018-07-24T23:25:31.363742334Z",
+                  "endsAt" => "0001-01-01T00:00:00Z",
+                  "generatorURL" => "http://supposed.to.be.a.url/graph?g0.expr=lolrus",
+                  "status" => {
+                    "state" => "suppressed",
+                    "silencedBy" => [

[... diff too long, it was truncated ...]

GitHub sha: 538edbf9

1 Like

Do this need to be a block? Can’t it be a one-liner?

I only changed the indentation of this line. I’ll be doing a big re-write in the next few days so will improve things then :slight_smile:

1 Like