FIX: Restore silenced alerts during 'process_grouped_alerts'

FIX: Restore silenced alerts during ‘process_grouped_alerts’

Also improved the job to merge the ‘silenced’ and ‘stale’ logic into the same loop

diff --git a/app/jobs/regular/process_grouped_alerts.rb b/app/jobs/regular/process_grouped_alerts.rb
index b49d813..cc5b478 100644
--- a/app/jobs/regular/process_grouped_alerts.rb
+++ b/app/jobs/regular/process_grouped_alerts.rb
@@ -26,8 +26,7 @@ module Jobs
         current_alerts = data
       end
 
-      mark_stale_alerts(receiver, current_alerts, graph_url)
-      process_silenced_alerts(receiver, current_alerts)
+      update_open_alerts(receiver, current_alerts, graph_url)
     end
 
     private
@@ -50,22 +49,23 @@ module Jobs
       end
     end
 
-    def mark_stale_alerts(receiver, current_alerts, graph_url)
-      Topic.firing_alerts.each do |topic|
+    def update_open_alerts(receiver, active_alerts, graph_url)
+      Topic.open_alerts.each do |topic|
         DistributedMutex.synchronize("prom_alert_receiver_topic_#{topic.id}") do
-          alerts = topic.custom_fields.dig(alert_history_key, 'alerts')
+          stored_alerts = topic.custom_fields.dig(alert_history_key, 'alerts')
           updated = false
 
-          alerts&.each do |alert|
-            if alert['graph_url'].include?(graph_url) && is_firing?(alert['status'])
-              is_stale = !current_alerts.any? do |current_alert|
-                current_alert['labels']['id'] == alert['id']
-              end
-
-              if is_stale &&
-                STALE_DURATION.minute.ago > DateTime.parse(alert["starts_at"])
+          stored_alerts&.each do |stored_alert|
+            if stored_alert['graph_url'].include?(graph_url) && stored_alert['status'] != 'resolved'
+              active_alert = active_alerts.find { |a| a['labels']['id'] == stored_alert['id'] }
 
-                alert["status"] = "stale"
+              if !active_alert && stored_alert["status"] != "stale" &&
+                  STALE_DURATION.minute.ago > DateTime.parse(stored_alert["starts_at"])
+                stored_alert["status"] = "stale"
+                updated = true
+              elsif active_alert && stored_alert["status"] != active_alert["status"]["state"]
+                stored_alert["status"] = active_alert["status"]["state"]
+                stored_alert["description"] = active_alert.dig("annotations", "description")
                 updated = true
               end
             end
@@ -76,7 +76,7 @@ module Jobs
             klass = DiscoursePrometheusAlertReceiver
 
             if base_title = topic.custom_fields[klass::TOPIC_BASE_TITLE_CUSTOM_FIELD]
-              title = generate_title(base_title, alerts)
+              title = generate_title(base_title, stored_alerts)
             else
               title = topic.custom_fields[klass::TOPIC_TITLE_CUSTOM_FIELD] || ''
             end
@@ -84,7 +84,7 @@ module Jobs
             raw = first_post_body(
               receiver: receiver,
               topic_body: topic.custom_fields[klass::TOPIC_BODY_CUSTOM_FIELD] || '',
-              alert_history: alerts,
+              alert_history: stored_alerts,
               prev_topic_id: topic.custom_fields[klass::PREVIOUS_TOPIC_CUSTOM_FIELD]
             )
 
@@ -92,8 +92,8 @@ module Jobs
               topic: topic,
               title: title,
               raw: raw,
-              datacenters: datacenters(alerts),
-              firing: alerts.any? { |alert| is_firing?(alert["status"]) }
+              datacenters: datacenters(stored_alerts),
+              firing: stored_alerts.any? { |alert| is_firing?(alert["status"]) }
             )
 
             publish_alert_counts
@@ -102,74 +102,6 @@ module Jobs
       end
     end
 
-    def process_silenced_alerts(receiver, current_alerts)
-      grouped_alerts = current_alerts.group_by { |a| a["labels"]["alertname"] }
-
-      grouped_alerts.each do |alertname, active_alerts|
-        topic = Topic.find_by(id: receiver[:topic_map][alertname])
-
-        if topic
-          DistributedMutex.synchronize("prom_alert_receiver_topic_#{topic.id}") do
-            annotations = active_alerts.first["annotations"]
-
-            stored_alerts = begin
-              topic.custom_fields[alert_history_key]&.dig('alerts') || []
-            end
-
-            silenced = silence_alerts(stored_alerts, active_alerts,
-              datacenter: active_alerts[0]["labels"]["datacenter"]
-            )
-
-            if silenced
-              topic.save_custom_fields(true)
-
-              raw = first_post_body(
-                receiver: receiver,
-                topic_body: annotations["topic_body"],
-                alert_history: stored_alerts,
-                prev_topic_id: topic.custom_fields[::DiscoursePrometheusAlertReceiver::PREVIOUS_TOPIC_CUSTOM_FIELD]
-              )
-
-              revise_topic(
-                topic: topic,
-                title: generate_title(annotations["topic_title"], stored_alerts),
-                raw: raw,
-                datacenters: datacenters(stored_alerts),
-                firing: stored_alerts.any? { |alert| is_firing?(alert["status"]) }
-              )
-
-              publish_alert_counts
-            end
-          end
-        end
-      end
-    end
-
-    def silence_alerts(stored_alerts, active_alerts, datacenter:)
-      silenced = false
-
-      stored_alerts.each do |alert|
-        active = active_alerts.find do |active_alert|
-          active_alert["labels"]["id"] == alert["id"] &&
-            alert['datacenter'] == datacenter &&
-            Date.parse(active_alert["startsAt"]).to_s == Date.parse(alert["starts_at"]).to_s &&
-            active_alert["status"]["state"] == "suppressed"
-        end
-
-        if active
-          alert["description"] = active.dig("annotations", "description")
-          state = active["status"]["state"]
-
-          if alert["status"] != state
-            alert["status"] = state
-            silenced ||= true
-          end
-        end
-      end
-
-      silenced
-    end
-
     def publish_alert_counts
       MessageBus.publish("/alert-receiver",
         firing_alerts_count: Topic.firing_alerts.count,
diff --git a/spec/integration/discourse_prometheus_alert_receiver/receiver_controller_spec.rb b/spec/integration/discourse_prometheus_alert_receiver/receiver_controller_spec.rb
index 220b19b..7d01c4e 100644
--- a/spec/integration/discourse_prometheus_alert_receiver/receiver_controller_spec.rb
+++ b/spec/integration/discourse_prometheus_alert_receiver/receiver_controller_spec.rb
@@ -445,6 +445,26 @@ RSpec.describe DiscoursePrometheusAlertReceiver::ReceiverController do
             ).to eq('some description')
           end
 
+          it "should restore alerts when they are unsilenced" do
+
+            post "/prometheus/receiver/grouped/alerts/#{token}", params: payload
+            expect(
+              topic.reload.custom_fields[custom_field_key]['alerts'].
+                  find { |a| a["id"] == "somethingnotfunny" }['status']
+            ).to eq('suppressed')
+
+            payload["data"].find {
+              |a| a["labels"]["id"] == "somethingnotfunny"
+            }["status"]["state"] = "firing"
+
+            post "/prometheus/receiver/grouped/alerts/#{token}", params: payload
+
+            expect(
+              topic.reload.custom_fields[custom_field_key]['alerts'].
+              find { |a| a["id"] == "somethingnotfunny" }['status']
+            ).to eq('firing')
+          end
+
           it 'should not update the topic if nothing has changed' do
             post "/prometheus/receiver/grouped/alerts/#{token}", params: payload

GitHub sha: dffb0baf