FEATURE: IMAP delete email sync for group inboxes (#10392)

FEATURE: IMAP delete email sync for group inboxes (#10392)

Adds functionality to reflect topic delete in Discourse to IMAP inbox (Gmail only for now) and reflecting Gmail deletes in Discourse.

Adding lots of tests, various refactors and code improvements.

When Discourse topic is destroyed in PostDestroyer mark the topic incoming email as imap_sync: true, and do the opposite when post is recovered.

diff --git a/app/controllers/webhooks_controller.rb b/app/controllers/webhooks_controller.rb
index 347ff52..93995df 100644
--- a/app/controllers/webhooks_controller.rb
+++ b/app/controllers/webhooks_controller.rb
@@ -13,7 +13,7 @@ class WebhooksController < ActionController::Base
   def sendgrid
     events = params["_json"] || [params]
     events.each do |event|
-      message_id = (event["smtp-id"] || "").tr("<>", "")
+      message_id = Email.message_id_clean((event["smtp-id"] || ""))
       to_address = event["email"]
       if event["event"] == "bounce"
         if event["status"]["4."]
@@ -150,7 +150,7 @@ class WebhooksController < ActionController::Base
     return mailgun_failure unless valid_mailgun_signature?(params["token"], params["timestamp"], params["signature"])
 
     event = params["event"]
-    message_id = params["Message-Id"].tr("<>", "")
+    message_id = Email.message_id_clean(params["Message-Id"])
     to_address = params["recipient"]
 
     # only handle soft bounces, because hard bounces are also handled
diff --git a/lib/email.rb b/lib/email.rb
index 1f2dac5..497c6b9 100644
--- a/lib/email.rb
+++ b/lib/email.rb
@@ -3,6 +3,8 @@
 require 'mail'
 
 module Email
+  # cute little guy ain't he?
+  MESSAGE_ID_REGEX = /<(.*@.*)+>/
 
   def self.is_valid?(email)
     return false unless String === email
@@ -39,4 +41,14 @@ module Email
     SiteSetting.email_site_title.presence || SiteSetting.title
   end
 
+  # https://tools.ietf.org/html/rfc850#section-2.1.7
+  def self.message_id_rfc_format(message_id)
+    return message_id if message_id =~ MESSAGE_ID_REGEX
+    "<#{message_id}>"
+  end
+
+  def self.message_id_clean(message_id)
+    return message_id if !(message_id =~ MESSAGE_ID_REGEX)
+    message_id.tr("<>", "")
+  end
 end
diff --git a/lib/email/receiver.rb b/lib/email/receiver.rb
index 44f26ef..f27b851 100644
--- a/lib/email/receiver.rb
+++ b/lib/email/receiver.rb
@@ -988,7 +988,9 @@ module Email
       if Array === references
         references
       elsif references.present?
-        references.split(/[\s,]/).map { |r| r.tr("<>", "") }
+        references.split(/[\s,]/).map do |r|
+          Email.message_id_clean(r)
+        end
       end
     end
 
diff --git a/lib/imap/providers/generic.rb b/lib/imap/providers/generic.rb
index 0801c9d..4b82e40 100644
--- a/lib/imap/providers/generic.rb
+++ b/lib/imap/providers/generic.rb
@@ -6,6 +6,19 @@ module Imap
   module Providers
     class WriteDisabledError < StandardError; end
 
+    class TrashedMailResponse
+      attr_accessor :trashed_emails, :trash_uid_validity
+    end
+
+    class BasicMail
+      attr_accessor :uid, :message_id
+
+      def initialize(uid: nil, message_id: nil)
+        @uid = uid
+        @message_id = message_id
+      end
+    end
+
     class Generic
       def initialize(server, options = {})
         @server = server
@@ -16,6 +29,10 @@ module Imap
         @timeout = options[:timeout] || 10
       end
 
+      def account_digest
+        @account_digest ||= Digest::MD5.hexdigest("#{@username}:#{@server}")
+      end
+
       def imap
         @imap ||= Net::IMAP.new(@server, port: @port, ssl: @ssl, open_timeout: @timeout)
       end
@@ -121,9 +138,27 @@ module Imap
         tag
       end
 
-      def list_mailboxes
-        imap.list('', '*').map do |m|
-          next if m.attr.include?(:Noselect)
+      def list_mailboxes(attr_filter = nil)
+        # Basically, list all mailboxes in the root of the server.
+        # ref: https://tools.ietf.org/html/rfc3501#section-6.3.8
+        imap.list('', '*').reject do |m|
+
+          # Noselect cannot be selected with the SELECT command.
+          # technically we could use this for readonly mode when
+          # SiteSetting.imap_write is disabled...maybe a later TODO
+          # ref: https://tools.ietf.org/html/rfc3501#section-7.2.2
+          m.attr.include?(:Noselect)
+        end.select do |m|
+
+          # There are Special-Use mailboxes denoted by an attribute. For
+          # example, some common ones are \Trash or \Sent.
+          # ref: https://tools.ietf.org/html/rfc6154
+          if attr_filter
+            m.attr.include? attr_filter
+          else
+            true
+          end
+        end.map do |m|
           m.name
         end
       end
@@ -131,6 +166,83 @@ module Imap
       def archive(uid)
         # do nothing by default, just removing the Inbox label should be enough
       end
+
+      def unarchive(uid)
+        # same as above
+      end
+
+      # Look for the special Trash XLIST attribute.
+      # TODO: It might be more efficient to just store this against the group.
+      # Another table is looking more and more attractive....
+      def trash_mailbox
+        Discourse.cache.fetch("imap_trash_mailbox_#{account_digest}", expires_in: 30.minutes) do
+          list_mailboxes(:Trash).first
+        end
+      end
+
+      # open the trash mailbox for inspection or writing. after the yield we
+      # close the trash and reopen the original mailbox to continue operations.
+      # the normal open_mailbox call can be made if more extensive trash ops
+      # need to be done.
+      def open_trash_mailbox(write: false)
+        open_mailbox_before_trash = @open_mailbox_name
+        open_mailbox_before_trash_write = @open_mailbox_write
+
+        trash_uid_validity = open_mailbox(trash_mailbox, write: write)[:uid_validity]
+
+        yield(trash_uid_validity) if block_given?
+
+        open_mailbox(open_mailbox_before_trash, write: open_mailbox_before_trash_write)
+        trash_uid_validity
+      end
+
+      def find_trashed_by_message_ids(message_ids)
+        trashed_emails = []
+        trash_uid_validity = open_trash_mailbox do
+          header_message_id_terms = message_ids.map do |msgid|
+            "HEADER Message-ID '#{Email.message_id_rfc_format(msgid)}'"
+          end
+
+          # OR clauses are written in Polish notation...so the query looks like this:
+          # OR OR HEADER Message-ID XXXX HEADER Message-ID XXXX HEADER Message-ID XXXX
+          or_clauses = 'OR ' * (header_message_id_terms.length - 1)
+          query = "#{or_clauses}#{header_message_id_terms.join(" ")}"
+
+          trashed_email_uids = imap.uid_search(query)
+          if trashed_email_uids.any?
+            trashed_emails = emails(trashed_email_uids, ["UID", "ENVELOPE"]).map do |e|
+              BasicMail.new(message_id: Email.message_id_clean(e['ENVELOPE'].message_id), uid: e['UID'])
+            end
+          end
+        end
+
+        TrashedMailResponse.new.tap do |resp|
+          resp.trashed_emails = trashed_emails
+          resp.trash_uid_validity = trash_uid_validity
+        end
+      end
+
+      def trash(uid)
+        # MOVE is way easier than doing the COPY \Deleted EXPUNGE dance ourselves.
+        # It is supported by Gmail and Outlook.
+        if can?('MOVE')
+          trash_move(uid)
+        else
+
+          # default behaviour for IMAP servers is to add the \Deleted flag
+          # then EXPUNGE the mailbox which permanently deletes these messages
+          # https://tools.ietf.org/html/rfc3501#section-6.4.3
+          #
+          # TODO: We may want to add the option at some point to copy to some
+          # other mailbox first before doing this (e.g. Trash)
+          store(uid, 'FLAGS', [], ["\\Deleted"])
+          imap.expunge
+        end
+      end
+
+      def trash_move(uid)
+        # up to the provider
+      end
     end
   end
 end
diff --git a/lib/imap/providers/gmail.rb b/lib/imap/providers/gmail.rb
index 82a9ecd..da32755 100644
--- a/lib/imap/providers/gmail.rb
+++ b/lib/imap/providers/gmail.rb
@@ -2,6 +2,11 @@
 
 module Imap
   module Providers
+    # Gmail has a special header for both labels (X-GM-LABELS) and their
+    # threading system (X-GM-THRID). We need to monkey-patch Net::IMAP to
+    # get access to these. Also the archiving functionality is custom,

[... diff too long, it was truncated ...]

GitHub sha: 95b71b35

This commit appears in #10392 which was merged by martin.