FEATURE: IMAP delete email sync for group inboxes (#10392)

Adds functionality to reflect topic delete in Discourse to IMAP inbox (Gmail only for now) and reflecting Gmail deletes in Discourse.

Adding lots of tests, various refactors and code improvements.

When Discourse topic is destroyed in PostDestroyer mark the topic incoming email as imap_sync: true, and do the opposite when post is recovered.

lib/imap/providers/generic.rb

@@ -6,6 +6,19 @@ module Imap
   module Providers
     class WriteDisabledError < StandardError; end
 
+    class TrashedMailResponse
+      attr_accessor :trashed_emails, :trash_uid_validity
+    end
+
+    class BasicMail
+      attr_accessor :uid, :message_id
+
+      def initialize(uid: nil, message_id: nil)
+        @uid = uid
+        @message_id = message_id
+      end
+    end
+
     class Generic
       def initialize(server, options = {})
         @server = server
@@ -16,6 +29,10 @@ module Imap
         @timeout = options[:timeout] || 10
       end
 
+      def account_digest
+        @account_digest ||= Digest::MD5.hexdigest("#{@username}:#{@server}")
+      end
+
       def imap
         @imap ||= Net::IMAP.new(@server, port: @port, ssl: @ssl, open_timeout: @timeout)
       end
@@ -121,9 +138,27 @@ module Imap
         tag
       end
 
-      def list_mailboxes
-        imap.list('', '*').map do |m|
-          next if m.attr.include?(:Noselect)
+      def list_mailboxes(attr_filter = nil)
+        # Basically, list all mailboxes in the root of the server.
+        # ref: https://tools.ietf.org/html/rfc3501#section-6.3.8
+        imap.list('', '*').reject do |m|
+
+          # Noselect cannot be selected with the SELECT command.
+          # technically we could use this for readonly mode when
+          # SiteSetting.imap_write is disabled...maybe a later TODO
+          # ref: https://tools.ietf.org/html/rfc3501#section-7.2.2
+          m.attr.include?(:Noselect)
+        end.select do |m|
+
+          # There are Special-Use mailboxes denoted by an attribute. For
+          # example, some common ones are \Trash or \Sent.
+          # ref: https://tools.ietf.org/html/rfc6154
+          if attr_filter
+            m.attr.include? attr_filter
+          else
+            true
+          end
+        end.map do |m|
           m.name
         end
       end
@@ -131,6 +166,83 @@ module Imap
       def archive(uid)
         # do nothing by default, just removing the Inbox label should be enough
       end
+
+      def unarchive(uid)
+        # same as above
+      end
+
+      # Look for the special Trash XLIST attribute.
+      # TODO: It might be more efficient to just store this against the group.
+      # Another table is looking more and more attractive....
+      def trash_mailbox
+        Discourse.cache.fetch("imap_trash_mailbox_#{account_digest}", expires_in: 30.minutes) do
+          list_mailboxes(:Trash).first
+        end
+      end
+
+      # open the trash mailbox for inspection or writing. after the yield we
+      # close the trash and reopen the original mailbox to continue operations.
+      # the normal open_mailbox call can be made if more extensive trash ops
+      # need to be done.
+      def open_trash_mailbox(write: false)
+        open_mailbox_before_trash = @open_mailbox_name
+        open_mailbox_before_trash_write = @open_mailbox_write
+
+        trash_uid_validity = open_mailbox(trash_mailbox, write: write)[:uid_validity]
+
+        yield(trash_uid_validity) if block_given?
+
+        open_mailbox(open_mailbox_before_trash, write: open_mailbox_before_trash_write)
+        trash_uid_validity
+      end
+
+      def find_trashed_by_message_ids(message_ids)
+        trashed_emails = []
+        trash_uid_validity = open_trash_mailbox do
+          header_message_id_terms = message_ids.map do |msgid|
+            "HEADER Message-ID '#{Email.message_id_rfc_format(msgid)}'"
+          end
+
+          # OR clauses are written in Polish notation...so the query looks like this:
+          # OR OR HEADER Message-ID XXXX HEADER Message-ID XXXX HEADER Message-ID XXXX
+          or_clauses = 'OR ' * (header_message_id_terms.length - 1)
+          query = "#{or_clauses}#{header_message_id_terms.join(" ")}"
+
+          trashed_email_uids = imap.uid_search(query)
+          if trashed_email_uids.any?
+            trashed_emails = emails(trashed_email_uids, ["UID", "ENVELOPE"]).map do |e|
+              BasicMail.new(message_id: Email.message_id_clean(e['ENVELOPE'].message_id), uid: e['UID'])
+            end
+          end
+        end
+
+        TrashedMailResponse.new.tap do |resp|
+          resp.trashed_emails = trashed_emails
+          resp.trash_uid_validity = trash_uid_validity
+        end
+      end
+
+      def trash(uid)
+        # MOVE is way easier than doing the COPY \Deleted EXPUNGE dance ourselves.
+        # It is supported by Gmail and Outlook.
+        if can?('MOVE')
+          trash_move(uid)
+        else
+
+          # default behaviour for IMAP servers is to add the \Deleted flag
+          # then EXPUNGE the mailbox which permanently deletes these messages
+          # https://tools.ietf.org/html/rfc3501#section-6.4.3
+          #
+          # TODO: We may want to add the option at some point to copy to some
+          # other mailbox first before doing this (e.g. Trash)
+          store(uid, 'FLAGS', [], ["\\Deleted"])
+          imap.expunge
+        end
+      end
+
+      def trash_move(uid)
+        # up to the provider
+      end
     end
   end
 end

lib/imap/providers/gmail.rb

@@ -2,6 +2,11 @@
 
 module Imap
   module Providers
+    # Gmail has a special header for both labels (X-GM-LABELS) and their
+    # threading system (X-GM-THRID). We need to monkey-patch Net::IMAP to
+    # get access to these. Also the archiving functionality is custom,
+    # all UIDs in a thread must have the \\Inbox label removed.
+    #
     class Gmail < Generic
       X_GM_LABELS = 'X-GM-LABELS'
       X_GM_THRID = 'X-GM-THRID'
@@ -62,9 +67,9 @@ module Imap
         super(tag)
       end
 
+      # All emails in the thread must be archived in Gmail for the thread
+      # to get removed from the inbox
       def archive(uid)
-        # all emails in the thread must be archived in Gmail for the thread
-        # to get removed from the inbox
         thread_id = thread_id_from_uid(uid)
         emails_to_archive = emails_in_thread(thread_id)
         emails_to_archive.each do |email|
@@ -75,6 +80,23 @@ module Imap
         Imap::Sync::Logger.log("[IMAP] Thread ID #{thread_id} (UID #{uid}) archived in Gmail mailbox for #{@username}")
       end
 
+      # Though Gmail considers the email thread unarchived if the first email
+      # has the \\Inbox label applied, we want to do this to all emails in the
+      # thread to be consistent with archive behaviour.
+      def unarchive(uid)
+        thread_id = thread_id_from_uid(uid)
+        emails_to_unarchive = emails_in_thread(thread_id)
+        emails_to_unarchive.each do |email|
+          labels = email['LABELS']
+          new_labels = labels.dup
+          if !new_labels.include?("\\Inbox")
+            new_labels << "\\Inbox"
+          end
+          store(email["UID"], "LABELS", labels, new_labels)
+        end
+        Imap::Sync::Logger.log("[IMAP] Thread ID #{thread_id} (UID #{uid}) unarchived in Gmail mailbox for #{@username}")
+      end
+
       def thread_id_from_uid(uid)
         fetched = imap.uid_fetch(uid, [X_GM_THRID])
         if !fetched
@@ -89,6 +111,15 @@ module Imap
         emails(uids_to_fetch, ["UID", "LABELS"])
       end
 
+      def trash_move(uid)
+        thread_id = thread_id_from_uid(uid)
+        email_uids_to_trash = emails_in_thread(thread_id).map { |e| e['UID'] }
+
+        imap.uid_move(email_uids_to_trash, trash_mailbox)
+        Imap::Sync::Logger.log("[IMAP] Thread ID #{thread_id} (UID #{uid}) trashed in Gmail mailbox for #{@username}")
+        { trash_uid_validity: open_trash_mailbox, email_uids_to_trash: email_uids_to_trash }
+      end
+
       private
 
       def apply_gmail_patch(imap)