mirror of
https://github.com/discourse/discourse.git
synced 2025-05-22 07:53:49 +08:00
FEATURE: IMAP delete email sync for group inboxes (#10392)
Adds functionality to reflect topic delete in Discourse to IMAP inbox (Gmail only for now) and reflecting Gmail deletes in Discourse. Adding lots of tests, various refactors and code improvements. When Discourse topic is destroyed in PostDestroyer mark the topic incoming email as imap_sync: true, and do the opposite when post is recovered.
This commit is contained in:
Martin Brennan
@ -6,6 +6,19 @@ module Imap
|
||||
module Providers
|
||||
class WriteDisabledError < StandardError; end
|
||||
|
||||
class TrashedMailResponse
|
||||
attr_accessor :trashed_emails, :trash_uid_validity
|
||||
end
|
||||
|
||||
class BasicMail
|
||||
attr_accessor :uid, :message_id
|
||||
|
||||
def initialize(uid: nil, message_id: nil)
|
||||
@uid = uid
|
||||
@message_id = message_id
|
||||
end
|
||||
end
|
||||
|
||||
class Generic
|
||||
def initialize(server, options = {})
|
||||
@server = server
|
||||
@ -16,6 +29,10 @@ module Imap
|
||||
@timeout = options[:timeout] || 10
|
||||
end
|
||||
|
||||
def account_digest
|
||||
@account_digest ||= Digest::MD5.hexdigest("#{@username}:#{@server}")
|
||||
end
|
||||
|
||||
def imap
|
||||
@imap ||= Net::IMAP.new(@server, port: @port, ssl: @ssl, open_timeout: @timeout)
|
||||
end
|
||||
@ -121,9 +138,27 @@ module Imap
|
||||
tag
|
||||
end
|
||||
|
||||
def list_mailboxes
|
||||
imap.list('', '*').map do |m|
|
||||
next if m.attr.include?(:Noselect)
|
||||
def list_mailboxes(attr_filter = nil)
|
||||
# Basically, list all mailboxes in the root of the server.
|
||||
# ref: https://tools.ietf.org/html/rfc3501#section-6.3.8
|
||||
imap.list('', '*').reject do |m|
|
||||
|
||||
# Noselect cannot be selected with the SELECT command.
|
||||
# technically we could use this for readonly mode when
|
||||
# SiteSetting.imap_write is disabled...maybe a later TODO
|
||||
# ref: https://tools.ietf.org/html/rfc3501#section-7.2.2
|
||||
m.attr.include?(:Noselect)
|
||||
end.select do |m|
|
||||
|
||||
# There are Special-Use mailboxes denoted by an attribute. For
|
||||
# example, some common ones are \Trash or \Sent.
|
||||
# ref: https://tools.ietf.org/html/rfc6154
|
||||
if attr_filter
|
||||
m.attr.include? attr_filter
|
||||
else
|
||||
true
|
||||
end
|
||||
end.map do |m|
|
||||
m.name
|
||||
end
|
||||
end
|
||||
@ -131,6 +166,83 @@ module Imap
|
||||
def archive(uid)
|
||||
# do nothing by default, just removing the Inbox label should be enough
|
||||
end
|
||||
|
||||
def unarchive(uid)
|
||||
# same as above
|
||||
end
|
||||
|
||||
# Look for the special Trash XLIST attribute.
|
||||
# TODO: It might be more efficient to just store this against the group.
|
||||
# Another table is looking more and more attractive....
|
||||
def trash_mailbox
|
||||
Discourse.cache.fetch("imap_trash_mailbox_#{account_digest}", expires_in: 30.minutes) do
|
||||
list_mailboxes(:Trash).first
|
||||
end
|
||||
end
|
||||
|
||||
# open the trash mailbox for inspection or writing. after the yield we
|
||||
# close the trash and reopen the original mailbox to continue operations.
|
||||
# the normal open_mailbox call can be made if more extensive trash ops
|
||||
# need to be done.
|
||||
def open_trash_mailbox(write: false)
|
||||
open_mailbox_before_trash = @open_mailbox_name
|
||||
open_mailbox_before_trash_write = @open_mailbox_write
|
||||
|
||||
trash_uid_validity = open_mailbox(trash_mailbox, write: write)[:uid_validity]
|
||||
|
||||
yield(trash_uid_validity) if block_given?
|
||||
|
||||
open_mailbox(open_mailbox_before_trash, write: open_mailbox_before_trash_write)
|
||||
trash_uid_validity
|
||||
end
|
||||
|
||||
def find_trashed_by_message_ids(message_ids)
|
||||
trashed_emails = []
|
||||
trash_uid_validity = open_trash_mailbox do
|
||||
header_message_id_terms = message_ids.map do |msgid|
|
||||
"HEADER Message-ID '#{Email.message_id_rfc_format(msgid)}'"
|
||||
end
|
||||
|
||||
# OR clauses are written in Polish notation...so the query looks like this:
|
||||
# OR OR HEADER Message-ID XXXX HEADER Message-ID XXXX HEADER Message-ID XXXX
|
||||
or_clauses = 'OR ' * (header_message_id_terms.length - 1)
|
||||
query = "#{or_clauses}#{header_message_id_terms.join(" ")}"
|
||||
|
||||
trashed_email_uids = imap.uid_search(query)
|
||||
if trashed_email_uids.any?
|
||||
trashed_emails = emails(trashed_email_uids, ["UID", "ENVELOPE"]).map do |e|
|
||||
BasicMail.new(message_id: Email.message_id_clean(e['ENVELOPE'].message_id), uid: e['UID'])
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
TrashedMailResponse.new.tap do |resp|
|
||||
resp.trashed_emails = trashed_emails
|
||||
resp.trash_uid_validity = trash_uid_validity
|
||||
end
|
||||
end
|
||||
|
||||
def trash(uid)
|
||||
# MOVE is way easier than doing the COPY \Deleted EXPUNGE dance ourselves.
|
||||
# It is supported by Gmail and Outlook.
|
||||
if can?('MOVE')
|
||||
trash_move(uid)
|
||||
else
|
||||
|
||||
# default behaviour for IMAP servers is to add the \Deleted flag
|
||||
# then EXPUNGE the mailbox which permanently deletes these messages
|
||||
# https://tools.ietf.org/html/rfc3501#section-6.4.3
|
||||
#
|
||||
# TODO: We may want to add the option at some point to copy to some
|
||||
# other mailbox first before doing this (e.g. Trash)
|
||||
store(uid, 'FLAGS', [], ["\\Deleted"])
|
||||
imap.expunge
|
||||
end
|
||||
end
|
||||
|
||||
def trash_move(uid)
|
||||
# up to the provider
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
@ -2,6 +2,11 @@
|
||||
|
||||
module Imap
|
||||
module Providers
|
||||
# Gmail has a special header for both labels (X-GM-LABELS) and their
|
||||
# threading system (X-GM-THRID). We need to monkey-patch Net::IMAP to
|
||||
# get access to these. Also the archiving functionality is custom,
|
||||
# all UIDs in a thread must have the \\Inbox label removed.
|
||||
#
|
||||
class Gmail < Generic
|
||||
X_GM_LABELS = 'X-GM-LABELS'
|
||||
X_GM_THRID = 'X-GM-THRID'
|
||||
@ -62,9 +67,9 @@ module Imap
|
||||
super(tag)
|
||||
end
|
||||
|
||||
# All emails in the thread must be archived in Gmail for the thread
|
||||
# to get removed from the inbox
|
||||
def archive(uid)
|
||||
# all emails in the thread must be archived in Gmail for the thread
|
||||
# to get removed from the inbox
|
||||
thread_id = thread_id_from_uid(uid)
|
||||
emails_to_archive = emails_in_thread(thread_id)
|
||||
emails_to_archive.each do |email|
|
||||
@ -75,6 +80,23 @@ module Imap
|
||||
Imap::Sync::Logger.log("[IMAP] Thread ID #{thread_id} (UID #{uid}) archived in Gmail mailbox for #{@username}")
|
||||
end
|
||||
|
||||
# Though Gmail considers the email thread unarchived if the first email
|
||||
# has the \\Inbox label applied, we want to do this to all emails in the
|
||||
# thread to be consistent with archive behaviour.
|
||||
def unarchive(uid)
|
||||
thread_id = thread_id_from_uid(uid)
|
||||
emails_to_unarchive = emails_in_thread(thread_id)
|
||||
emails_to_unarchive.each do |email|
|
||||
labels = email['LABELS']
|
||||
new_labels = labels.dup
|
||||
if !new_labels.include?("\\Inbox")
|
||||
new_labels << "\\Inbox"
|
||||
end
|
||||
store(email["UID"], "LABELS", labels, new_labels)
|
||||
end
|
||||
Imap::Sync::Logger.log("[IMAP] Thread ID #{thread_id} (UID #{uid}) unarchived in Gmail mailbox for #{@username}")
|
||||
end
|
||||
|
||||
def thread_id_from_uid(uid)
|
||||
fetched = imap.uid_fetch(uid, [X_GM_THRID])
|
||||
if !fetched
|
||||
@ -89,6 +111,15 @@ module Imap
|
||||
emails(uids_to_fetch, ["UID", "LABELS"])
|
||||
end
|
||||
|
||||
def trash_move(uid)
|
||||
thread_id = thread_id_from_uid(uid)
|
||||
email_uids_to_trash = emails_in_thread(thread_id).map { |e| e['UID'] }
|
||||
|
||||
imap.uid_move(email_uids_to_trash, trash_mailbox)
|
||||
Imap::Sync::Logger.log("[IMAP] Thread ID #{thread_id} (UID #{uid}) trashed in Gmail mailbox for #{@username}")
|
||||
{ trash_uid_validity: open_trash_mailbox, email_uids_to_trash: email_uids_to_trash }
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def apply_gmail_patch(imap)
|
||||
|
||||
@ -107,9 +107,9 @@ module Imap
|
||||
old_uids = old_uids.sample(old_emails_limit).sort! if old_emails_limit > -1
|
||||
new_uids = new_uids[0..new_emails_limit - 1] if new_emails_limit > 0
|
||||
|
||||
if old_uids.present?
|
||||
process_old_uids(old_uids)
|
||||
end
|
||||
# if there are no old_uids that is OK, this could indicate that some
|
||||
# UIDs have been sent to the trash
|
||||
process_old_uids(old_uids)
|
||||
|
||||
if new_uids.present?
|
||||
process_new_uids(new_uids, import_mode, all_old_uids_size, all_new_uids_size)
|
||||
@ -135,7 +135,7 @@ module Imap
|
||||
|
||||
def process_old_uids(old_uids)
|
||||
Logger.log("[IMAP] (#{@group.name}) Syncing #{old_uids.size} randomly-selected old emails")
|
||||
emails = @provider.emails(old_uids, ['UID', 'FLAGS', 'LABELS', 'ENVELOPE'])
|
||||
emails = old_uids.empty? ? [] : @provider.emails(old_uids, ['UID', 'FLAGS', 'LABELS', 'ENVELOPE'])
|
||||
emails.each do |email|
|
||||
incoming_email = IncomingEmail.find_by(
|
||||
imap_uid_validity: @status[:uid_validity],
|
||||
@ -148,7 +148,7 @@ module Imap
|
||||
else
|
||||
# try finding email by message-id instead, we may be able to set the uid etc.
|
||||
incoming_email = IncomingEmail.where(
|
||||
message_id: email['ENVELOPE'].message_id.tr("<>", ""),
|
||||
message_id: Email.message_id_clean(email['ENVELOPE'].message_id),
|
||||
imap_uid: nil,
|
||||
imap_uid_validity: nil
|
||||
).where("to_addresses LIKE '%#{@group.email_username}%'").first
|
||||
@ -165,6 +165,53 @@ module Imap
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
handle_missing_uids(old_uids)
|
||||
end
|
||||
|
||||
def handle_missing_uids(old_uids)
|
||||
# If there are any UIDs for the mailbox missing from old_uids, this means they have been moved
|
||||
# to some other mailbox in the mail server. They could be possibly deleted. first we can check
|
||||
# if they have been deleted and if so delete the associated post/topic. then the remaining we
|
||||
# can just remove the imap details from the IncomingEmail table and if they end up back in the
|
||||
# original mailbox then they will be picked up in a future resync.
|
||||
existing_incoming = IncomingEmail.includes(:post).where(
|
||||
imap_group_id: @group.id, imap_uid_validity: @status[:uid_validity]
|
||||
).where.not(imap_uid: nil)
|
||||
|
||||
existing_uids = existing_incoming.map(&:imap_uid)
|
||||
missing_uids = existing_uids - old_uids
|
||||
missing_message_ids = existing_incoming.select do |incoming|
|
||||
missing_uids.include?(incoming.imap_uid)
|
||||
end.map(&:message_id)
|
||||
|
||||
return if missing_message_ids.empty?
|
||||
|
||||
# This can be done because Message-ID is unique on a mail server between mailboxes,
|
||||
# where the UID will have changed when moving into the Trash mailbox. We need to get
|
||||
# the new UID from the trash.
|
||||
response = @provider.find_trashed_by_message_ids(missing_message_ids)
|
||||
existing_incoming.each do |incoming|
|
||||
matching_trashed = response.trashed_emails.find { |email| email.message_id == incoming.message_id }
|
||||
|
||||
# if the email is not in the trash then we don't know where it is... could
|
||||
# be in any mailbox on the server or could be permanently deleted. TODO
|
||||
# here would be some sort of more complex detection of "where in the world
|
||||
# has this UID gone?"
|
||||
next if !matching_trashed
|
||||
|
||||
# if we deleted the topic/post ourselves in discourse then the post will
|
||||
# not exist, and this sync is just updating the old UIDs to the new ones
|
||||
# in the trash, and we don't need to re-destroy the post
|
||||
if incoming.post
|
||||
Logger.log("[IMAP] (#{@group.name}) Deleting post ID #{incoming.post_id}; it has been deleted on the IMAP server.")
|
||||
PostDestroyer.new(Discourse.system_user, incoming.post).destroy
|
||||
end
|
||||
|
||||
# the email has moved mailboxes, we don't want to try trashing again next time
|
||||
Logger.log("[IMAP] (#{@group.name}) Updating incoming ID #{incoming.id} uid data FROM [UID #{incoming.imap_uid} | UIDVALIDITY #{incoming.imap_uid_validity}] TO [UID #{matching_trashed.uid} | UIDVALIDITY #{response.trash_uid_validity}] (TRASHED)")
|
||||
incoming.update(imap_uid_validity: response.trash_uid_validity, imap_uid: matching_trashed.uid)
|
||||
end
|
||||
end
|
||||
|
||||
def process_new_uids(new_uids, import_mode, all_old_uids_size, all_new_uids_size)
|
||||
@ -266,7 +313,19 @@ module Imap
|
||||
|
||||
def update_email(incoming_email)
|
||||
return if !SiteSetting.tagging_enabled || !SiteSetting.allow_staff_to_tag_pms
|
||||
return if incoming_email&.post&.post_number != 1 || !incoming_email.imap_sync
|
||||
return if !incoming_email || !incoming_email.imap_sync
|
||||
|
||||
post = incoming_email.post
|
||||
if !post && incoming_email.post_id
|
||||
# post was likely deleted because topic was deleted, let's try get it
|
||||
post = Post.with_deleted.find(incoming_email.post_id)
|
||||
end
|
||||
|
||||
# don't do any of these type of updates on anything but the OP in the
|
||||
# email thread -- archiving and deleting will be handled for the whole
|
||||
# thread depending on provider
|
||||
return if post&.post_number != 1
|
||||
topic = incoming_email.topic
|
||||
|
||||
# if email is nil, the UID does not exist in the provider, meaning....
|
||||
#
|
||||
@ -279,11 +338,14 @@ module Imap
|
||||
|
||||
labels = email['LABELS']
|
||||
flags = email['FLAGS']
|
||||
topic = incoming_email.topic
|
||||
|
||||
# TODO: Delete remote email if topic no longer exists
|
||||
# new_flags << Net::IMAP::DELETED if !incoming_email.topic
|
||||
return if !topic
|
||||
# Topic has been deleted if it is not present from the post, so we need
|
||||
# to trash the IMAP server email
|
||||
if !topic
|
||||
# no need to do anything further here, we will recognize the UIDs in the
|
||||
# mail server email thread have been trashed on next sync
|
||||
return @provider.trash(incoming_email.imap_uid)
|
||||
end
|
||||
|
||||
# Sync topic status and labels with email flags and labels.
|
||||
tags = topic.tags.pluck(:name)
|
||||
@ -294,7 +356,15 @@ module Imap
|
||||
# server
|
||||
topic_archived = topic.group_archived_messages.any?
|
||||
if !topic_archived
|
||||
new_labels << '\\Inbox'
|
||||
# TODO: This is needed right now so the store below does not take it
|
||||
# away again...ideally we should unarchive and store the tag-labels
|
||||
# at the same time.
|
||||
new_labels << "\\Inbox"
|
||||
|
||||
Logger.log("[IMAP] (#{@group.name}) Unarchiving UID #{incoming_email.imap_uid}")
|
||||
|
||||
# some providers need special handling for unarchiving too
|
||||
@provider.unarchive(incoming_email.imap_uid)
|
||||
else
|
||||
Logger.log("[IMAP] (#{@group.name}) Archiving UID #{incoming_email.imap_uid}")
|
||||
|
||||
|
||||
Reference in New Issue
Block a user