mirror of
https://github.com/discourse/discourse.git
synced 2025-05-31 21:25:24 +08:00
FEATURE: Implement support for IMAP and SMTP email protocols. (#8301)
Co-authored-by: Joffrey JAFFEUX <j.jaffeux@gmail.com>
This commit is contained in:
Dan Ungureanu
116
lib/imap/providers/generic.rb
Normal file
116
lib/imap/providers/generic.rb
Normal file
@ -0,0 +1,116 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
require 'net/imap'
|
||||
|
||||
module Imap
|
||||
module Providers
|
||||
class Generic
|
||||
|
||||
def initialize(server, options = {})
|
||||
@server = server
|
||||
@port = options[:port] || 993
|
||||
@ssl = options[:ssl] || true
|
||||
@username = options[:username]
|
||||
@password = options[:password]
|
||||
@timeout = options[:timeout] || 10
|
||||
end
|
||||
|
||||
def imap
|
||||
@imap ||= Net::IMAP.new(@server, port: @port, ssl: @ssl, open_timeout: @timeout)
|
||||
end
|
||||
|
||||
def disconnected?
|
||||
@imap && @imap.disconnected?
|
||||
end
|
||||
|
||||
def connect!
|
||||
imap.login(@username, @password)
|
||||
end
|
||||
|
||||
def disconnect!
|
||||
imap.logout rescue nil
|
||||
imap.disconnect
|
||||
end
|
||||
|
||||
def can?(capability)
|
||||
@capabilities ||= imap.responses['CAPABILITY'][-1] || imap.capability
|
||||
@capabilities.include?(capability)
|
||||
end
|
||||
|
||||
def uids(opts = {})
|
||||
if opts[:from] && opts[:to]
|
||||
imap.uid_search("UID #{opts[:from]}:#{opts[:to]}")
|
||||
elsif opts[:from]
|
||||
imap.uid_search("UID #{opts[:from]}:*")
|
||||
elsif opts[:to]
|
||||
imap.uid_search("UID 1:#{opts[:to]}")
|
||||
else
|
||||
imap.uid_search('ALL')
|
||||
end
|
||||
end
|
||||
|
||||
def labels
|
||||
@labels ||= begin
|
||||
labels = {}
|
||||
|
||||
list_mailboxes.each do |name|
|
||||
if tag = to_tag(name)
|
||||
labels[tag] = name
|
||||
end
|
||||
end
|
||||
|
||||
labels
|
||||
end
|
||||
end
|
||||
|
||||
def open_mailbox(mailbox_name, write: false)
|
||||
if write
|
||||
raise 'two-way IMAP sync is disabled' if !SiteSetting.enable_imap_write
|
||||
imap.select(mailbox_name)
|
||||
else
|
||||
imap.examine(mailbox_name)
|
||||
end
|
||||
|
||||
{
|
||||
uid_validity: imap.responses['UIDVALIDITY'][-1]
|
||||
}
|
||||
end
|
||||
|
||||
def emails(uids, fields, opts = {})
|
||||
imap.uid_fetch(uids, fields).map do |email|
|
||||
attributes = {}
|
||||
|
||||
fields.each do |field|
|
||||
attributes[field] = email.attr[field]
|
||||
end
|
||||
|
||||
attributes
|
||||
end
|
||||
end
|
||||
|
||||
def store(uid, attribute, old_set, new_set)
|
||||
additions = new_set.reject { |val| old_set.include?(val) }
|
||||
imap.uid_store(uid, "+#{attribute}", additions) if additions.length > 0
|
||||
removals = old_set.reject { |val| new_set.include?(val) }
|
||||
imap.uid_store(uid, "-#{attribute}", removals) if removals.length > 0
|
||||
end
|
||||
|
||||
def to_tag(label)
|
||||
label = DiscourseTagging.clean_tag(label.to_s)
|
||||
label if label != 'inbox' && label != 'sent'
|
||||
end
|
||||
|
||||
def tag_to_flag(tag)
|
||||
:Seen if tag == 'seen'
|
||||
end
|
||||
|
||||
def tag_to_label(tag)
|
||||
labels[tag]
|
||||
end
|
||||
|
||||
def list_mailboxes
|
||||
imap.list('', '*').map(&:name)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
145
lib/imap/providers/gmail.rb
Normal file
145
lib/imap/providers/gmail.rb
Normal file
@ -0,0 +1,145 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
module Imap
|
||||
module Providers
|
||||
class Gmail < Generic
|
||||
X_GM_LABELS = 'X-GM-LABELS'
|
||||
|
||||
def imap
|
||||
@imap ||= super.tap { |imap| apply_gmail_patch(imap) }
|
||||
end
|
||||
|
||||
def emails(uids, fields, opts = {})
|
||||
fields[fields.index('LABELS')] = X_GM_LABELS
|
||||
|
||||
emails = super(uids, fields, opts)
|
||||
|
||||
emails.each do |email|
|
||||
email['LABELS'] = Array(email['LABELS'])
|
||||
|
||||
if email[X_GM_LABELS]
|
||||
email['LABELS'] << Array(email.delete(X_GM_LABELS))
|
||||
email['LABELS'].flatten!
|
||||
end
|
||||
|
||||
email['LABELS'] << '\\Inbox' if opts[:mailbox] == 'INBOX'
|
||||
|
||||
email['LABELS'].uniq!
|
||||
end
|
||||
|
||||
emails
|
||||
end
|
||||
|
||||
def store(uid, attribute, old_set, new_set)
|
||||
attribute = X_GM_LABELS if attribute == 'LABELS'
|
||||
super(uid, attribute, old_set, new_set)
|
||||
end
|
||||
|
||||
def to_tag(label)
|
||||
# Label `\\Starred` is Gmail equivalent of :Flagged (both present)
|
||||
return 'starred' if label == :Flagged
|
||||
return if label == '[Gmail]/All Mail'
|
||||
|
||||
label = label.to_s.gsub('[Gmail]/', '')
|
||||
super(label)
|
||||
end
|
||||
|
||||
def tag_to_flag(tag)
|
||||
return :Flagged if tag == 'starred'
|
||||
|
||||
super(tag)
|
||||
end
|
||||
|
||||
def tag_to_label(tag)
|
||||
return '\\Important' if tag == 'important'
|
||||
return '\\Starred' if tag == 'starred'
|
||||
|
||||
super(tag)
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def apply_gmail_patch(imap)
|
||||
class << imap.instance_variable_get('@parser')
|
||||
|
||||
# Modified version of the original `msg_att` from here:
|
||||
# https://github.com/ruby/ruby/blob/1cc8ff001da217d0e98d13fe61fbc9f5547ef722/lib/net/imap.rb#L2346
|
||||
# rubocop:disable Style/RedundantReturn
|
||||
def msg_att(n)
|
||||
match(T_LPAR)
|
||||
attr = {}
|
||||
while true
|
||||
token = lookahead
|
||||
case token.symbol
|
||||
when T_RPAR
|
||||
shift_token
|
||||
break
|
||||
when T_SPACE
|
||||
shift_token
|
||||
next
|
||||
end
|
||||
case token.value
|
||||
when /\A(?:ENVELOPE)\z/ni
|
||||
name, val = envelope_data
|
||||
when /\A(?:FLAGS)\z/ni
|
||||
name, val = flags_data
|
||||
when /\A(?:INTERNALDATE)\z/ni
|
||||
name, val = internaldate_data
|
||||
when /\A(?:RFC822(?:\.HEADER|\.TEXT)?)\z/ni
|
||||
name, val = rfc822_text
|
||||
when /\A(?:RFC822\.SIZE)\z/ni
|
||||
name, val = rfc822_size
|
||||
when /\A(?:BODY(?:STRUCTURE)?)\z/ni
|
||||
name, val = body_data
|
||||
when /\A(?:UID)\z/ni
|
||||
name, val = uid_data
|
||||
when /\A(?:MODSEQ)\z/ni
|
||||
name, val = modseq_data
|
||||
# Adding support for GMail extended attributes.
|
||||
when /\A(?:X-GM-LABELS)\z/ni
|
||||
name, val = label_data
|
||||
when /\A(?:X-GM-MSGID)\z/ni
|
||||
name, val = uid_data
|
||||
when /\A(?:X-GM-THRID)\z/ni
|
||||
name, val = uid_data
|
||||
else
|
||||
parse_error("unknown attribute `%s' for {%d}", token.value, n)
|
||||
end
|
||||
attr[name] = val
|
||||
end
|
||||
return attr
|
||||
end
|
||||
|
||||
def label_data
|
||||
token = match(T_ATOM)
|
||||
name = token.value.upcase
|
||||
|
||||
match(T_SPACE)
|
||||
match(T_LPAR)
|
||||
|
||||
result = []
|
||||
while true
|
||||
token = lookahead
|
||||
case token.symbol
|
||||
when T_RPAR
|
||||
shift_token
|
||||
break
|
||||
when T_SPACE
|
||||
shift_token
|
||||
end
|
||||
|
||||
token = lookahead
|
||||
if string_token?(token)
|
||||
result.push(string)
|
||||
else
|
||||
result.push(atom)
|
||||
end
|
||||
end
|
||||
return name, result
|
||||
end
|
||||
# rubocop:enable Style/RedundantReturn
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
258
lib/imap/sync.rb
Normal file
258
lib/imap/sync.rb
Normal file
@ -0,0 +1,258 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
require 'net/imap'
|
||||
|
||||
module Imap
|
||||
class Sync
|
||||
def self.for_group(group, opts = {})
|
||||
if group.imap_server == 'imap.gmail.com'
|
||||
opts[:provider] ||= Imap::Providers::Gmail
|
||||
end
|
||||
|
||||
Imap::Sync.new(group, opts)
|
||||
end
|
||||
|
||||
def initialize(group, opts = {})
|
||||
@group = group
|
||||
|
||||
provider_klass ||= opts[:provider] || Imap::Providers::Generic
|
||||
@provider = provider_klass.new(@group.imap_server,
|
||||
port: @group.imap_port,
|
||||
ssl: @group.imap_ssl,
|
||||
username: @group.email_username,
|
||||
password: @group.email_password
|
||||
)
|
||||
|
||||
connect!
|
||||
end
|
||||
|
||||
def connect!
|
||||
@provider.connect!
|
||||
end
|
||||
|
||||
def disconnect!
|
||||
@provider.disconnect!
|
||||
end
|
||||
|
||||
def disconnected?
|
||||
@provider.disconnected?
|
||||
end
|
||||
|
||||
def can_idle?
|
||||
SiteSetting.enable_imap_idle && @provider.can?('IDLE')
|
||||
end
|
||||
|
||||
def process(idle: false, import_limit: nil, old_emails_limit: nil, new_emails_limit: nil)
|
||||
raise 'disconnected' if disconnected?
|
||||
|
||||
import_limit ||= SiteSetting.imap_batch_import_email
|
||||
old_emails_limit ||= SiteSetting.imap_polling_old_emails
|
||||
new_emails_limit ||= SiteSetting.imap_polling_new_emails
|
||||
|
||||
# IMAP server -> Discourse (download): discovers updates to old emails
|
||||
# (synced emails) and fetches new emails.
|
||||
|
||||
# TODO: Use `Net::IMAP.encode_utf7(@group.imap_mailbox_name)`?
|
||||
@status = @provider.open_mailbox(@group.imap_mailbox_name)
|
||||
|
||||
if @status[:uid_validity] != @group.imap_uid_validity
|
||||
# If UID validity changes, the whole mailbox must be synchronized (all
|
||||
# emails are considered new and will be associated to existent topics
|
||||
# in Email::Reciever by matching Message-Ids).
|
||||
Rails.logger.warn("[IMAP] UIDVALIDITY = #{@status[:uid_validity]} does not match expected #{@group.imap_uid_validity}, invalidating IMAP cache and resyncing emails for group #{@group.name} and mailbox #{@group.imap_mailbox_name}")
|
||||
@group.imap_last_uid = 0
|
||||
end
|
||||
|
||||
if idle && !can_idle?
|
||||
Rails.logger.warn("[IMAP] IMAP server for group #{@group.name} cannot IDLE")
|
||||
idle = false
|
||||
end
|
||||
|
||||
if idle
|
||||
raise 'IMAP IDLE is disabled' if !SiteSetting.enable_imap_idle
|
||||
|
||||
# Thread goes into sleep and it is better to return any connection
|
||||
# back to the pool.
|
||||
ActiveRecord::Base.connection_handler.clear_active_connections!
|
||||
|
||||
@provider.imap.idle(SiteSetting.imap_polling_period_mins.minutes.to_i) do |resp|
|
||||
if resp.kind_of?(Net::IMAP::UntaggedResponse) && resp.name == 'EXISTS'
|
||||
@provider.imap.idle_done
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
# Fetching UIDs of old (already imported into Discourse, but might need
|
||||
# update) and new (not downloaded yet) emails.
|
||||
if @group.imap_last_uid == 0
|
||||
old_uids = []
|
||||
new_uids = @provider.uids
|
||||
else
|
||||
old_uids = @provider.uids(to: @group.imap_last_uid) # 1 .. seen
|
||||
new_uids = @provider.uids(from: @group.imap_last_uid + 1) # seen+1 .. inf
|
||||
end
|
||||
|
||||
# Sometimes, new_uids contains elements from old_uids.
|
||||
new_uids = new_uids - old_uids
|
||||
|
||||
Rails.logger.debug("[IMAP] Remote email server has #{old_uids.size} old emails and #{new_uids.size} new emails")
|
||||
|
||||
all_old_uids_size = old_uids.size
|
||||
all_new_uids_size = new_uids.size
|
||||
|
||||
@group.update_columns(
|
||||
imap_last_error: nil,
|
||||
imap_old_emails: all_old_uids_size,
|
||||
imap_new_emails: all_new_uids_size
|
||||
)
|
||||
|
||||
import_mode = import_limit > -1 && new_uids.size > import_limit
|
||||
old_uids = old_uids.sample(old_emails_limit).sort! if old_emails_limit > -1
|
||||
new_uids = new_uids[0..new_emails_limit - 1] if new_emails_limit > 0
|
||||
|
||||
if old_uids.present?
|
||||
Rails.logger.debug("[IMAP] Syncing #{old_uids.size} randomly-selected old emails")
|
||||
emails = @provider.emails(old_uids, ['UID', 'FLAGS', 'LABELS'], mailbox: @group.imap_mailbox_name)
|
||||
emails.each do |email|
|
||||
incoming_email = IncomingEmail.find_by(
|
||||
imap_uid_validity: @status[:uid_validity],
|
||||
imap_uid: email['UID']
|
||||
)
|
||||
|
||||
if incoming_email.present?
|
||||
update_topic(email, incoming_email, mailbox_name: @group.imap_mailbox_name)
|
||||
else
|
||||
Rails.logger.warn("[IMAP] Could not find old email (UIDVALIDITY = #{@status[:uid_validity]}, UID = #{email['UID']})")
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
if new_uids.present?
|
||||
Rails.logger.debug("[IMAP] Syncing #{new_uids.size} new emails (oldest first)")
|
||||
|
||||
emails = @provider.emails(new_uids, ['UID', 'FLAGS', 'LABELS', 'RFC822'], mailbox: @group.imap_mailbox_name)
|
||||
processed = 0
|
||||
|
||||
emails.each do |email|
|
||||
# Synchronously process emails because the order of emails matter
|
||||
# (for example replies must be processed after the original email
|
||||
# to have a topic where the reply can be posted).
|
||||
begin
|
||||
receiver = Email::Receiver.new(email['RFC822'],
|
||||
allow_auto_generated: true,
|
||||
import_mode: import_mode,
|
||||
destinations: [@group],
|
||||
uid_validity: @status[:uid_validity],
|
||||
uid: email['UID']
|
||||
)
|
||||
receiver.process!
|
||||
update_topic(email, receiver.incoming_email, mailbox_name: @group.imap_mailbox_name)
|
||||
rescue Email::Receiver::ProcessingError => e
|
||||
Rails.logger.warn("[IMAP] Could not process (UIDVALIDITY = #{@status[:uid_validity]}, UID = #{email['UID']}): #{e.message}")
|
||||
end
|
||||
|
||||
processed += 1
|
||||
@group.update_columns(
|
||||
imap_uid_validity: @status[:uid_validity],
|
||||
imap_last_uid: email['UID'],
|
||||
imap_old_emails: all_old_uids_size + processed,
|
||||
imap_new_emails: all_new_uids_size - processed
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
# Discourse -> IMAP server (upload): syncs updated flags and labels.
|
||||
if SiteSetting.enable_imap_write
|
||||
to_sync = IncomingEmail.where(imap_sync: true)
|
||||
if to_sync.size > 0
|
||||
@provider.open_mailbox(@group.imap_mailbox_name, write: true)
|
||||
to_sync.each do |incoming_email|
|
||||
Rails.logger.debug("[IMAP] Updating email for #{@group.name} and incoming email ID = #{incoming_email.id}")
|
||||
update_email(@group.imap_mailbox_name, incoming_email)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
{ remaining: all_new_uids_size - new_uids.size }
|
||||
end
|
||||
|
||||
def update_topic(email, incoming_email, opts = {})
|
||||
return if !incoming_email ||
|
||||
incoming_email.imap_sync ||
|
||||
!incoming_email.topic ||
|
||||
incoming_email.post&.post_number != 1
|
||||
|
||||
update_topic_archived_state(email, incoming_email, opts)
|
||||
update_topic_tags(email, incoming_email, opts)
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def update_topic_archived_state(email, incoming_email, opts = {})
|
||||
topic = incoming_email.topic
|
||||
|
||||
topic_is_archived = topic.group_archived_messages.size > 0
|
||||
email_is_archived = !email['LABELS'].include?('\\Inbox') && !email['LABELS'].include?('INBOX')
|
||||
|
||||
if topic_is_archived && !email_is_archived
|
||||
GroupArchivedMessage.move_to_inbox!(@group.id, topic, skip_imap_sync: true)
|
||||
elsif !topic_is_archived && email_is_archived
|
||||
GroupArchivedMessage.archive!(@group.id, topic, skip_imap_sync: true)
|
||||
end
|
||||
end
|
||||
|
||||
def update_topic_tags(email, incoming_email, opts = {})
|
||||
group_email_regex = @group.email_username_regex
|
||||
topic = incoming_email.topic
|
||||
|
||||
tags = Set.new
|
||||
|
||||
# "Plus" part from the destination email address
|
||||
to_addresses = incoming_email.to_addresses&.split(";") || []
|
||||
cc_addresses = incoming_email.cc_addresses&.split(";") || []
|
||||
(to_addresses + cc_addresses).each do |address|
|
||||
if plus_part = address&.scan(group_email_regex)&.first&.first
|
||||
tags.add("plus:#{plus_part[1..-1]}") if plus_part.length > 0
|
||||
end
|
||||
end
|
||||
|
||||
# Mailbox name
|
||||
tags.add(@provider.to_tag(opts[:mailbox_name])) if opts[:mailbox_name]
|
||||
|
||||
# Flags and labels
|
||||
email['FLAGS'].each { |flag| tags.add(@provider.to_tag(flag)) }
|
||||
email['LABELS'].each { |label| tags.add(@provider.to_tag(label)) }
|
||||
|
||||
tags.subtract([nil, ''])
|
||||
|
||||
# TODO: Optimize tagging.
|
||||
# `DiscourseTagging.tag_topic_by_names` does a lot of lookups in the
|
||||
# database and some of them could be cached in this context.
|
||||
DiscourseTagging.tag_topic_by_names(topic, Guardian.new(Discourse.system_user), tags.to_a)
|
||||
end
|
||||
|
||||
def update_email(mailbox_name, incoming_email)
|
||||
return if !SiteSetting.tagging_enabled || !SiteSetting.allow_staff_to_tag_pms
|
||||
return if incoming_email&.post&.post_number != 1 || !incoming_email.imap_sync
|
||||
return unless email = @provider.emails(incoming_email.imap_uid, ['FLAGS', 'LABELS'], mailbox: mailbox_name).first
|
||||
incoming_email.update(imap_sync: false)
|
||||
|
||||
labels = email['LABELS']
|
||||
flags = email['FLAGS']
|
||||
topic = incoming_email.topic
|
||||
|
||||
# TODO: Delete remote email if topic no longer exists
|
||||
# new_flags << Net::IMAP::DELETED if !incoming_email.topic
|
||||
return if !topic
|
||||
|
||||
# Sync topic status and labels with email flags and labels.
|
||||
tags = topic.tags.pluck(:name)
|
||||
new_flags = tags.map { |tag| @provider.tag_to_flag(tag) }.reject(&:blank?)
|
||||
# new_flags << Net::IMAP::DELETED if !incoming_email.topic
|
||||
new_labels = tags.map { |tag| @provider.tag_to_label(tag) }.reject(&:blank?)
|
||||
new_labels << '\\Inbox' if topic.group_archived_messages.length == 0
|
||||
@provider.store(incoming_email.imap_uid, 'FLAGS', flags, new_flags)
|
||||
@provider.store(incoming_email.imap_uid, 'LABELS', labels, new_labels)
|
||||
end
|
||||
end
|
||||
end
|
||||
Reference in New Issue
Block a user