FEATURE: better email in support

FEATURE: new incoming_email model
FEATURE: infinite scrolling in emails admin
FEATURE: new 'emails:import' rake task
This commit is contained in:
Régis Hanol
2016-01-19 00:57:55 +01:00
parent d0bcea3411
commit 3083657358
119 changed files with 1560 additions and 4408 deletions

View File

@ -1,144 +1,198 @@
require_dependency 'new_post_manager'
require_dependency 'email/html_cleaner'
require_dependency 'post_action_creator'
require_dependency "new_post_manager"
require_dependency "post_action_creator"
require_dependency "email/html_cleaner"
module Email
class Receiver
include ActionView::Helpers::NumberHelper
class ProcessingError < StandardError; end
class EmptyEmailError < ProcessingError; end
class NoMessageIdError < ProcessingError; end
class AutoGeneratedEmailError < ProcessingError; end
class NoBodyDetectedError < ProcessingError; end
class InactiveUserError < ProcessingError; end
class BadDestinationAddress < ProcessingError; end
class StrangersNotAllowedError < ProcessingError; end
class InsufficientTrustLevelError < ProcessingError; end
class ReplyUserNotMatchingError < ProcessingError; end
class TopicNotFoundError < ProcessingError; end
class TopicClosedError < ProcessingError; end
class InvalidPost < ProcessingError; end
class InvalidPostAction < ProcessingError; end
class ProcessingError < StandardError; end
class EmailUnparsableError < ProcessingError; end
class EmptyEmailError < ProcessingError; end
class UserNotFoundError < ProcessingError; end
class UserNotSufficientTrustLevelError < ProcessingError; end
class BadDestinationAddress < ProcessingError; end
class TopicNotFoundError < ProcessingError; end
class TopicClosedError < ProcessingError; end
class AutoGeneratedEmailError < ProcessingError; end
class EmailLogNotFound < ProcessingError; end
class InvalidPost < ProcessingError; end
class ReplyUserNotFoundError < ProcessingError; end
class ReplyUserNotMatchingError < ProcessingError; end
class InactiveUserError < ProcessingError; end
class InvalidPostAction < ProcessingError; end
attr_reader :body, :email_log
def initialize(raw, opts=nil)
@raw = raw
@opts = opts || {}
def initialize(mail_string)
raise EmptyEmailError if mail_string.blank?
@raw_email = mail_string
@mail = Mail.new(@raw_email)
raise NoMessageIdError if @mail.message_id.blank?
end
def process
raise EmptyEmailError if @raw.blank?
@message = Mail.new(@raw)
raise AutoGeneratedEmailError if @message.header.to_s =~ /auto-(replied|generated)/
@body = parse_body(@message)
# 'smtp_envelope_to' is a combination of: to, cc and bcc fields
# prioriziting the `:reply` types
dest_infos = @message.smtp_envelope_to
.map { |to_address| check_address(to_address) }
.compact
.sort do |a, b|
if a[:type] == :reply && b[:type] != :reply
1
elsif a[:type] != :reply && b[:type] == :reply
-1
else
0
end
end
raise BadDestinationAddress if dest_infos.empty?
from = @message[:from].address_list.addresses.first
user_email = from.address
user_name = from.display_name
user = User.find_by_email(user_email)
raise InactiveUserError if user.present? && !user.active && !user.staged
# TODO: take advantage of all the "TO"s
dest_info = dest_infos[0]
case dest_info[:type]
when :group
group = dest_info[:obj]
if user.blank?
if SiteSetting.allow_staged_accounts
user = create_staged_account(user_email, user_name)
else
wrap_body_in_quote(user_email)
user = Discourse.system_user
end
end
create_new_topic(user, archetype: Archetype.private_message, target_group_names: [group.name])
when :category
category = dest_info[:obj]
if user.blank? && category.email_in_allow_strangers
if SiteSetting.allow_staged_accounts
user = create_staged_account(user_email)
else
wrap_body_in_quote(user_email)
user = Discourse.system_user
end
end
raise UserNotFoundError if user.blank?
raise UserNotSufficientTrustLevelError.new(user) unless category.email_in_allow_strangers || user.has_trust_level?(TrustLevel[SiteSetting.email_in_min_trust.to_i])
create_new_topic(user, category: category.id)
when :reply
@email_log = dest_info[:obj]
raise EmailLogNotFound if @email_log.blank?
raise TopicNotFoundError if Topic.find_by_id(@email_log.topic_id).nil?
raise TopicClosedError if Topic.find_by_id(@email_log.topic_id).closed?
raise ReplyUserNotFoundError if user.blank?
raise ReplyUserNotMatchingError if @email_log.user_id != user.id
if post_action_type = post_action_for(@body)
create_post_action(@email_log, post_action_type)
else
create_reply(@email_log)
end
end
rescue Encoding::UndefinedConversionError, Encoding::InvalidByteSequenceError => e
raise EmailUnparsableError.new(e)
@incoming_email = find_or_create_incoming_email
process_internal
rescue => e
@incoming_email.update_columns(error: e.to_s)
raise
end
def create_staged_account(email, name=nil)
User.create(
email: email,
username: UserNameSuggester.suggest(name.presence || email),
name: name.presence || User.suggest_name(email),
staged: true,
)
def find_or_create_incoming_email
IncomingEmail.find_or_create_by(message_id: @mail.message_id) do |incoming_email|
incoming_email.raw = @raw_email
incoming_email.subject = @mail.subject
incoming_email.from_address = @mail.from.first.downcase
incoming_email.to_addresses = @mail.to.map(&:downcase).join(";") if @mail.to.present?
incoming_email.cc_addresses = @mail.cc.map(&:downcase).join(";") if @mail.cc.present?
end
end
def process_internal
raise AutoGeneratedEmailError if is_auto_generated?
body = select_body || ""
raise NoBodyDetectedError if body.blank? && !@mail.has_attachments?
user = find_or_create_user(from)
@incoming_email.update_columns(user_id: user.id)
raise InactiveUserError if !user.active && !user.staged
if post = find_related_post
create_reply(user: user, raw: body, post: post, topic: post.topic)
else
destination = destinations.first
raise BadDestinationAddress if destination.blank?
case destination[:type]
when :group
group = destination[:obj]
create_topic(user: user, raw: body, title: @mail.subject, archetype: Archetype.private_message, target_group_names: [group.name], skip_validations: true)
when :category
category = destination[:obj]
raise StrangersNotAllowedError if user.staged? && !category.email_in_allow_strangers
raise InsufficientTrustLevelError if !user.has_trust_level?(SiteSetting.email_in_min_trust)
create_topic(user: user, raw: body, title: @mail.subject, category: category.id)
when :reply
email_log = destination[:obj]
raise ReplyUserNotMatchingError if email_log.user_id != user.id
create_reply(user: user, raw: body, post: email_log.post, topic: email_log.post.topic)
end
end
end
def is_auto_generated?
@mail.return_path.blank? ||
@mail[:precedence].to_s[/list|junk|bulk|auto_reply/] ||
@mail.header.to_s[/auto-(submitted|replied|generated)/]
end
def select_body
text = nil
html = nil
if @mail.multipart?
text = fix_charset(@mail.text_part)
html = fix_charset(@mail.html_part)
elsif @mail.content_type.to_s["text/html"]
html = fix_charset(@mail)
else
text = fix_charset(@mail)
end
# prefer text over html
if text.present?
text_encoding = text.encoding
text = DiscourseEmailParser.parse_reply(text)
text = try_to_encode(text, text_encoding)
return text if text.present?
end
# clean the html if that's all we've got
if html.present?
html_encoding = html.encoding
html = Email::HtmlCleaner.new(html).output_html
html = DiscourseEmailParser.parse_reply(html)
html = try_to_encode(html, html_encoding)
return html if html.present?
end
end
def fix_charset(mail_part)
return nil if mail_part.blank? || mail_part.body.blank?
string = mail_part.body.to_s
# TODO (use charlock_holmes to properly detect encoding)
# 1) use the charset provided
if mail_part.charset.present?
fixed = try_to_encode(string, mail_part.charset)
return fixed if fixed.present?
end
# 2) default to UTF-8
try_to_encode(string, "UTF-8")
end
def try_to_encode(string, encoding)
string.encode("UTF-8", encoding)
rescue Encoding::InvalidByteSequenceError, Encoding::UndefinedConversionError
nil
end
def from
@from ||= @mail[:from].address_list.addresses.first
end
def find_or_create_user(address_field)
# decode the address field
address_field.decoded
# extract email and name
email = address_field.address.downcase
name = address_field.display_name.try(:to_s)
username = UserNameSuggester.sanitize_username(name) if name.present?
User.find_or_create_by(email: email) do |user|
user.username = UserNameSuggester.suggest(username.presence || email)
user.name = name.presence || User.suggest_name(email)
user.staged = true
end
end
def destinations
[ @mail.destinations,
[@mail[:x_forwarded_to]].flatten.compact.map(&:decoded),
[@mail[:delivered_to]].flatten.compact.map(&:decoded),
].flatten
.select(&:present?)
.uniq
.lazy
.map { |d| check_address(d) }
.drop_while(&:blank?)
end
def check_address(address)
# only check for a group/category when 'email_in' is enabled
if SiteSetting.email_in
group = Group.find_by_email(address)
return { address: address, type: :group, obj: group } if group
return { type: :group, obj: group } if group
category = Category.find_by_email(address)
return { address: address, type: :category, obj: category } if category
return { type: :category, obj: category } if category
end
# reply
match = reply_by_email_address_regex.match(address)
if match && match[1].present?
email_log = EmailLog.for(match[1])
return { address: address, type: :reply, obj: email_log }
return { type: :reply, obj: email_log } if email_log
end
end
@ -147,173 +201,89 @@ module Email
.gsub(Regexp.escape("%{reply_key}"), "([[:xdigit:]]{32})")
end
def parse_body(message)
body = select_body(message)
encoding = body.encoding
raise EmptyEmailError if body.strip.blank?
def find_related_post
message_ids = [@mail.in_reply_to, extract_references]
message_ids.flatten!
message_ids.select!(&:present?)
message_ids.uniq!
return if message_ids.empty?
body = discourse_email_trimmer(body)
raise EmptyEmailError if body.strip.blank?
body = DiscourseEmailParser.parse_reply(body)
raise EmptyEmailError if body.strip.blank?
body.force_encoding(encoding).encode("UTF-8")
IncomingEmail.where.not(post_id: nil)
.where(message_id: message_ids)
.first
.try(:post)
end
def select_body(message)
html = nil
if message.multipart?
text = fix_charset message.text_part
# prefer text over html
return text if text
html = fix_charset message.html_part
elsif message.content_type =~ /text\/html/
html = fix_charset message
def extract_references
if Array === @mail.references
@mail.references
elsif @mail.references.present?
@mail.references.split(/[\s,]/).map { |r| r.sub(/^</, "").sub(/>$/, "") }
end
if html
body = HtmlCleaner.new(html).output_html
else
body = fix_charset message
end
return body if @opts[:skip_sanity_check]
# Certain trigger phrases that means we didn't parse correctly
if body =~ /Content\-Type\:/ || body =~ /multipart\/alternative/ || body =~ /text\/plain/
raise EmptyEmailError
end
body
end
# Force encoding to UTF-8 on a Mail::Message or Mail::Part
def fix_charset(object)
return nil if object.nil?
if object.charset
object.body.decoded.force_encoding(object.charset.gsub(/utf8/i, "UTF-8")).encode("UTF-8").to_s
else
object.body.to_s
end
rescue
nil
end
REPLYING_HEADER_LABELS = ['From', 'Sent', 'To', 'Subject', 'In-Reply-To', 'Cc', 'Bcc', 'Date']
REPLYING_HEADER_REGEX = Regexp.union(REPLYING_HEADER_LABELS.map { |lbl| "#{lbl}:" })
def line_is_quote?(l)
l =~ /\A\s*\-{3,80}\s*\z/ ||
l =~ Regexp.new("\\A\\s*" + I18n.t('user_notifications.previous_discussion') + "\\s*\\Z") ||
(l =~ /via #{SiteSetting.title}(.*)\:$/) ||
# This one might be controversial but so many reply lines have years, times and end with a colon.
# Let's try it and see how well it works.
(l =~ /\d{4}/ && l =~ /\d:\d\d/ && l =~ /\:$/) ||
(l =~ /On [\w, ]+\d+.*wrote:/)
end
def discourse_email_trimmer(body)
lines = body.scrub.lines.to_a
range_start = 0
range_end = 0
# If we started with a quote, skip it
lines.each_with_index do |l, idx|
break unless line_is_quote?(l) or l =~ /^>/ or l.blank?
range_start = idx + 1
end
lines[range_start..-1].each_with_index do |l, idx|
break if line_is_quote?(l)
# Headers on subsequent lines
break if (0..2).all? { |off| lines[idx+off] =~ REPLYING_HEADER_REGEX }
# Headers on the same line
break if REPLYING_HEADER_LABELS.count { |lbl| l.include? lbl } >= 3
range_end = range_start + idx
end
lines[range_start..range_end].join.strip
end
private
def wrap_body_in_quote(user_email)
@body = "[quote=\"#{user_email}\"]\n#{@body}\n[/quote]"
end
def create_post_action(email_log, type)
PostActionCreator.new(email_log.user, email_log.post).perform(type)
rescue Discourse::InvalidAccess, PostAction::AlreadyActed => e
raise InvalidPostAction.new(e)
def likes
@likes ||= Set.new ["+1", I18n.t('post_action_types.like.title').downcase]
end
def post_action_for(body)
if ['+1', I18n.t('post_action_types.like.title').downcase].include? body.downcase
if likes.include?(body.strip.downcase)
PostActionType.types[:like]
end
end
def create_reply(email_log)
create_post_with_attachments(email_log.user,
raw: @body,
topic_id: email_log.topic_id,
reply_to_post_number: email_log.post.post_number)
def create_topic(options={})
create_post_with_attachments(options)
end
def create_new_topic(user, topic_options={})
topic_options[:raw] = @body
topic_options[:title] = @message.subject
def create_reply(options={})
raise TopicNotFoundError if options[:topic].nil? || options[:topic].trashed?
raise TopicClosedError if options[:topic].closed?
result = create_post_with_attachments(user, topic_options)
topic_id = result.post.present? ? result.post.topic_id : nil
EmailLog.create(
email_type: "topic_via_incoming_email",
to_address: user.email,
topic_id: topic_id,
user_id: user.id,
)
result
if post_action_type = post_action_for(options[:raw])
create_post_action(options[:user], options[:post], post_action_type)
else
options[:topic_id] = options[:post].try(:topic_id)
options[:reply_to_post_number] = options[:post].try(:post_number)
create_post_with_attachments(options)
end
end
def create_post_with_attachments(user, post_options={})
options = {
cooking_options: { traditional_markdown_linebreaks: true },
}.merge(post_options)
raw = options[:raw]
def create_post_action(user, post, type)
PostActionCreator.new(user, post).perform(type)
rescue PostAction::AlreadyActed
# it's cool, don't care
rescue Discourse::InvalidAccess => e
raise InvalidPostAction.new(e)
end
def create_post_with_attachments(options={})
# deal with attachments
@message.attachments.each do |attachment|
@mail.attachments.each do |attachment|
tmp = Tempfile.new("discourse-email-attachment")
begin
# read attachment
File.open(tmp.path, "w+b") { |f| f.write attachment.body.decoded }
# create the upload for the user
upload = Upload.create_for(user.id, tmp, attachment.filename, tmp.size)
upload = Upload.create_for(options[:user].id, tmp, attachment.filename, tmp.size)
if upload && upload.errors.empty?
# try to inline images
if attachment.content_type.start_with?("image/")
if raw =~ /\[image: Inline image \d+\]/
raw.sub!(/\[image: Inline image \d+\]/, attachment_markdown(upload))
next
end
if attachment.content_type.start_with?("image/") && options[:raw][/\[image: .+ \d+\]/]
options[:raw].sub!(/\[image: .+ \d+\]/, attachment_markdown(upload))
else
options[:raw] << "\n#{attachment_markdown(upload)}\n"
end
raw << "\n#{attachment_markdown(upload)}\n"
end
ensure
tmp.close!
tmp.try(:close!) rescue nil
end
end
options[:raw] = raw
post_options = {
cooking_options: { traditional_markdown_linebreaks: true },
}.merge(options)
create_post(user, options)
create_post(post_options)
end
def attachment_markdown(upload)
@ -324,20 +294,46 @@ module Email
end
end
def create_post(user, options)
# Mark the reply as incoming via email
def create_post(options={})
options[:via_email] = true
options[:raw_email] = @raw
options[:raw_email] = @raw_email
manager = NewPostManager.new(user, options)
# ensure posts aren't created in the future
options[:created_at] = [@mail.date, DateTime.now].min
manager = NewPostManager.new(options[:user], options)
result = manager.perform
if result.errors.present?
raise InvalidPost, result.errors.full_messages.join("\n")
end
raise InvalidPost, result.errors.full_messages.join("\n") if result.errors.any?
result
if result.post
@incoming_email.update_columns(topic_id: result.post.topic_id, post_id: result.post.id)
if result.post.topic && result.post.topic.private_message?
add_other_addresses(result.post.topic, options[:user])
end
end
end
def add_other_addresses(topic, sender)
%i(to cc bcc).each do |d|
if @mail[d] && @mail[d].address_list && @mail[d].address_list.addresses
@mail[d].address_list.addresses.each do |address|
begin
if user = find_or_create_user(address)
unless topic.topic_allowed_users.where(user_id: user.id).exists? &&
topic.topic_allowed_groups.where("group_id IN (SELECT group_id FROM group_users WHERE user_id = ?)", user.id).exists?
topic.topic_allowed_users.create!(user_id: user.id)
topic.add_small_action(sender, "invited_user", user.username)
end
end
rescue ActiveRecord::RecordInvalid
# don't care if user already allowed
end
end
end
end
end
end
end