FEATURE: Add more columns to outbound EmailLog (#13449)

FEATURE: Add more columns to outbound EmailLog (#13449)

This adds the following columns to EmailLog:

  • cc_addresses
  • cc_user_ids
  • topic_id
  • raw

This is to bring the EmailLog table closer in parity to IncomingEmail so it can be better utilized for Group SMTP and IMAP mailing.

The raw column contains the full content of the outbound email, but only if the new hidden site setting enable_raw_outbound_email_logging is enabled. Most sites do not need it, and it’s mostly required for IMAP and SMTP sending.

In the next pull request, there will be a migration to backfill topic_id on the EmailLog table, at which point we can remove the topic fallback method on EmailLog.

diff --git a/app/models/email_log.rb b/app/models/email_log.rb
index 0770698..2e93daa 100644
--- a/app/models/email_log.rb
+++ b/app/models/email_log.rb
@@ -18,8 +18,6 @@ class EmailLog < ActiveRecord::Base
   belongs_to :post
   belongs_to :smtp_group, class_name: 'Group'
 
-  has_one :topic, through: :post
-
   validates :email_type, :to_address, presence: true
 
   scope :bounced, -> { where(bounced: true) }
@@ -40,6 +38,10 @@ class EmailLog < ActiveRecord::Base
     User.where(id: user_id).update_all("last_emailed_at = CURRENT_TIMESTAMP") if user_id.present?
   end
 
+  def topic
+    @topic ||= self.topic_id.present? ? Topic.find_by(id: self.topic_id) : self.post&.topic
+  end
+
   def self.unique_email_per_post(post, user)
     return yield unless post && user
 
@@ -85,6 +87,14 @@ class EmailLog < ActiveRecord::Base
     super&.delete('-')
   end
 
+  def cc_users
+    return [] if !self.cc_user_ids
+    @cc_users ||= User.where(id: self.cc_user_ids)
+  end
+
+  def cc_addresses_split
+    @cc_addresses_split ||= self.cc_addresses&.split(";") || []
+  end
 end
 
 # == Schema Information
@@ -102,14 +112,18 @@ end
 #  bounced       :boolean          default(FALSE), not null
 #  message_id    :string
 #  smtp_group_id :integer
+#  cc_addresses  :text
+#  cc_user_ids   :integer          is an Array
+#  raw           :text
+#  topic_id      :integer
 #
 # Indexes
 #
-#  idx_email_logs_on_smtp_group_id  (smtp_group_id)
-#  index_email_logs_on_bounce_key   (bounce_key) UNIQUE WHERE (bounce_key IS NOT NULL)
-#  index_email_logs_on_bounced      (bounced)
-#  index_email_logs_on_created_at   (created_at)
-#  index_email_logs_on_message_id   (message_id)
-#  index_email_logs_on_post_id      (post_id)
-#  index_email_logs_on_user_id      (user_id)
+#  index_email_logs_on_bounce_key  (bounce_key) UNIQUE WHERE (bounce_key IS NOT NULL)
+#  index_email_logs_on_bounced     (bounced)
+#  index_email_logs_on_created_at  (created_at)
+#  index_email_logs_on_message_id  (message_id)
+#  index_email_logs_on_post_id     (post_id)
+#  index_email_logs_on_topic_id    (topic_id) WHERE (topic_id IS NOT NULL)
+#  index_email_logs_on_user_id     (user_id)
 #
diff --git a/db/migrate/20210621002201_add_columns_to_email_log_to_match_incoming_for_smtp_imap.rb b/db/migrate/20210621002201_add_columns_to_email_log_to_match_incoming_for_smtp_imap.rb
new file mode 100644
index 0000000..b9fea5f
--- /dev/null
+++ b/db/migrate/20210621002201_add_columns_to_email_log_to_match_incoming_for_smtp_imap.rb
@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+class AddColumnsToEmailLogToMatchIncomingForSmtpImap < ActiveRecord::Migration[6.1]
+  def up
+    add_column :email_logs, :cc_addresses, :text, null: true
+    add_column :email_logs, :cc_user_ids, :integer, array: true, null: true
+    add_column :email_logs, :raw, :text, null: true
+    add_column :email_logs, :topic_id, :integer, null: true
+
+    add_index :email_logs, :topic_id, where: 'topic_id IS NOT NULL'
+  end
+
+  def down
+    remove_column :email_logs, :cc_addresses if column_exists?(:email_logs, :cc_addresses)
+    remove_column :email_logs, :cc_user_ids if column_exists?(:email_logs, :cc_user_ids)
+    remove_column :email_logs, :raw if column_exists?(:email_logs, :raw)
+    remove_column :email_logs, :topic_id if column_exists?(:email_logs, :topic_id)
+
+    remove_index :email_logs, :topic_id if index_exists?(:email_logs, [:topic_id])
+  end
+end
diff --git a/lib/email/sender.rb b/lib/email/sender.rb
index 489af2f..75a3a97 100644
--- a/lib/email/sender.rb
+++ b/lib/email/sender.rb
@@ -92,6 +92,11 @@ module Email
         user_id: user_id
       )
 
+      if cc_addresses.any?
+        email_log.cc_addresses = cc_addresses.join(";")
+        email_log.cc_user_ids = User.with_email(cc_addresses).pluck(:id)
+      end
+
       host = Email::Sender.host_for(Discourse.base_url)
 
       post_id   = header_value('X-Discourse-Post-Id')
@@ -201,6 +206,7 @@ module Email
       end
 
       email_log.post_id = post_id if post_id.present?
+      email_log.topic_id = topic_id if topic_id.present?
 
       # Remove headers we don't need anymore
       @message.header['X-Discourse-Topic-Id'] = nil if topic_id.present?
@@ -243,7 +249,16 @@ module Email
 
       # Log when a message is being sent from a group SMTP address, so we
       # can debug deliverability issues.
-      email_log.smtp_group_id = smtp_group_id
+      if smtp_group_id
+        email_log.smtp_group_id = smtp_group_id
+
+        # Store contents of all outgoing emails using group SMTP
+        # for greater visibility and debugging. If the size of this
+        # gets out of hand, we should look into a group-level setting
+        # to enable this; size should be kept in check by regular purging
+        # of EmailLog though.
+        email_log.raw = Email::Cleaner.new(@message).execute
+      end
 
       DiscourseEvent.trigger(:before_email_send, @message, @email_type)
 
@@ -270,6 +285,12 @@ module Email
       end
     end
 
+    def cc_addresses
+      @cc_addresses ||= begin
+        @message.try(:cc) || []
+      end
+    end
+
     def self.host_for(base_url)
       host = "localhost"
       if base_url.present?
diff --git a/spec/components/email/sender_spec.rb b/spec/components/email/sender_spec.rb
index 3c56e93..fbdbacb 100644
--- a/spec/components/email/sender_spec.rb
+++ b/spec/components/email/sender_spec.rb
@@ -336,6 +336,7 @@ describe Email::Sender do
         expect(email_log.email_type).to eq('valid_type')
         expect(email_log.to_address).to eq('eviltrout@test.domain')
         expect(email_log.user_id).to be_blank
+        expect(email_log.raw).to eq(nil)
       end
 
       context 'when the email is sent using group SMTP credentials' do
@@ -355,7 +356,7 @@ describe Email::Sender do
           SiteSetting.enable_smtp = true
         end
 
-        it 'adds the group id to the email log' do
+        it 'adds the group id and raw content to the email log' do
           TopicAllowedGroup.create(topic: post.topic, group: group)
 
           email_sender.send
@@ -365,6 +366,7 @@ describe Email::Sender do
           expect(email_log.to_address).to eq(post.user.email)
           expect(email_log.user_id).to be_blank
           expect(email_log.smtp_group_id).to eq(group.id)
+          expect(email_log.raw).to include("Hello world")
         end
 
         it "does not add any of the mailing list headers" do
@@ -393,6 +395,7 @@ describe Email::Sender do
       it 'should create the right log' do
         email_sender.send
         expect(email_log.post_id).to eq(post.id)
+        expect(email_log.topic_id).to eq(topic.id)
         expect(email_log.topic.id).to eq(topic.id)
       end
     end
@@ -687,4 +690,30 @@ describe Email::Sender do
     end
   end
 
+  context "with cc addresses" do
+    let(:message) do
+      message = Mail::Message.new to: 'eviltrout@test.domain', body: 'test body', cc: 'someguy@test.com;otherguy@xyz.com'
+      message.stubs(:deliver_now)
+      message
+    end
+
+    fab!(:user) { Fabricate(:user) }
+    let(:email_sender) { Email::Sender.new(message, :valid_type, user) }
+
+    it "logs the cc addresses in the email log (but not users if they do not match the emails)" do
+      email_sender.send
+      email_log = EmailLog.last
+      expect(email_log.cc_addresses).to eq("someguy@test.com;otherguy@xyz.com")
+      expect(email_log.cc_users).to eq([])
+    end
+
+    it "logs the cc users if they match the emails" do
+      user1 = Fabricate(:user, email: "someguy@test.com")
+      user2 = Fabricate(:user, email: "otherguy@xyz.com")
+      email_sender.send
+      email_log = EmailLog.last
+      expect(email_log.cc_addresses).to eq("someguy@test.com;otherguy@xyz.com")
+      expect(email_log.cc_users).to match_array([user1, user2])
+    end
+  end
 end
diff --git a/spec/models/email_log_spec.rb b/spec/models/email_log_spec.rb
index e77c8ad..0e6206b 100644
--- a/spec/models/email_log_spec.rb

[... diff too long, it was truncated ...]

GitHub sha: 5222247746340a2f915b17ae15f156e6324ae79a

This commit appears in #13449 which was approved by tgxworld. It was merged by martin.