FEATURE: send max 200 emails every minute for bulk invites (#7875)

FEATURE: send max 200 emails every minute for bulk invites (#7875)

DEV: deprecate invite.via_email in favor of invite.emailed_status

This commit adds a new column emailed_status in invites table for tracking email sending status. 0 - not required 1 - pending 2 - bulk pending 3 - sending 4 - sent

For normal email invites, invite record is created with emailed_status set to ‘pending’.

When bulk invites are sent invite record is created with emailed_status set to ‘bulk pending’.

For invites that generates link, invite record is created with emailed_status set to ‘not required’.

When invite email is in queue emailed_status is updated to ‘sending’

Once the email is sent via InviteEmail job the invite emailed_status is updated to ‘sent’.

diff --git a/app/jobs/regular/bulk_invite.rb b/app/jobs/regular/bulk_invite.rb
index 7c3d1a6..f7ccd44 100644
--- a/app/jobs/regular/bulk_invite.rb
+++ b/app/jobs/regular/bulk_invite.rb
@@ -23,8 +23,13 @@ module Jobs
       @current_user = User.find_by(id: args[:current_user_id])
       raise Discourse::InvalidParameters.new(:current_user_id) unless @current_user
       @guardian = Guardian.new(@current_user)
+      @total_invites = invites.length
 
       process_invites(invites)
+
+      if @total_invites > Invite::BULK_INVITE_EMAIL_LIMIT
+        Jobs.enqueue(:process_bulk_invite_emails)
+      end
     ensure
       notify_user
     end
@@ -104,7 +109,15 @@ module Jobs
             end
           end
         else
-          Invite.invite_by_email(email, @current_user, topic, groups.map(&:id))
+          if @total_invites > Invite::BULK_INVITE_EMAIL_LIMIT
+            invite = Invite.create_invite_by_email(email, @current_user,
+              topic: topic,
+              group_ids: groups.map(&:id),
+              emailed_status: Invite.emailed_status_types[:bulk_pending]
+            )
+          else
+            Invite.invite_by_email(email, @current_user, topic, groups.map(&:id))
+          end
         end
       rescue => e
         save_log "Error inviting '#{email}' -- #{Rails::Html::FullSanitizer.new.sanitize(e.message)}"
diff --git a/app/jobs/regular/invite_email.rb b/app/jobs/regular/invite_email.rb
index 0db080c..c91e67a 100644
--- a/app/jobs/regular/invite_email.rb
+++ b/app/jobs/regular/invite_email.rb
@@ -15,8 +15,10 @@ module Jobs
 
       message = InviteMailer.send_invite(invite)
       Email::Sender.new(message, :invite).send
-    end
 
+      if invite.emailed_status != Invite.emailed_status_types[:not_required]
+        invite.update_column(:emailed_status, Invite.emailed_status_types[:sent])
+      end
+    end
   end
-
 end
diff --git a/app/jobs/regular/process_bulk_invite_emails.rb b/app/jobs/regular/process_bulk_invite_emails.rb
new file mode 100644
index 0000000..f4d057e
--- /dev/null
+++ b/app/jobs/regular/process_bulk_invite_emails.rb
@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require_dependency 'email/sender'
+
+module Jobs
+
+  class ProcessBulkInviteEmails < Jobs::Base
+
+    def execute(args)
+      pending_invite_ids = Invite.where(emailed_status: Invite.emailed_status_types[:bulk_pending]).limit(Invite::BULK_INVITE_EMAIL_LIMIT).pluck(:id)
+
+      if pending_invite_ids.length > 0
+        Invite.where(id: pending_invite_ids).update_all(emailed_status: Invite.emailed_status_types[:sending])
+        pending_invite_ids.each do |invite_id|
+          Jobs.enqueue(:invite_email, invite_id: invite_id)
+        end
+        Jobs.enqueue_in(1.minute, :process_bulk_invite_emails)
+      end
+    end
+  end
+end
diff --git a/app/models/invite.rb b/app/models/invite.rb
index 1a909a2..e92d5cd 100644
--- a/app/models/invite.rb
+++ b/app/models/invite.rb
@@ -3,10 +3,16 @@
 require_dependency 'rate_limiter'
 
 class Invite < ActiveRecord::Base
+  self.ignored_columns = %w{
+    via_email
+  }
+
   class UserExists < StandardError; end
   include RateLimiter::OnCreateRecord
   include Trashable
 
+  BULK_INVITE_EMAIL_LIMIT = 200
+
   rate_limit :limit_invites_per_day
 
   belongs_to :user
@@ -31,6 +37,10 @@ class Invite < ActiveRecord::Base
   validate :user_doesnt_already_exist
   attr_accessor :email_already_exists
 
+  def self.emailed_status_types
+    @emailed_status_types ||= Enum.new(not_required: 0, pending: 1, bulk_pending: 2, sending: 3, sent: 4)
+  end
+
   def user_doesnt_already_exist
     @email_already_exists = false
     return if email.blank?
@@ -66,7 +76,7 @@ class Invite < ActiveRecord::Base
       topic: topic,
       group_ids: group_ids,
       custom_message: custom_message,
-      send_email: true
+      emailed_status: emailed_status_types[:pending]
     )
   end
 
@@ -75,7 +85,7 @@ class Invite < ActiveRecord::Base
     invite = create_invite_by_email(email, invited_by,
       topic: topic,
       group_ids: group_ids,
-      send_email: false
+      emailed_status: emailed_status_types[:not_required]
     )
 
     "#{Discourse.base_url}/invites/#{invite.invite_key}" if invite
@@ -89,8 +99,8 @@ class Invite < ActiveRecord::Base
 
     topic = opts[:topic]
     group_ids = opts[:group_ids]
-    send_email = opts[:send_email].nil? ? true : opts[:send_email]
     custom_message = opts[:custom_message]
+    emailed_status = opts[:emailed_status] || emailed_status_types[:pending]
     lower_email = Email.downcase(email)
 
     if user = find_user_by_email(lower_email)
@@ -112,16 +122,20 @@ class Invite < ActiveRecord::Base
     end
 
     if invite
+      if invite.emailed_status == Invite.emailed_status_types[:not_required]
+        emailed_status = invite.emailed_status
+      end
+
       invite.update_columns(
         created_at: Time.zone.now,
         updated_at: Time.zone.now,
-        via_email: invite.via_email && send_email
+        emailed_status: emailed_status
       )
     else
       create_args = {
         invited_by: invited_by,
         email: lower_email,
-        via_email: send_email
+        emailed_status: emailed_status
       }
 
       create_args[:moderator] = true if opts[:moderator]
@@ -143,7 +157,10 @@ class Invite < ActiveRecord::Base
       end
     end
 
-    Jobs.enqueue(:invite_email, invite_id: invite.id) if send_email
+    if emailed_status == emailed_status_types[:pending]
+      invite.update_column(:emailed_status, Invite.emailed_status_types[:sending])
+      Jobs.enqueue(:invite_email, invite_id: invite.id)
+    end
 
     invite.reload
     invite
@@ -261,10 +278,11 @@ end
 #  invalidated_at :datetime
 #  moderator      :boolean          default(FALSE), not null
 #  custom_message :text
-#  via_email      :boolean          default(FALSE), not null
+#  emailed_status :integer
 #
 # Indexes
 #
 #  index_invites_on_email_and_invited_by_id  (email,invited_by_id)
+#  index_invites_on_emailed_status           (emailed_status)
 #  index_invites_on_invite_key               (invite_key) UNIQUE
 #
diff --git a/app/models/invite_redeemer.rb b/app/models/invite_redeemer.rb
index d41662b..755241e 100644
--- a/app/models/invite_redeemer.rb
+++ b/app/models/invite_redeemer.rb
@@ -60,7 +60,7 @@ InviteRedeemer = Struct.new(:invite, :username, :name, :password, :user_custom_f
 
     user.save!
 
-    if invite.via_email
+    if invite.emailed_status != Invite.emailed_status_types[:not_required]
       user.email_tokens.create!(email: user.email)
       user.activate
     end
diff --git a/app/models/user_second_factor.rb b/app/models/user_second_factor.rb
index 0d3d8ee..88dcfef 100644
--- a/app/models/user_second_factor.rb
+++ b/app/models/user_second_factor.rb
@@ -44,6 +44,7 @@ end
 #  last_used  :datetime
 #  created_at :datetime         not null
 #  updated_at :datetime         not null
+#  name       :string
 #
 # Indexes
 #
diff --git a/db/migrate/20190711154946_add_emailed_status_to_invite.rb b/db/migrate/20190711154946_add_emailed_status_to_invite.rb
new file mode 100644
index 0000000..9123f50
--- /dev/null
+++ b/db/migrate/20190711154946_add_emailed_status_to_invite.rb
@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+class AddEmailedStatusToInvite < ActiveRecord::Migration[5.2]
+  def change
+    add_column :invites, :emailed_status, :integer
+    add_index :invites, :emailed_status
+
+    DB.exec <<~SQL
+      UPDATE invites
+      SET emailed_status = 0
+      WHERE via_email = false
+    SQL
+  end
+end
diff --git a/db/post_migrate/20190716124050_remove_via_email_from_invite.rb b/db/post_migrate/20190716124050_remove_via_email_from_invite.rb
new file mode 100644
index 0000000..676ea11
--- /dev/null
+++ b/db/post_migrate/20190716124050_remove_via_email_from_invite.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require 'migration/column_dropper'
+
+class RemoveViaEmailFromInvite < ActiveRecord::Migration[5.2]
+  def up
+    Migration::ColumnDropper.execute_drop(:invites, %i{via_email})
+  end
+
+  def down

[... diff too long, it was truncated ...]

GitHub sha: eb9155f3

1 Like

This commit has been mentioned on Discourse Meta. There might be relevant details there: