FEATURE: option to skip posts with ignored missing uploads

FEATURE: option to skip posts with ignored missing uploads

diff --git a/app/models/post.rb b/app/models/post.rb
index 65187ae..d5d56b7 100644
--- a/app/models/post.rb
+++ b/app/models/post.rb
@@ -64,10 +64,12 @@ class Post < ActiveRecord::Base
   BROKEN_IMAGES     ||= "broken_images".freeze
   DOWNLOADED_IMAGES ||= "downloaded_images".freeze
   MISSING_UPLOADS ||= "missing uploads".freeze
+  MISSING_UPLOADS_IGNORED ||= "missing uploads ignored".freeze
 
   SHORT_POST_CHARS ||= 1200
 
   register_custom_field_type(MISSING_UPLOADS, :json)
+  register_custom_field_type(MISSING_UPLOADS_IGNORED, :boolean)
 
   scope :private_posts_for_user, ->(user) {
     where("posts.topic_id IN (SELECT topic_id
@@ -918,8 +920,13 @@ class Post < ActiveRecord::Base
     PostCustomField.where(name: Post::MISSING_UPLOADS).delete_all
     missing_uploads = []
     missing_post_uploads = {}
+    query = Post
+              .have_uploads
+              .joins("LEFT JOIN post_custom_fields ON posts.id = post_custom_fields.post_id AND post_custom_fields.name = '#{Post::MISSING_UPLOADS_IGNORED}'")
+              .where("post_custom_fields.id IS NULL")
+              .select(:id, :cooked)
 
-    Post.have_uploads.select(:id, :cooked).find_in_batches do |posts|
+    query.find_in_batches do |posts|
       ids = posts.pluck(:id)
       sha1s = Upload.joins(:post_uploads).where("post_uploads.post_id >= ? AND post_uploads.post_id <= ?", ids.min, ids.max).pluck(:sha1)
 
diff --git a/db/migrate/20190508193900_add_missing_uploads_ignored_index_to_post_custom_fields.rb b/db/migrate/20190508193900_add_missing_uploads_ignored_index_to_post_custom_fields.rb
new file mode 100644
index 0000000..2672e17
--- /dev/null
+++ b/db/migrate/20190508193900_add_missing_uploads_ignored_index_to_post_custom_fields.rb
@@ -0,0 +1,5 @@
+class AddMissingUploadsIgnoredIndexToPostCustomFields < ActiveRecord::Migration[5.2]
+  def change
+    add_index :post_custom_fields, :post_id, unique: true, where: "name = 'missing uploads ignored'", name: "index_post_id_where_missing_uploads_ignored"
+  end
+end
diff --git a/lib/tasks/posts.rake b/lib/tasks/posts.rake
index 241dbbf..e4adc80 100644
--- a/lib/tasks/posts.rake
+++ b/lib/tasks/posts.rake
@@ -447,3 +447,7 @@ task 'posts:missing_uploads' => :environment do
     puts "#{missing[:post_uploads].count} of #{Post.count} posts are affected.", ""
   end
 end
+
+desc 'Finds missing post upload records from cooked HTML content'
+task 'posts:missing_uploads' => :environment do
+end
diff --git a/spec/tasks/posts_spec.rb b/spec/tasks/posts_spec.rb
index 24f12fd..0188fae 100644
--- a/spec/tasks/posts_spec.rb
+++ b/spec/tasks/posts_spec.rb
@@ -70,5 +70,17 @@ RSpec.describe "Post rake tasks" do
       post.reload
       expect(post.custom_fields[Post::MISSING_UPLOADS]).to eq([url])
     end
+
+    it 'should skip all the posts with "ignored" custom field' do
+      post = Fabricate(:post, raw: "A sample post <img src='#{url}'>")
+      post.custom_fields[Post::MISSING_UPLOADS_IGNORED] = true
+      post.save_custom_fields
+      upload.destroy!
+
+      Rake::Task['posts:missing_uploads'].invoke
+
+      post.reload
+      expect(post.custom_fields[Post::MISSING_UPLOADS]).to be_nil
+    end
   end
 end

GitHub sha: 87cd4701

@tgxworld I need you to review this. Is this okay to mark posts as ignored?

Not sure if I am missing something here but is this blank rake task corrrect?

@vinothkannans It doesn’t look like we’re making posts as MISSING_UPLOADS_IGNORED?

Oops! I should remove that. It is a leftover from my local testing.

For now that’s a manual process. We have to investigate the issue and mark it as “ignored” if there is a known problem. I will create a rake task if needed.

1 Like