FIX: Ensure CORS rules exist for S3 using rake task (#14802)

FIX: Ensure CORS rules exist for S3 using rake task (#14802)

This commit introduces a new s3:ensure_cors_rules rake task that is run as a prerequisite to s3:upload_assets. This rake task calls out to the S3CorsRulesets class to ensure that the 3 relevant sets of CORS rules are applied, depending on site settings:

  • assets
  • direct S3 backups
  • direct S3 uploads

This works for both Global S3 settings and Database S3 settings (the latter set directly via SiteSetting).

As it is, only one rule can be applied, which is generally the assets rule as it is called first. This commit changes the ensure_cors! method to be able to apply new rules as well as the existing ones.

This commit also slightly changes the existing rules to cover direct S3 uploads via uppy, especially multipart, which requires some more headers.

diff --git a/app/controllers/uploads_controller.rb b/app/controllers/uploads_controller.rb
index 142b6a0..b95b33b 100644
--- a/app/controllers/uploads_controller.rb
+++ b/app/controllers/uploads_controller.rb
@@ -224,22 +224,15 @@ class UploadsController < ApplicationController
       )
     end
 
-    metadata = parse_allowed_metadata(params[:metadata])
-
-    url = Discourse.store.signed_url_for_temporary_upload(
-      file_name, metadata: metadata
-    )
-    key = Discourse.store.path_from_url(url)
-
-    upload_stub = ExternalUploadStub.create!(
-      key: key,
-      created_by: current_user,
-      original_filename: file_name,
+    external_upload_data = ExternalUploadManager.create_direct_upload(
+      current_user: current_user,
+      file_name: file_name,
+      file_size: file_size,
       upload_type: type,
-      filesize: file_size
+      metadata: parse_allowed_metadata(params[:metadata])
     )
 
-    render json: { url: url, key: key, unique_identifier: upload_stub.unique_identifier }
+    render json: external_upload_data
   end
 
   def complete_external_upload
@@ -307,7 +300,6 @@ class UploadsController < ApplicationController
     file_name = params.require(:file_name)
     file_size = params.require(:file_size).to_i
     upload_type = params.require(:upload_type)
-    content_type = MiniMime.lookup_by_filename(file_name)&.content_type
 
     if file_size_too_big?(file_name, file_size)
       return render_json_error(
@@ -316,11 +308,13 @@ class UploadsController < ApplicationController
       )
     end
 
-    metadata = parse_allowed_metadata(params[:metadata])
-
     begin
-      multipart_upload = Discourse.store.create_multipart(
-        file_name, content_type, metadata: metadata
+      external_upload_data = ExternalUploadManager.create_direct_multipart_upload(
+        current_user: current_user,
+        file_name: file_name,
+        file_size: file_size,
+        upload_type: upload_type,
+        metadata: parse_allowed_metadata(params[:metadata])
       )
     rescue Aws::S3::Errors::ServiceError => err
       return render_json_error(
@@ -329,21 +323,7 @@ class UploadsController < ApplicationController
       )
     end
 
-    upload_stub = ExternalUploadStub.create!(
-      key: multipart_upload[:key],
-      created_by: current_user,
-      original_filename: file_name,
-      upload_type: upload_type,
-      external_upload_identifier: multipart_upload[:upload_id],
-      multipart: true,
-      filesize: file_size
-    )
-
-    render json: {
-      external_upload_identifier: upload_stub.external_upload_identifier,
-      key: upload_stub.key,
-      unique_identifier: upload_stub.unique_identifier
-    }
+    render json: external_upload_data
   end
 
   def batch_presign_multipart_parts
diff --git a/app/services/external_upload_manager.rb b/app/services/external_upload_manager.rb
index e5c032b..d435723 100644
--- a/app/services/external_upload_manager.rb
+++ b/app/services/external_upload_manager.rb
@@ -20,6 +20,48 @@ class ExternalUploadManager
     Discourse.redis.get("#{BAN_USER_REDIS_PREFIX}#{user.id}") == "1"
   end
 
+  def self.create_direct_upload(current_user:, file_name:, file_size:, upload_type:, metadata: {})
+    url = Discourse.store.signed_url_for_temporary_upload(
+      file_name, metadata: metadata
+    )
+    key = Discourse.store.path_from_url(url)
+
+    upload_stub = ExternalUploadStub.create!(
+      key: key,
+      created_by: current_user,
+      original_filename: file_name,
+      upload_type: upload_type,
+      filesize: file_size
+    )
+
+    { url: url, key: key, unique_identifier: upload_stub.unique_identifier }
+  end
+
+  def self.create_direct_multipart_upload(
+    current_user:, file_name:, file_size:, upload_type:, metadata: {}
+  )
+    content_type = MiniMime.lookup_by_filename(file_name)&.content_type
+    multipart_upload = Discourse.store.create_multipart(
+      file_name, content_type, metadata: metadata
+    )
+
+    upload_stub = ExternalUploadStub.create!(
+      key: multipart_upload[:key],
+      created_by: current_user,
+      original_filename: file_name,
+      upload_type: upload_type,
+      external_upload_identifier: multipart_upload[:upload_id],
+      multipart: true,
+      filesize: file_size
+    )
+
+    {
+      external_upload_identifier: upload_stub.external_upload_identifier,
+      key: upload_stub.key,
+      unique_identifier: upload_stub.unique_identifier
+    }
+  end
+
   def initialize(external_upload_stub, upload_create_opts = {})
     @external_upload_stub = external_upload_stub
     @upload_create_opts = upload_create_opts
diff --git a/lib/backup_restore/s3_backup_store.rb b/lib/backup_restore/s3_backup_store.rb
index cb2fcd7..016a2bf 100644
--- a/lib/backup_restore/s3_backup_store.rb
+++ b/lib/backup_restore/s3_backup_store.rb
@@ -44,7 +44,11 @@ module BackupRestore
       obj = @s3_helper.object(filename)
       raise BackupFileExists.new if obj.exists?
 
-      ensure_cors!
+      # TODO (martin) We can remove this at a later date when we move this
+      # ensure CORS for backups and direct uploads to a post-site-setting
+      # change event, so the rake task doesn't have to be run manually.
+      @s3_helper.ensure_cors!([S3CorsRulesets::BACKUP_DIRECT_UPLOAD])
+
       presigned_url(obj, :put, UPLOAD_URL_EXPIRES_AFTER_SECONDS)
     rescue Aws::Errors::ServiceError => e
       Rails.logger.warn("Failed to generate upload URL for S3: #{e.message.presence || e.class.name}")
@@ -82,10 +86,6 @@ module BackupRestore
       obj.presigned_url(method, expires_in: expires_in_seconds)
     end
 
-    def ensure_cors!
-      @s3_helper.ensure_cors!([S3CorsRulesets::BACKUP_DIRECT_UPLOAD])
-    end
-
     def cleanup_allowed?
       !SiteSetting.s3_disable_cleanup
     end
diff --git a/lib/s3_cors_rulesets.rb b/lib/s3_cors_rulesets.rb
index 88f76f3..d982d18 100644
--- a/lib/s3_cors_rulesets.rb
+++ b/lib/s3_cors_rulesets.rb
@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_dependency "s3_helper"
+
 class S3CorsRulesets
   ASSETS = {
     allowed_headers: ["Authorization"],
@@ -10,8 +12,59 @@ class S3CorsRulesets
 
   BACKUP_DIRECT_UPLOAD = {
     allowed_headers: ["*"],
-    allowed_methods: ["PUT"],
-    allowed_origins: [Discourse.base_url_no_prefix],
+    expose_headers: ["ETag"],
+    allowed_methods: ["GET", "HEAD", "PUT"],
+    allowed_origins: ["*"],
+    max_age_seconds: 3000
+  }.freeze
+
+  DIRECT_UPLOAD = {
+    allowed_headers: ["Authorization", "Content-Disposition", "Content-Type"],
+    expose_headers: ["ETag"],
+    allowed_methods: ["GET", "HEAD", "PUT"],
+    allowed_origins: ["*"],
     max_age_seconds: 3000
   }.freeze
+
+  ##
+  # Used by the s3:ensure_cors_rules rake task to make sure the
+  # relevant CORS rules are applied to allow for direct uploads to
+  # S3, and in the case of assets rules so there are fonts and other
+  # public assets for the site loaded correctly.
+  #
+  # The use_db_s3_config param comes from ENV, and if the S3 client
+  # is not provided it is initialized by the S3Helper.
+  def self.sync(use_db_s3_config:, s3_client: nil)
+    return if !SiteSetting.s3_install_cors_rule
+    return if !(GlobalSetting.use_s3? || SiteSetting.enable_s3_uploads)
+
+    assets_rules_applied = false
+    backup_rules_applied = false
+    direct_upload_rules_applied = false
+
+    s3_helper = S3Helper.build_from_config(
+      s3_client: s3_client, use_db_s3_config: use_db_s3_config
+    )
+    puts "Attempting to apply ASSETS S3 CORS ruleset in bucket #{s3_helper.s3_bucket_name}."
+    assets_rules_applied = s3_helper.ensure_cors!([S3CorsRulesets::ASSETS])
+
+    if SiteSetting.enable_backups? && SiteSetting.backup_location == BackupLocationSiteSetting::S3
+      backup_s3_helper = S3Helper.build_from_config(
+        s3_client: s3_client, use_db_s3_config: use_db_s3_config, for_backup: true
+      )

[... diff too long, it was truncated ...]

GitHub sha: 9a72a0945f87e11542d884f6f103dd21779eaeb5

This commit appears in #14802 which was approved by tgxworld. It was merged by martin.