FEATURE: Multisite support for S3 backup store (#6700)

FEATURE: Multisite support for S3 backup store (#6700)

From 99117d664cffec80874d4bb71390a7470d3c492c Mon Sep 17 00:00:00 2001
From: Gerhard Schlager <mail@gerhard-schlager.at>
Date: Wed, 5 Dec 2018 03:10:39 +0100
Subject: [PATCH] FEATURE: Multisite support for S3 backup store (#6700)


diff --git a/lib/backup_restore/backuper.rb b/lib/backup_restore/backuper.rb
index f03d3dd..376c0a8 100644
--- a/lib/backup_restore/backuper.rb
+++ b/lib/backup_restore/backuper.rb
@@ -83,7 +83,7 @@ module BackupRestore
       @timestamp = Time.now.strftime("%Y-%m-%d-%H%M%S")
       @tmp_directory = File.join(Rails.root, "tmp", "backups", @current_db, @timestamp)
       @dump_filename = File.join(@tmp_directory, BackupRestore::DUMP_FILE)
-      @archive_directory = BackupRestore::LocalBackupStore.base_directory(@current_db)
+      @archive_directory = BackupRestore::LocalBackupStore.base_directory(db: @current_db)
       filename = @filename_override || "#{SiteSetting.title.parameterize}-#{@timestamp}"
       @archive_basename = File.join(@archive_directory, "#{filename}-#{BackupRestore::VERSION_PREFIX}#{BackupRestore.current_version}")
 
diff --git a/lib/backup_restore/local_backup_store.rb b/lib/backup_restore/local_backup_store.rb
index 8b0148e..fce72d2 100644
--- a/lib/backup_restore/local_backup_store.rb
+++ b/lib/backup_restore/local_backup_store.rb
@@ -3,9 +3,11 @@ require_dependency "disk_space"
 
 module BackupRestore
   class LocalBackupStore < BackupStore
-    def self.base_directory(current_db = nil)
-      current_db ||= RailsMultisite::ConnectionManagement.current_db
-      base_directory = File.join(Rails.root, "public", "backups", current_db)
+    def self.base_directory(db: nil, root_directory: nil)
+      current_db = db || RailsMultisite::ConnectionManagement.current_db
+      root_directory ||= File.join(Rails.root, "public", "backups")
+
+      base_directory = File.join(root_directory, current_db)
       FileUtils.mkdir_p(base_directory) unless Dir.exists?(base_directory)
       base_directory
     end
@@ -15,7 +17,7 @@ module BackupRestore
     end
 
     def initialize(opts = {})
-      @base_directory = opts[:base_directory] || LocalBackupStore.base_directory
+      @base_directory = LocalBackupStore.base_directory(root_directory: opts[:root_directory])
     end
 
     def remote?
diff --git a/lib/backup_restore/s3_backup_store.rb b/lib/backup_restore/s3_backup_store.rb
index cbbc916..2fae590 100644
--- a/lib/backup_restore/s3_backup_store.rb
+++ b/lib/backup_restore/s3_backup_store.rb
@@ -5,11 +5,12 @@ module BackupRestore
   class S3BackupStore < BackupStore
     DOWNLOAD_URL_EXPIRES_AFTER_SECONDS ||= 15
     UPLOAD_URL_EXPIRES_AFTER_SECONDS ||= 21_600 # 6 hours
+    MULTISITE_PREFIX = "backups"
 
     def initialize(opts = {})
       s3_options = S3Helper.s3_options(SiteSetting)
       s3_options.merge!(opts[:s3_options]) if opts[:s3_options]
-      @s3_helper = S3Helper.new(SiteSetting.s3_backup_bucket, '', s3_options)
+      @s3_helper = S3Helper.new(s3_bucket_name_with_prefix, '', s3_options)
     end
 
     def remote?
@@ -91,5 +92,13 @@ module BackupRestore
     def cleanup_allowed?
       !SiteSetting.s3_disable_cleanup
     end
+
+    def s3_bucket_name_with_prefix
+      if Rails.configuration.multisite
+        File.join(SiteSetting.s3_backup_bucket, MULTISITE_PREFIX, RailsMultisite::ConnectionManagement.current_db)
+      else
+        SiteSetting.s3_backup_bucket
+      end
+    end
   end
 end
diff --git a/spec/lib/backup_restore/local_backup_store_spec.rb b/spec/lib/backup_restore/local_backup_store_spec.rb
index efe8758..bd5f627 100644
--- a/spec/lib/backup_restore/local_backup_store_spec.rb
+++ b/spec/lib/backup_restore/local_backup_store_spec.rb
@@ -4,19 +4,19 @@ require_relative 'shared_examples_for_backup_store'
 
 describe BackupRestore::LocalBackupStore do
   before(:all) do
-    @base_directory = Dir.mktmpdir
+    @root_directory = Dir.mktmpdir
     @paths = []
   end
 
   after(:all) do
-    FileUtils.remove_dir(@base_directory, true)
+    FileUtils.remove_dir(@root_directory, true)
   end
 
   before do
     SiteSetting.backup_location = BackupLocationSiteSetting::LOCAL
   end
 
-  subject(:store) { BackupRestore::BackupStore.create(base_directory: @base_directory) }
+  subject(:store) { BackupRestore::BackupStore.create(root_directory: @root_directory) }
   let(:expected_type) { BackupRestore::LocalBackupStore }
 
   it_behaves_like "backup store"
@@ -26,10 +26,13 @@ describe BackupRestore::LocalBackupStore do
   end
 
   def create_backups
-    create_file(filename: "b.tar.gz", last_modified: "2018-09-13T15:10:00Z", size_in_bytes: 17)
-    create_file(filename: "a.tgz", last_modified: "2018-02-11T09:27:00Z", size_in_bytes: 29)
-    create_file(filename: "r.sql.gz", last_modified: "2017-12-20T03:48:00Z", size_in_bytes: 11)
-    create_file(filename: "no-backup.txt", last_modified: "2018-09-05T14:27:00Z", size_in_bytes: 12)
+    create_file(db_name: "default", filename: "b.tar.gz", last_modified: "2018-09-13T15:10:00Z", size_in_bytes: 17)
+    create_file(db_name: "default", filename: "a.tgz", last_modified: "2018-02-11T09:27:00Z", size_in_bytes: 29)
+    create_file(db_name: "default", filename: "r.sql.gz", last_modified: "2017-12-20T03:48:00Z", size_in_bytes: 11)
+    create_file(db_name: "default", filename: "no-backup.txt", last_modified: "2018-09-05T14:27:00Z", size_in_bytes: 12)
+
+    create_file(db_name: "second", filename: "multi-2.tar.gz", last_modified: "2018-11-27T03:16:54Z", size_in_bytes: 19)
+    create_file(db_name: "second", filename: "multi-1.tar.gz", last_modified: "2018-11-26T03:17:09Z", size_in_bytes: 22)
   end
 
   def remove_backups
@@ -37,8 +40,11 @@ describe BackupRestore::LocalBackupStore do
     @paths.clear
   end
 
-  def create_file(filename:, last_modified:, size_in_bytes:)
-    path = File.join(@base_directory, filename)
+  def create_file(db_name:, filename:, last_modified:, size_in_bytes:)
+    path = File.join(@root_directory, db_name)
+    Dir.mkdir(path) unless Dir.exists?(path)
+
+    path = File.join(path, filename)
     return if File.exists?(path)
 
     @paths << path
@@ -49,8 +55,8 @@ describe BackupRestore::LocalBackupStore do
     File.utime(time, time, path)
   end
 
-  def source_regex(filename)
-    path = File.join(@base_directory, filename)
+  def source_regex(db_name, filename, multisite:)
+    path = File.join(@root_directory, db_name, filename)
     /^#{Regexp.escape(path)}$/
   end
 end
diff --git a/spec/lib/backup_restore/s3_backup_store_spec.rb b/spec/lib/backup_restore/s3_backup_store_spec.rb
index 33cfa94..3020cab 100644
--- a/spec/lib/backup_restore/s3_backup_store_spec.rb
+++ b/spec/lib/backup_restore/s3_backup_store_spec.rb
@@ -9,22 +9,32 @@ describe BackupRestore::S3BackupStore do
 
     @objects = []
 
-    @s3_client.stub_responses(:list_objects, -> (context) do
+    def expected_prefix
+      Rails.configuration.multisite ? "backups/#{RailsMultisite::ConnectionManagement.current_db}/" : ""
+    end
+
+    def check_context(context)
       expect(context.params[:bucket]).to eq(SiteSetting.s3_backup_bucket)
-      expect(context.params[:prefix]).to be_blank
+      expect(context.params[:key]).to start_with(expected_prefix) if context.params.key?(:key)
+      expect(context.params[:prefix]).to eq(expected_prefix) if context.params.key?(:prefix)
+    end
+
+    @s3_client.stub_responses(:list_objects, -> (context) do
+      check_context(context)
 
-      { contents: @objects }
+      { contents: objects_with_prefix(context) }
     end)
 
     @s3_client.stub_responses(:delete_object, -> (context) do
-      expect(context.params[:bucket]).to eq(SiteSetting.s3_backup_bucket)
+      check_context(context)
+
       expect do
         @objects.delete_if { |obj| obj[:key] == context.params[:key] }
       end.to change { @objects }
     end)
 
     @s3_client.stub_responses(:head_object, -> (context) do
-      expect(context.params[:bucket]).to eq(SiteSetting.s3_backup_bucket)
+      check_context(context)
 
       i

GitHub

2 Likes

Usually I use https://ruby-doc.org/stdlib-1.9.3/libdoc/fileutils/rdoc/FileUtils.html#method-c-mkdir_p

2 Likes