Improvements to phpBB3 import script (#10999)

Improvements to phpBB3 import script (#10999)

  • FEATURE: Import attachments

  • FEATURE: Add support for importing multiple forums in one

  • FEATURE: Add support for category and tag mapping

  • FEATURE: Import groups

  • FIX: Add spaces around images

  • FEATURE: Custom mapping of user rank to trust levels

  • FIX: Do not fail import if it cannot import polls

  • FIX: Optimize existing records lookup

Co-authored-by: Gerhard Schlager mail@gerhard-schlager.at Co-authored-by: Jarek Radosz jradosz@gmail.com

diff --git a/script/import_scripts/base.rb b/script/import_scripts/base.rb
index 39530d7..aaef4ef 100644
--- a/script/import_scripts/base.rb
+++ b/script/import_scripts/base.rb
@@ -606,10 +606,15 @@ class ImportScripts::Base
           skipped += 1
           puts "Skipping bookmark for user id #{params[:user_id]} and post id #{params[:post_id]}"
         else
-          result = BookmarkManager.new(user).create(post_id: post.id)
+          begin
+            manager = BookmarkManager.new(user)
+            bookmark = manager.create(post_id: post.id)
 
-          created += 1 if result.errors.none?
-          skipped += 1 if result.errors.any?
+            created += 1 if manager.errors.none?
+            skipped += 1 if manager.errors.any?
+          rescue
+            skipped += 1
+          end
         end
       end
 
diff --git a/script/import_scripts/base/lookup_container.rb b/script/import_scripts/base/lookup_container.rb
index 6c4169b..caef17c 100644
--- a/script/import_scripts/base/lookup_container.rb
+++ b/script/import_scripts/base/lookup_container.rb
@@ -57,6 +57,11 @@ module ImportScripts
       UserCustomField.where(name: 'import_id', value: import_id.to_s).first.try(:user)
     end
 
+    def find_username_by_import_id(import_id)
+      user_id = user_id_from_imported_user_id(import_id)
+      User.where(id: user_id).pluck(:username).first if user_id.present?
+    end
+
     # Get the Discourse Category id based on the id of the source category
     def category_id_from_imported_category_id(import_id)
       @categories[import_id] || @categories[import_id.to_s]
diff --git a/script/import_scripts/phpbb3/database/database.rb b/script/import_scripts/phpbb3/database/database.rb
index 6b34feb..725d932 100644
--- a/script/import_scripts/phpbb3/database/database.rb
+++ b/script/import_scripts/phpbb3/database/database.rb
@@ -22,13 +22,13 @@ module ImportScripts::PhpBB3
       if version.start_with?('3.0')
         require_relative 'database_3_0'
         Database_3_0.new(@database_client, @database_settings)
-      elsif version.start_with?('3.1')
+      elsif version.start_with?('3.1') || version.start_with?('3.2')
         require_relative 'database_3_1'
         Database_3_1.new(@database_client, @database_settings)
       else
         raise UnsupportedVersionError, <<~MSG
           Unsupported version (#{version}) of phpBB detected.
-          Currently only 3.0.x and 3.1.x are supported by this importer.
+          Currently only version 3.0, 3.1 and 3.2 are supported by this importer.
         MSG
       end
     end
diff --git a/script/import_scripts/phpbb3/database/database_3_0.rb b/script/import_scripts/phpbb3/database/database_3_0.rb
index 0799b92..a69a9f9 100644
--- a/script/import_scripts/phpbb3/database/database_3_0.rb
+++ b/script/import_scripts/phpbb3/database/database_3_0.rb
@@ -53,6 +53,20 @@ module ImportScripts::PhpBB3
       SQL
     end
 
+    def fetch_groups
+      query(<<-SQL)
+        SELECT g.group_id, g.group_type, g.group_name, g.group_desc
+        FROM #{@table_prefix}groups g
+      SQL
+    end
+
+    def fetch_group_users
+      query(<<-SQL)
+        SELECT ug.group_id, ug.user_id, ug.group_leader
+        FROM #{@table_prefix}user_group ug
+      SQL
+    end
+
     def fetch_categories
       query(<<-SQL)
         SELECT f.forum_id, f.parent_id, f.forum_name, f.forum_desc, x.first_post_time
@@ -213,12 +227,20 @@ module ImportScripts::PhpBB3
         SELECT b.user_id, t.topic_first_post_id
         FROM #{@table_prefix}bookmarks b
           JOIN #{@table_prefix}topics t ON (b.topic_id = t.topic_id)
-        WHERE b.user_id > #{last_user_id} AND b.topic_id > #{last_topic_id}
+        WHERE b.user_id > #{last_user_id}
         ORDER BY b.user_id, b.topic_id
         LIMIT #{@batch_size}
       SQL
     end
 
+    def get_smiley(smiley_code)
+      query(<<-SQL).first
+        SELECT emotion, smiley_url
+        FROM #{@table_prefix}smilies
+        WHERE code = '#{smiley_code}'
+      SQL
+    end
+
     def get_config_values
       query(<<-SQL).first
         SELECT
diff --git a/script/import_scripts/phpbb3/importer.rb b/script/import_scripts/phpbb3/importer.rb
index ca32740..f9ee6e0 100644
--- a/script/import_scripts/phpbb3/importer.rb
+++ b/script/import_scripts/phpbb3/importer.rb
@@ -27,8 +27,13 @@ module ImportScripts::PhpBB3
     def execute
       puts '', "importing from phpBB #{@php_config[:phpbb_version]}"
 
+      SiteSetting.tagging_enabled = true if @settings.tag_mappings.present?
+
       import_users
       import_anonymous_users if @settings.import_anonymous_users
+      import_groups
+      import_user_groups
+      import_new_categories
       import_categories
       import_posts
       import_private_messages if @settings.import_private_messages
@@ -67,12 +72,12 @@ module ImportScripts::PhpBB3
 
       batches do |offset|
         rows, last_user_id = @database.fetch_users(last_user_id)
+        rows = rows.to_a.uniq { |row| row[:user_id] }
         break if rows.size < 1
 
-        next if all_records_exist?(:users, importer.map_users_to_import_ids(rows))
-
         create_users(rows, total: total_count, offset: offset) do |row|
           begin
+            next if user_id_from_imported_user_id(@settings.prefix(row[:user_id]))
             importer.map_user(row)
           rescue => e
             log_error("Failed to map user with ID #{row[:user_id]}", e)
@@ -91,10 +96,9 @@ module ImportScripts::PhpBB3
         rows, last_username = @database.fetch_anonymous_users(last_username)
         break if rows.size < 1
 
-        next if all_records_exist?(:users, importer.map_anonymous_users_to_import_ids(rows))
-
         create_users(rows, total: total_count, offset: offset) do |row|
           begin
+            next if user_id_from_imported_user_id(@settings.prefix(row[:post_username]))
             importer.map_anonymous_user(row)
           rescue => e
             log_error("Failed to map anonymous user with ID #{row[:user_id]}", e)
@@ -103,12 +107,74 @@ module ImportScripts::PhpBB3
       end
     end
 
+    def import_groups
+      puts '', 'creating groups'
+      rows = @database.fetch_groups
+
+      create_groups(rows) do |row|
+        begin
+          next if row[:group_type] == 3
+
+          group_name = if @settings.site_name.present?
+            "#{@settings.site_name}_#{row[:group_name]}"
+          else
+            row[:group_name]
+          end[0..19].gsub(/[^a-zA-Z0-9\-_. ]/, '_')
+
+          bio_raw = @importers.text_processor.process_raw_text(row[:group_desc]) rescue row[:group_desc]
+
+          {
+            id: @settings.prefix(row[:group_id]),
+            name: group_name,
+            full_name: row[:group_name],
+            bio_raw: bio_raw
+          }
+        rescue => e
+          log_error("Failed to map group with ID #{row[:group_id]}", e)
+        end
+      end
+    end
+
+    def import_user_groups
+      puts '', 'creating user groups'
+      rows = @database.fetch_group_users
+
+      rows.each do |row|
+        group_id = @lookup.group_id_from_imported_group_id(@settings.prefix(row[:group_id]))
+        next if !group_id
+
+        user_id = @lookup.user_id_from_imported_user_id(@settings.prefix(row[:user_id]))
+
+        begin
+          GroupUser.find_or_create_by(user_id: user_id, group_id: group_id, owner: row[:group_leader])
+        rescue => e
+          log_error("Failed to add user #{row[:user_id]} to group #{row[:group_id]}", e)
+        end
+      end
+    end
+
+    def import_new_categories
+      puts '', 'creating new categories'
+
+      create_categories(@settings.new_categories) do |row|
+        next if row == "SKIP"
+
+        {
+          id: @settings.prefix(row[:forum_id]),
+          name: row[:name],
+          parent_category_id: @lookup.category_id_from_imported_category_id(@settings.prefix(row[:parent_id]))
+        }
+      end
+    end
+
     def import_categories
       puts '', 'creating categories'

[... diff too long, it was truncated ...]

GitHub sha: a71b219c

This commit appears in #10999 which was approved by gschlager. It was merged by nbianca.

This commit has been mentioned on Discourse Meta. There might be relevant details there: