Improvements to phpBB3 import script (#10999)
-
FEATURE: Import attachments
-
FEATURE: Add support for importing multiple forums in one
-
FEATURE: Add support for category and tag mapping
-
FEATURE: Import groups
-
FIX: Add spaces around images
-
FEATURE: Custom mapping of user rank to trust levels
-
FIX: Do not fail import if it cannot import polls
-
FIX: Optimize existing records lookup
Co-authored-by: Gerhard Schlager mail@gerhard-schlager.at Co-authored-by: Jarek Radosz jradosz@gmail.com
diff --git a/script/import_scripts/base.rb b/script/import_scripts/base.rb
index 39530d7..aaef4ef 100644
--- a/script/import_scripts/base.rb
+++ b/script/import_scripts/base.rb
@@ -606,10 +606,15 @@ class ImportScripts::Base
skipped += 1
puts "Skipping bookmark for user id #{params[:user_id]} and post id #{params[:post_id]}"
else
- result = BookmarkManager.new(user).create(post_id: post.id)
+ begin
+ manager = BookmarkManager.new(user)
+ bookmark = manager.create(post_id: post.id)
- created += 1 if result.errors.none?
- skipped += 1 if result.errors.any?
+ created += 1 if manager.errors.none?
+ skipped += 1 if manager.errors.any?
+ rescue
+ skipped += 1
+ end
end
end
diff --git a/script/import_scripts/base/lookup_container.rb b/script/import_scripts/base/lookup_container.rb
index 6c4169b..caef17c 100644
--- a/script/import_scripts/base/lookup_container.rb
+++ b/script/import_scripts/base/lookup_container.rb
@@ -57,6 +57,11 @@ module ImportScripts
UserCustomField.where(name: 'import_id', value: import_id.to_s).first.try(:user)
end
+ def find_username_by_import_id(import_id)
+ user_id = user_id_from_imported_user_id(import_id)
+ User.where(id: user_id).pluck(:username).first if user_id.present?
+ end
+
# Get the Discourse Category id based on the id of the source category
def category_id_from_imported_category_id(import_id)
@categories[import_id] || @categories[import_id.to_s]
diff --git a/script/import_scripts/phpbb3/database/database.rb b/script/import_scripts/phpbb3/database/database.rb
index 6b34feb..725d932 100644
--- a/script/import_scripts/phpbb3/database/database.rb
+++ b/script/import_scripts/phpbb3/database/database.rb
@@ -22,13 +22,13 @@ module ImportScripts::PhpBB3
if version.start_with?('3.0')
require_relative 'database_3_0'
Database_3_0.new(@database_client, @database_settings)
- elsif version.start_with?('3.1')
+ elsif version.start_with?('3.1') || version.start_with?('3.2')
require_relative 'database_3_1'
Database_3_1.new(@database_client, @database_settings)
else
raise UnsupportedVersionError, <<~MSG
Unsupported version (#{version}) of phpBB detected.
- Currently only 3.0.x and 3.1.x are supported by this importer.
+ Currently only version 3.0, 3.1 and 3.2 are supported by this importer.
MSG
end
end
diff --git a/script/import_scripts/phpbb3/database/database_3_0.rb b/script/import_scripts/phpbb3/database/database_3_0.rb
index 0799b92..a69a9f9 100644
--- a/script/import_scripts/phpbb3/database/database_3_0.rb
+++ b/script/import_scripts/phpbb3/database/database_3_0.rb
@@ -53,6 +53,20 @@ module ImportScripts::PhpBB3
SQL
end
+ def fetch_groups
+ query(<<-SQL)
+ SELECT g.group_id, g.group_type, g.group_name, g.group_desc
+ FROM #{@table_prefix}groups g
+ SQL
+ end
+
+ def fetch_group_users
+ query(<<-SQL)
+ SELECT ug.group_id, ug.user_id, ug.group_leader
+ FROM #{@table_prefix}user_group ug
+ SQL
+ end
+
def fetch_categories
query(<<-SQL)
SELECT f.forum_id, f.parent_id, f.forum_name, f.forum_desc, x.first_post_time
@@ -213,12 +227,20 @@ module ImportScripts::PhpBB3
SELECT b.user_id, t.topic_first_post_id
FROM #{@table_prefix}bookmarks b
JOIN #{@table_prefix}topics t ON (b.topic_id = t.topic_id)
- WHERE b.user_id > #{last_user_id} AND b.topic_id > #{last_topic_id}
+ WHERE b.user_id > #{last_user_id}
ORDER BY b.user_id, b.topic_id
LIMIT #{@batch_size}
SQL
end
+ def get_smiley(smiley_code)
+ query(<<-SQL).first
+ SELECT emotion, smiley_url
+ FROM #{@table_prefix}smilies
+ WHERE code = '#{smiley_code}'
+ SQL
+ end
+
def get_config_values
query(<<-SQL).first
SELECT
diff --git a/script/import_scripts/phpbb3/importer.rb b/script/import_scripts/phpbb3/importer.rb
index ca32740..f9ee6e0 100644
--- a/script/import_scripts/phpbb3/importer.rb
+++ b/script/import_scripts/phpbb3/importer.rb
@@ -27,8 +27,13 @@ module ImportScripts::PhpBB3
def execute
puts '', "importing from phpBB #{@php_config[:phpbb_version]}"
+ SiteSetting.tagging_enabled = true if @settings.tag_mappings.present?
+
import_users
import_anonymous_users if @settings.import_anonymous_users
+ import_groups
+ import_user_groups
+ import_new_categories
import_categories
import_posts
import_private_messages if @settings.import_private_messages
@@ -67,12 +72,12 @@ module ImportScripts::PhpBB3
batches do |offset|
rows, last_user_id = @database.fetch_users(last_user_id)
+ rows = rows.to_a.uniq { |row| row[:user_id] }
break if rows.size < 1
- next if all_records_exist?(:users, importer.map_users_to_import_ids(rows))
-
create_users(rows, total: total_count, offset: offset) do |row|
begin
+ next if user_id_from_imported_user_id(@settings.prefix(row[:user_id]))
importer.map_user(row)
rescue => e
log_error("Failed to map user with ID #{row[:user_id]}", e)
@@ -91,10 +96,9 @@ module ImportScripts::PhpBB3
rows, last_username = @database.fetch_anonymous_users(last_username)
break if rows.size < 1
- next if all_records_exist?(:users, importer.map_anonymous_users_to_import_ids(rows))
-
create_users(rows, total: total_count, offset: offset) do |row|
begin
+ next if user_id_from_imported_user_id(@settings.prefix(row[:post_username]))
importer.map_anonymous_user(row)
rescue => e
log_error("Failed to map anonymous user with ID #{row[:user_id]}", e)
@@ -103,12 +107,74 @@ module ImportScripts::PhpBB3
end
end
+ def import_groups
+ puts '', 'creating groups'
+ rows = @database.fetch_groups
+
+ create_groups(rows) do |row|
+ begin
+ next if row[:group_type] == 3
+
+ group_name = if @settings.site_name.present?
+ "#{@settings.site_name}_#{row[:group_name]}"
+ else
+ row[:group_name]
+ end[0..19].gsub(/[^a-zA-Z0-9\-_. ]/, '_')
+
+ bio_raw = @importers.text_processor.process_raw_text(row[:group_desc]) rescue row[:group_desc]
+
+ {
+ id: @settings.prefix(row[:group_id]),
+ name: group_name,
+ full_name: row[:group_name],
+ bio_raw: bio_raw
+ }
+ rescue => e
+ log_error("Failed to map group with ID #{row[:group_id]}", e)
+ end
+ end
+ end
+
+ def import_user_groups
+ puts '', 'creating user groups'
+ rows = @database.fetch_group_users
+
+ rows.each do |row|
+ group_id = @lookup.group_id_from_imported_group_id(@settings.prefix(row[:group_id]))
+ next if !group_id
+
+ user_id = @lookup.user_id_from_imported_user_id(@settings.prefix(row[:user_id]))
+
+ begin
+ GroupUser.find_or_create_by(user_id: user_id, group_id: group_id, owner: row[:group_leader])
+ rescue => e
+ log_error("Failed to add user #{row[:user_id]} to group #{row[:group_id]}", e)
+ end
+ end
+ end
+
+ def import_new_categories
+ puts '', 'creating new categories'
+
+ create_categories(@settings.new_categories) do |row|
+ next if row == "SKIP"
+
+ {
+ id: @settings.prefix(row[:forum_id]),
+ name: row[:name],
+ parent_category_id: @lookup.category_id_from_imported_category_id(@settings.prefix(row[:parent_id]))
+ }
+ end
+ end
+
def import_categories
puts '', 'creating categories'
[... diff too long, it was truncated ...]
GitHub sha: a71b219c