github-mirror/discourse
2
0
Fork 0
mirror of https://github.com/discourse/discourse.git synced 2025-05-22 04:31:10 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

DEV: remove exec_sql and replace with mini_sql

Browse Source 
Introduce new patterns for direct sql that are safe and fast.

MiniSql is not prone to memory bloat that can happen with direct PG usage.
It also has an extremely fast materializer and very a convenient API

- DB.exec(sql, *params) => runs sql returns row count
- DB.query(sql, *params) => runs sql returns usable objects (not a hash)
- DB.query_hash(sql, *params) => runs sql returns an array of hashes
- DB.query_single(sql, *params) => runs sql and returns a flat one dimensional array
- DB.build(sql) => returns a sql builder

See more at: https://github.com/discourse/mini_sql
This commit is contained in:
Sam
2018-06-19 16:13:14 +10:00
parent cc3fc87dd7
commit 5f64fd0a21
112 changed files with 782 additions and 763 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
script/import_scripts/base.rb
View File

@ -626,7 +626,7 @@ class ImportScripts::Base
def update_topic_status
puts "", "Updating topic status"
Topic.exec_sql(<<~SQL)
DB.exec(<<~SQL)
UPDATE topics AS t
SET closed = TRUE
WHERE EXISTS(
@ -636,7 +636,7 @@ class ImportScripts::Base
)
SQL
Topic.exec_sql(<<~SQL)
DB.exec(<<~SQL)
UPDATE topics AS t
SET archived = TRUE
WHERE EXISTS(
@ -646,7 +646,7 @@ class ImportScripts::Base
)
SQL
TopicCustomField.exec_sql(<<~SQL)
DB.exec(<<~SQL)
DELETE FROM topic_custom_fields
WHERE name IN ('import_closed', 'import_archived')
SQL
@ -654,7 +654,7 @@ class ImportScripts::Base
def update_bumped_at
puts "", "Updating bumped_at on topics"
Post.exec_sql("update topics t set bumped_at = COALESCE((select max(created_at) from posts where topic_id = t.id and post_type = #{Post.types[:regular]}), bumped_at)")
DB.exec("update topics t set bumped_at = COALESCE((select max(created_at) from posts where topic_id = t.id and post_type = #{Post.types[:regular]}), bumped_at)")
end
def update_last_posted_at
@ -674,7 +674,7 @@ class ImportScripts::Base
AND users.last_posted_at <> lpa.last_posted_at
SQL
User.exec_sql(sql)
DB.exec(sql)
end
def update_user_stats
@ -707,7 +707,7 @@ class ImportScripts::Base
AND user_stats.first_post_created_at <> sub.first_post_created_at
SQL
User.exec_sql(sql)
DB.exec(sql)
puts "", "Updating user post_count..."
@ -725,7 +725,7 @@ class ImportScripts::Base
AND user_stats.post_count <> sub.post_count
SQL
User.exec_sql(sql)
DB.exec(sql)
puts "", "Updating user topic_count..."
@ -743,15 +743,15 @@ class ImportScripts::Base
AND user_stats.topic_count <> sub.topic_count
SQL
User.exec_sql(sql)
DB.exec(sql)
end
# scripts that are able to import last_seen_at from the source data should override this method
def update_last_seen_at
puts "", "Updating last seen at on users"
User.exec_sql("UPDATE users SET last_seen_at = created_at WHERE last_seen_at IS NULL")
User.exec_sql("UPDATE users SET last_seen_at = last_posted_at WHERE last_posted_at IS NOT NULL")
DB.exec("UPDATE users SET last_seen_at = created_at WHERE last_seen_at IS NULL")
DB.exec("UPDATE users SET last_seen_at = last_posted_at WHERE last_posted_at IS NOT NULL")
end
def update_feature_topic_users

2
script/import_scripts/ipboard3.rb
View File

@ -267,7 +267,7 @@ class ImportScripts::IPBoard3 < ImportScripts::Base
WHERE id IN (SELECT topic_id FROM closed_topic_ids)
SQL
Topic.exec_sql(sql, @closed_topic_ids)
DB.exec(sql, @closed_topic_ids)
end
def import_personal_topics

2
script/import_scripts/jive_api.rb
View File

@ -325,7 +325,7 @@ class ImportScripts::JiveApi < ImportScripts::Base
def mark_topics_as_solved
puts "", "Marking topics as solved..."
PostAction.exec_sql <<-SQL
DB.exec <<~SQL
INSERT INTO topic_custom_fields (name, value, topic_id, created_at, updated_at)
SELECT 'accepted_answer_post_id', pcf.post_id, p.topic_id, p.created_at, p.created_at
FROM post_custom_fields pcf

48
script/import_scripts/lithium.rb
View File

@ -535,7 +535,7 @@ class ImportScripts::Lithium < ImportScripts::Base
end
puts "loading data into temp table"
PostAction.exec_sql("create temp table like_data(user_id int, post_id int, created_at timestamp without time zone)")
DB.exec("create temp table like_data(user_id int, post_id int, created_at timestamp without time zone)")
PostAction.transaction do
results.each do |result|
@ -544,17 +544,17 @@ class ImportScripts::Lithium < ImportScripts::Base
next unless result["user_id"] && result["post_id"]
PostAction.exec_sql("INSERT INTO like_data VALUES (:user_id,:post_id,:created_at)",
user_id: result["user_id"],
post_id: result["post_id"],
created_at: result["created_at"]
)
DB.exec("INSERT INTO like_data VALUES (:user_id,:post_id,:created_at)",
user_id: result["user_id"],
post_id: result["post_id"],
created_at: result["created_at"]
)
end
end
puts "creating missing post actions"
PostAction.exec_sql <<-SQL
DB.exec <<~SQL
INSERT INTO post_actions (post_id, user_id, post_action_type_id, created_at, updated_at)
SELECT l.post_id, l.user_id, 2, l.created_at, l.created_at FROM like_data l
@ -563,7 +563,7 @@ class ImportScripts::Lithium < ImportScripts::Base
SQL
puts "creating missing user actions"
UserAction.exec_sql <<-SQL
DB.exec <<~SQL
INSERT INTO user_actions (user_id, action_type, target_topic_id, target_post_id, acting_user_id, created_at, updated_at)
SELECT pa.user_id, 1, p.topic_id, p.id, pa.user_id, pa.created_at, pa.created_at
FROM post_actions pa
@ -574,7 +574,7 @@ class ImportScripts::Lithium < ImportScripts::Base
SQL
# reverse action
UserAction.exec_sql <<-SQL
DB.exec <<~SQL
INSERT INTO user_actions (user_id, action_type, target_topic_id, target_post_id, acting_user_id, created_at, updated_at)
SELECT p.user_id, 2, p.topic_id, p.id, pa.user_id, pa.created_at, pa.created_at
FROM post_actions pa
@ -586,7 +586,7 @@ class ImportScripts::Lithium < ImportScripts::Base
SQL
puts "updating like counts on posts"
Post.exec_sql <<-SQL
DB.exec <<~SQL
UPDATE posts SET like_count = coalesce(cnt,0)
FROM (
SELECT post_id, count(*) cnt
@ -600,7 +600,7 @@ class ImportScripts::Lithium < ImportScripts::Base
puts "updating like counts on topics"
Post.exec_sql <<-SQL
DB.exec <<-SQL
UPDATE topics SET like_count = coalesce(cnt,0)
FROM (
SELECT topic_id, sum(like_count) cnt
@ -627,7 +627,7 @@ class ImportScripts::Lithium < ImportScripts::Base
end
puts "loading data into temp table"
PostAction.exec_sql("create temp table accepted_data(post_id int primary key)")
DB.exec("create temp table accepted_data(post_id int primary key)")
PostAction.transaction do
results.each do |result|
@ -635,7 +635,7 @@ class ImportScripts::Lithium < ImportScripts::Base
next unless result["post_id"]
PostAction.exec_sql("INSERT INTO accepted_data VALUES (:post_id)",
DB.exec("INSERT INTO accepted_data VALUES (:post_id)",
post_id: result["post_id"]
)
@ -643,7 +643,7 @@ class ImportScripts::Lithium < ImportScripts::Base
end
puts "deleting dupe answers"
PostAction.exec_sql <<-SQL
DB.exec <<~SQL
DELETE FROM accepted_data WHERE post_id NOT IN (
SELECT post_id FROM
(
@ -656,7 +656,7 @@ class ImportScripts::Lithium < ImportScripts::Base
SQL
puts "importing accepted answers"
PostAction.exec_sql <<-SQL
DB.exec <<~SQL
INSERT into post_custom_fields (name, value, post_id, created_at, updated_at)
SELECT 'is_accepted_answer', 'true', a.post_id, current_timestamp, current_timestamp
FROM accepted_data a
@ -665,7 +665,7 @@ class ImportScripts::Lithium < ImportScripts::Base
SQL
puts "marking accepted topics"
PostAction.exec_sql <<-SQL
DB.exec <<~SQL
INSERT into topic_custom_fields (name, value, topic_id, created_at, updated_at)
SELECT 'accepted_answer_post_id', a.post_id::varchar, p.topic_id, current_timestamp, current_timestamp
FROM accepted_data a
@ -797,10 +797,10 @@ class ImportScripts::Lithium < ImportScripts::Base
results.map { |r| r["post_id"] }.each_slice(500) do |ids|
mapped = ids.map { |id| existing_map[id] }.compact
Topic.exec_sql("
UPDATE topics SET closed = true
WHERE id IN (SELECT topic_id FROM posts where id in (:ids))
", ids: mapped) if mapped.present?
DB.exec(<<~SQL, ids: mapped) if mapped.present?
UPDATE topics SET closed = true
WHERE id IN (SELECT topic_id FROM posts where id in (:ids))
SQL
end
end
@ -819,8 +819,8 @@ class ImportScripts::Lithium < ImportScripts::Base
WHERE pm.id IS NULL AND f.name = 'import_unique_id'
SQL
r = Permalink.exec_sql sql
puts "#{r.cmd_tuples} permalinks to topics added!"
r = DB.exec sql
puts "#{r} permalinks to topics added!"
sql = <<-SQL
INSERT INTO permalinks (url, post_id, created_at, updated_at)
@ -831,8 +831,8 @@ SQL
WHERE pm.id IS NULL AND f.name = 'import_unique_id'
SQL
r = Permalink.exec_sql sql
puts "#{r.cmd_tuples} permalinks to posts added!"
r = DB.exec sql
puts "#{r} permalinks to posts added!"
end

4
script/import_scripts/modx.rb
View File

@ -236,7 +236,7 @@ FROM #{TABLE_PREFIX}discuss_users
def not_mark_topics_as_solved
puts "", "Marking topics as solved..."
PostAction.exec_sql <<-SQL
DB.exec <<~SQL
INSERT INTO topic_custom_fields (name, value, topic_id, created_at, updated_at)
SELECT 'accepted_answer_post_id', pcf.post_id, p.topic_id, p.created_at, p.created_at
FROM post_custom_fields pcf
@ -469,7 +469,7 @@ FROM #{TABLE_PREFIX}discuss_users
WHERE id IN (SELECT topic_id FROM closed_topic_ids)
SQL
Topic.exec_sql(sql, closed_topic_ids)
DB.exec(sql, closed_topic_ids)
end
def not_post_process_posts

2
script/import_scripts/stack_overflow.rb
View File

@ -237,7 +237,7 @@ class ImportScripts::StackOverflow < ImportScripts::Base
def mark_topics_as_solved
puts "", "Marking topics as solved..."
Topic.exec_sql <<~SQL
DB.exec <<~SQL
INSERT INTO topic_custom_fields (name, value, topic_id, created_at, updated_at)
SELECT 'accepted_answer_post_id', pcf.post_id, p.topic_id, p.created_at, p.created_at
FROM post_custom_fields pcf

8
script/import_scripts/vbulletin.rb
View File

@ -182,10 +182,8 @@ EOM
next if user_ids_in_group.size == 0
values = user_ids_in_group.map { |user_id| "(#{group.id}, #{user_id}, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)" }.join(",")
User.exec_sql <<-SQL
BEGIN;
INSERT INTO group_users (group_id, user_id, created_at, updated_at) VALUES #{values};
COMMIT;
DB.exec <<~SQL
INSERT INTO group_users (group_id, user_id, created_at, updated_at) VALUES #{values}
SQL
Group.reset_counters(group.id, :group_users)
@ -634,7 +632,7 @@ EOM
WHERE id IN (SELECT topic_id FROM closed_topic_ids)
SQL
Topic.exec_sql(sql, closed_topic_ids)
DB.exec(sql, closed_topic_ids)
end
def post_process_posts

2
script/import_scripts/vbulletin5.rb
View File

@ -418,7 +418,7 @@ class ImportScripts::VBulletin < ImportScripts::Base
WHERE id IN (SELECT topic_id FROM closed_topic_ids)
SQL
Topic.exec_sql(sql, @closed_topic_ids)
DB.exec(sql, @closed_topic_ids)
end
def post_process_posts