github-mirror/discourse
2
0
Fork 0
mirror of https://github.com/discourse/discourse.git synced 2025-05-22 07:53:49 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

DEV: Apply syntax_tree formatting to script/*

Browse Source
This commit is contained in:
David Taylor
2023-01-07 11:53:14 +00:00
parent ff508d1ae5
commit 436b3b392b
143 changed files with 8905 additions and 7353 deletions
Download Patch File Download Diff File Expand all files Collapse all files

92
script/import_scripts/vanilla_body_parser.rb
View File

@ -14,9 +14,9 @@ class VanillaBodyParser
end
def parse
return clean_up(@row['Body']) unless rich?
return clean_up(@row["Body"]) unless rich?
full_text = json.each_with_index.map(&method(:parse_fragment)).join('')
full_text = json.each_with_index.map(&method(:parse_fragment)).join("")
normalize full_text
end
@ -25,30 +25,46 @@ class VanillaBodyParser
def clean_up(text)
# <pre class="CodeBlock">...</pre>
text = text.gsub(/\<pre class="CodeBlock"\>(.*?)\<\/pre\>/im) { "\n```\n#{$1}\n```\n" }
text = text.gsub(%r{\<pre class="CodeBlock"\>(.*?)\</pre\>}im) { "\n```\n#{$1}\n```\n" }
# <pre>...</pre>
text = text.gsub(/\<pre\>(.*?)\<\/pre\>/im) { "\n```\n#{$1}\n```\n" }
text = text.gsub(%r{\<pre\>(.*?)\</pre\>}im) { "\n```\n#{$1}\n```\n" }
# <code></code>
text = text.gsub("\<code\>\</code\>", "").gsub(/\<code\>(.*?)\<\/code\>/im) { "#{$1}" }
text = text.gsub("\<code\>\</code\>", "").gsub(%r{\<code\>(.*?)\</code\>}im) { "#{$1}" }
# <div class="Quote">...</div>
text = text.gsub(/\<div class="Quote"\>(.*?)\<\/div\>/im) { "\n[quote]\n#{$1}\n[/quote]\n" }
text = text.gsub(%r{\<div class="Quote"\>(.*?)\</div\>}im) { "\n[quote]\n#{$1}\n[/quote]\n" }
# [code], [quote]
text = text.gsub(/\[\/?code\]/i, "\n```\n").gsub(/\[quote.*?\]/i, "\n" + '\0' + "\n").gsub(/\[\/quote\]/i, "\n" + '\0' + "\n")
text =
text
.gsub(%r{\[/?code\]}i, "\n```\n")
.gsub(/\[quote.*?\]/i, "\n" + '\0' + "\n")
.gsub(%r{\[/quote\]}i, "\n" + '\0' + "\n")
text.gsub(/<\/?font[^>]*>/, '').gsub(/<\/?span[^>]*>/, '').gsub(/<\/?div[^>]*>/, '').gsub(/^ +/, '').gsub(/ +/, ' ')
text
.gsub(%r{</?font[^>]*>}, "")
.gsub(%r{</?span[^>]*>}, "")
.gsub(%r{</?div[^>]*>}, "")
.gsub(/^ +/, "")
.gsub(/ +/, " ")
end
def rich?
@row['Format'].casecmp?('Rich')
@row["Format"].casecmp?("Rich")
end
def json
return nil unless rich?
@json ||= JSON.parse(@row['Body']).map(&:deep_symbolize_keys)
@json ||= JSON.parse(@row["Body"]).map(&:deep_symbolize_keys)
end
def parse_fragment(fragment, index)
text = fragment.keys.one? && fragment[:insert].is_a?(String) ? fragment[:insert] : rich_parse(fragment)
text =
(
if fragment.keys.one? && fragment[:insert].is_a?(String)
fragment[:insert]
else
rich_parse(fragment)
end
)
text = parse_code(text, fragment, index)
text = parse_list(text, fragment, index)
@ -59,16 +75,18 @@ class VanillaBodyParser
def rich_parse(fragment)
insert = fragment[:insert]
return parse_mention(insert[:mention]) if insert.respond_to?(:dig) && insert.dig(:mention, :userID)
if insert.respond_to?(:dig) && insert.dig(:mention, :userID)
return parse_mention(insert[:mention])
end
return parse_formatting(fragment) if fragment[:attributes]
embed_type = insert.dig(:'embed-external', :data, :embedType)
embed_type = insert.dig(:"embed-external", :data, :embedType)
quoting = embed_type == 'quote'
quoting = embed_type == "quote"
return parse_quote(insert) if quoting
embed = embed_type.in? ['image', 'link', 'file']
embed = embed_type.in? %w[image link file]
parse_embed(insert, embed_type) if embed
end
@ -101,10 +119,10 @@ class VanillaBodyParser
def parse_code(text, fragment, index)
next_fragment = next_fragment(index)
next_code = next_fragment.dig(:attributes, :'code-block')
next_code = next_fragment.dig(:attributes, :"code-block")
if next_code
previous_fragment = previous_fragment(index)
previous_code = previous_fragment.dig(:attributes, :'code-block')
previous_code = previous_fragment.dig(:attributes, :"code-block")
if previous_code
text = text.gsub(/\\n(.*?)\\n/) { "\n```\n#{$1}\n```\n" }
@ -112,7 +130,7 @@ class VanillaBodyParser
last_pos = text.rindex(/\n/)
if last_pos
array = [text[0..last_pos].strip, text[last_pos + 1 .. text.length].strip]
array = [text[0..last_pos].strip, text[last_pos + 1..text.length].strip]
text = array.join("\n```\n")
else
text = "\n```\n#{text}"
@ -120,10 +138,10 @@ class VanillaBodyParser
end
end
current_code = fragment.dig(:attributes, :'code-block')
current_code = fragment.dig(:attributes, :"code-block")
if current_code
second_next_fragment = second_next_fragment(index)
second_next_code = second_next_fragment.dig(:attributes, :'code-block')
second_next_code = second_next_fragment.dig(:attributes, :"code-block")
# if current is code and 2 after is not, prepend ```
text = "\n```\n#{text}" unless second_next_code
@ -138,13 +156,13 @@ class VanillaBodyParser
next_list = next_fragment.dig(:attributes, :list, :type)
if next_list
# if next is list, prepend <li>
text = '<li>' + text
text = "<li>" + text
previous_fragment = previous_fragment(index)
previous_list = previous_fragment.dig(:attributes, :list, :type)
# if next is list and previous is not, prepend <ol> or <ul>
list_tag = next_list == 'ordered' ? '<ol>' : '<ul>'
list_tag = next_list == "ordered" ? "<ol>" : "<ul>"
text = "\n#{list_tag}\n#{text}" unless previous_list
end
@ -152,13 +170,13 @@ class VanillaBodyParser
if current_list
# if current is list prepend </li>
tag_closings = '</li>'
tag_closings = "</li>"
second_next_fragment = second_next_fragment(index)
second_next_list = second_next_fragment.dig(:attributes, :list, :type)
# if current is list and 2 after is not, prepend </ol>
list_tag = current_list == 'ordered' ? '</ol>' : '</ul>'
list_tag = current_list == "ordered" ? "</ol>" : "</ul>"
tag_closings = "#{tag_closings}\n#{list_tag}" unless second_next_list
text = tag_closings + text
@ -180,24 +198,32 @@ class VanillaBodyParser
end
def parse_quote(insert)
embed = insert.dig(:'embed-external', :data)
embed = insert.dig(:"embed-external", :data)
import_post_id = "#{embed[:recordType]}##{embed[:recordID]}"
topic = @@lookup.topic_lookup_from_imported_post_id(import_post_id)
user = user_from_imported_id(embed.dig(:insertUser, :userID))
quote_info = topic && user ? "=\"#{user.username}, post: #{topic[:post_number]}, topic: #{topic[:topic_id]}\"" : ''
quote_info =
(
if topic && user
"=\"#{user.username}, post: #{topic[:post_number]}, topic: #{topic[:topic_id]}\""
else
""
end
)
"[quote#{quote_info}]\n#{embed[:body]}\n[/quote]\n\n"""
"[quote#{quote_info}]\n#{embed[:body]}\n[/quote]\n\n" \
""
end
def parse_embed(insert, embed_type)
embed = insert.dig(:'embed-external', :data)
embed = insert.dig(:"embed-external", :data)
url = embed[:url]
if /https?\:\/\/#{@@host}\/uploads\/.*/.match?(url)
remote_path = url.scan(/uploads\/(.*)/)
if %r{https?\://#{@@host}/uploads/.*}.match?(url)
remote_path = url.scan(%r{uploads/(.*)})
path = File.join(@@uploads_path, remote_path)
upload = @@uploader.create_upload(@user_id, path, embed[:name])
@ -206,7 +232,7 @@ class VanillaBodyParser
return "\n" + @@uploader.html_for_upload(upload, embed[:name]) + "\n"
else
puts "Failed to upload #{path}"
puts upload.errors.full_messages.join(', ') if upload
puts upload.errors.full_messages.join(", ") if upload
end
end
@ -222,9 +248,9 @@ class VanillaBodyParser
def normalize(full_text)
code_matcher = /```(.*\n)+```/
code_block = full_text[code_matcher]
full_text[code_matcher] = '{{{CODE_BLOCK}}}' if code_block
full_text[code_matcher] = "{{{CODE_BLOCK}}}" if code_block
full_text = double_new_lines(full_text)
full_text['{{{CODE_BLOCK}}}'] = code_block if code_block
full_text["{{{CODE_BLOCK}}}"] = code_block if code_block
full_text
end