github-mirror/discourse
2
0
Fork 0
mirror of https://github.com/discourse/discourse.git synced 2025-06-01 09:05:53 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

DEV: discourse-emojis gem (#31408)

Browse Source 
This commit moves most of emoji logic into the discourse-emojis gem:
https://github.com/discourse/discourse-emojis/

Most notably:
- images are now symlinked from the gem
- the gem provides path to the json files

Search aliases have also been made asynchronous and memoized. When you
will search for an emoji we will now load the aliases and store the list
for future use.

---------

Co-authored-by: David Taylor <david@taylorhq.com>
This commit is contained in:
Joffrey JAFFEUX
2025-03-03 13:09:08 +01:00
committed by GitHub
parent 6b6cffdf85
commit d38acc5df1
22153 changed files with 5792 additions and 34242 deletions
Download Patch File Download Diff File Expand all files Collapse all files

562
lib/tasks/emoji.rake
View File

@ -1,562 +0,0 @@
# frozen_string_literal: true
require "active_support/test_case"
require "fileutils"
require "json"
require "nokogiri"
require "open-uri"
require "file_helper"
EMOJI_GROUPS_PATH = "lib/emoji/groups.json"
EMOJI_DB_PATH = "lib/emoji/db.json"
EMOJI_IMAGES_PATH = "public/images/emoji"
EMOJI_ORDERING_URL = "http://www.unicode.org/emoji/charts/emoji-ordering.html"
# emoji aliases are actually created as images
# eg: "right_anger_bubble" => [ "anger_right" ]
# your app will physically have right_anger_bubble.png and anger_right.png
EMOJI_ALIASES = {
"right_anger_bubble" => ["anger_right"],
"ballot_box" => ["ballot_box_with_ballot"],
"basketball_man" => %w[basketball_player person_with_ball],
"beach_umbrella" => %w[umbrella_on_ground beach beach_with_umbrella],
"parasol_on_ground" => ["umbrella_on_ground"],
"bellhop_bell" => ["bellhop"],
"biohazard" => ["biohazard_sign"],
"bow_and_arrow" => ["archery"],
"spiral_calendar" => %w[calendar_spiral spiral_calendar_pad],
"card_file_box" => ["card_box"],
"champagne" => ["bottle_with_popping_cork"],
"cheese" => ["cheese_wedge"],
"city_sunset" => ["city_dusk"],
"couch_and_lamp" => ["couch"],
"crayon" => ["lower_left_crayon"],
"cricket_bat_and_ball" => ["cricket_bat_ball"],
"latin_cross" => ["cross"],
"dagger" => ["dagger_knife"],
"desktop_computer" => ["desktop"],
"card_index_dividers" => ["dividers"],
"dove" => ["dove_of_peace"],
"footprints" => ["feet"],
"fire" => ["flame"],
"black_flag" => %w[flag_black waving_black_flag],
"cn" => ["flag_cn"],
"de" => ["flag_de"],
"es" => ["flag_es"],
"fr" => ["flag_fr"],
"uk" => %w[gb flag_gb],
"it" => ["flag_it"],
"jp" => ["flag_jp"],
"kr" => ["flag_kr"],
"ru" => ["flag_ru"],
"us" => ["flag_us"],
"white_flag" => %w[flag_white waving_white_flag],
"plate_with_cutlery" => %w[fork_knife_plate fork_and_knife_with_plate],
"framed_picture" => %w[frame_photo frame_with_picture],
"hammer_and_pick" => ["hammer_pick"],
"heavy_heart_exclamation" => %w[heart_exclamation heavy_heart_exclamation_mark_ornament],
"houses" => %w[homes house_buildings],
"hotdog" => ["hot_dog"],
"derelict_house" => %w[house_abandoned derelict_house_building],
"desert_island" => ["island"],
"old_key" => ["key2"],
"laughing" => ["satisfied"],
"business_suit_levitating" => %w[levitate man_in_business_suit_levitating],
"weight_lifting_man" => %w[lifter weight_lifter],
"medal_sports" => %w[medal sports_medal],
"metal" => ["sign_of_the_horns"],
"fu" => %w[middle_finger reversed_hand_with_middle_finger_extended],
"motorcycle" => ["racing_motorcycle"],
"mountain_snow" => ["snow_capped_mountain"],
"newspaper_roll" => %w[newspaper2 rolled_up_newspaper],
"spiral_notepad" => %w[notepad_spiral spiral_note_pad],
"oil_drum" => ["oil"],
"older_woman" => ["grandma"],
"paintbrush" => ["lower_left_paintbrush"],
"paperclips" => ["linked_paperclips"],
"pause_button" => ["double_vertical_bar"],
"peace_symbol" => ["peace"],
"fountain_pen" => %w[pen_fountain lower_left_fountain_pen],
"ping_pong" => ["table_tennis"],
"place_of_worship" => ["worship_symbol"],
"poop" => %w[shit hankey poo],
"radioactive" => ["radioactive_sign"],
"railway_track" => ["railroad_track"],
"robot" => ["robot_face"],
"skull" => ["skeleton"],
"skull_and_crossbones" => ["skull_crossbones"],
"speaking_head" => ["speaking_head_in_silhouette"],
"male_detective" => %w[spy sleuth_or_spy],
"thinking" => ["thinking_face"],
"-1" => ["thumbsdown"],
"+1" => ["thumbsup"],
"cloud_with_lightning_and_rain" => %w[thunder_cloud_rain thunder_cloud_and_rain],
"tickets" => ["admission_tickets"],
"next_track_button" => %w[track_next next_track],
"previous_track_button" => %w[track_previous previous_track],
"unicorn" => ["unicorn_face"],
"funeral_urn" => ["urn"],
"sun_behind_large_cloud" => %w[white_sun_cloud white_sun_behind_cloud],
"sun_behind_rain_cloud" => %w[white_sun_rain_cloud white_sun_behind_cloud_with_rain],
"partly_sunny" => %w[white_sun_small_cloud white_sun_with_small_cloud],
"open_umbrella" => ["umbrella2"],
"hammer_and_wrench" => ["tools"],
"face_with_thermometer" => ["thermometer_face"],
"timer_clock" => ["timer"],
"keycap_ten" => ["ten"],
"memo" => ["pencil"],
"rescue_worker_helmet" => %w[helmet_with_cross helmet_with_white_cross],
"slightly_smiling_face" => %w[slightly_smiling slight_smile],
"construction_worker_man" => ["construction_worker"],
"upside_down_face" => ["upside_down"],
"money_mouth_face" => ["money_mouth"],
"nerd_face" => ["nerd"],
"hugs" => %w[hugging hugging_face],
"roll_eyes" => %w[rolling_eyes face_with_rolling_eyes],
"slightly_frowning_face" => %w[frowning slight_frown],
"frowning_face" => %w[frowning2 white_frowning_face],
"zipper_mouth_face" => ["zipper_mouth"],
"face_with_head_bandage" => ["head_bandage"],
"raised_hand_with_fingers_splayed" => ["hand_splayed"],
"raised_hand" => ["hand"],
"vulcan_salute" => %w[vulcan raised_hand_with_part_between_middle_and_ring_fingers],
"policeman" => ["cop"],
"running_man" => ["runner"],
"walking_man" => ["walking"],
"bowing_man" => ["bow"],
"no_good_woman" => ["no_good"],
"raising_hand_woman" => ["raising_hand"],
"pouting_woman" => ["person_with_pouting_face"],
"frowning_woman" => ["person_frowning"],
"haircut_woman" => ["haircut"],
"massage_woman" => ["massage"],
"tshirt" => ["shirt"],
"biking_man" => ["bicyclist"],
"mountain_biking_man" => ["mountain_bicyclist"],
"passenger_ship" => ["cruise_ship"],
"motor_boat" => %w[motorboat boat],
"flight_arrival" => ["airplane_arriving"],
"flight_departure" => ["airplane_departure"],
"small_airplane" => ["airplane_small"],
"racing_car" => ["race_car"],
"family_man_woman_boy_boy" => ["family_man_woman_boys"],
"family_man_woman_girl_girl" => ["family_man_woman_girls"],
"family_woman_woman_boy" => ["family_women_boy"],
"family_woman_woman_girl" => ["family_women_girl"],
"family_woman_woman_girl_boy" => ["family_women_girl_boy"],
"family_woman_woman_boy_boy" => ["family_women_boys"],
"family_woman_woman_girl_girl" => ["family_women_girls"],
"family_man_man_boy" => ["family_men_boy"],
"family_man_man_girl" => ["family_men_girl"],
"family_man_man_girl_boy" => ["family_men_girl_boy"],
"family_man_man_boy_boy" => ["family_men_boys"],
"family_man_man_girl_girl" => ["family_men_girls"],
"cloud_with_lightning" => ["cloud_lightning"],
"tornado" => %w[cloud_tornado cloud_with_tornado],
"cloud_with_rain" => ["cloud_rain"],
"cloud_with_snow" => ["cloud_snow"],
"asterisk" => ["keycap_star"],
"studio_microphone" => ["microphone2"],
"medal_military" => ["military_medal"],
"couple_with_heart_woman_woman" => ["female_couple_with_heart"],
"couple_with_heart_man_man" => ["male_couple_with_heart"],
"couplekiss_woman_woman" => ["female_couplekiss"],
"couplekiss_man_man" => ["male_couplekiss"],
"honeybee" => ["bee"],
"lion" => ["lion_face"],
"artificial_satellite" => ["satellite_orbital"],
"computer_mouse" => %w[mouse_three_button three_button_mouse],
"hocho" => ["knife"],
"swimming_man" => ["swimmer"],
"wind_face" => ["wind_blowing_face"],
"golfing_man" => ["golfer"],
"facepunch" => ["punch"],
"building_construction" => ["construction_site"],
"family_man_woman_girl_boy" => ["family"],
"ice_hockey" => ["hockey"],
"snowman_with_snow" => ["snowman2"],
"play_or_pause_button" => ["play_pause"],
"film_projector" => ["projector"],
"shopping" => ["shopping_bags"],
"open_book" => ["book"],
"national_park" => ["park"],
"world_map" => ["map"],
"pen" => %w[pen_ballpoint lower_left_ballpoint_pen],
"email" => %w[envelope e-mail],
"phone" => ["telephone"],
"atom_symbol" => ["atom"],
"mantelpiece_clock" => ["clock"],
"camera_flash" => ["camera_with_flash"],
"film_strip" => ["film_frames"],
"balance_scale" => ["scales"],
"surfing_man" => ["surfer"],
"couplekiss_man_woman" => ["couplekiss"],
"couple_with_heart_woman_man" => ["couple_with_heart"],
"clamp" => ["compression"],
"dancing_women" => ["dancers"],
"blonde_man" => ["person_with_blond_hair"],
"sleeping_bed" => ["sleeping_accommodation"],
"om" => ["om_symbol"],
"tipping_hand_woman" => ["information_desk_person"],
"rowing_man" => ["rowboat"],
"new_moon" => ["moon"],
"oncoming_automobile" => %w[car automobile],
"fleur_de_lis" => ["fleur-de-lis"],
"face_vomiting" => ["puke"],
"smile" => ["grinning_face_with_smiling_eyes"],
"frowning_with_open_mouth" => ["frowning_face_with_open_mouth"],
}
EMOJI_GROUPS = [
{ "name" => "smileys_&_emotion", "tabicon" => "grinning" },
{ "name" => "people_&_body", "tabicon" => "wave" },
{ "name" => "animals_&_nature", "tabicon" => "evergreen_tree" },
{ "name" => "food_&_drink", "tabicon" => "hamburger" },
{ "name" => "travel_&_places", "tabicon" => "airplane" },
{ "name" => "activities", "tabicon" => "soccer" },
{ "name" => "objects", "tabicon" => "eyeglasses" },
{ "name" => "symbols", "tabicon" => "white_check_mark" },
{ "name" => "flags", "tabicon" => "checkered_flag" },
]
FITZPATRICK_SCALE = %w[1f3fb 1f3fc 1f3fd 1f3fe 1f3ff]
DEFAULT_SET = "twitter"
# Replace the platform by another when downloading the image (accepts names or categories)
EMOJI_IMAGES_PATCH = {
"apple" => {
"snowboarder" => "twitter",
},
"windows" => {
"country-flag" => "twitter",
},
}
EMOJI_SETS = {
"apple" => "apple",
"google" => "google",
"google_blob" => "google_classic",
"facebook" => "facebook_messenger",
"twitter" => "twitter",
"windows" => "win10",
}
EMOJI_DB_REPO = "git@github.com:xfalcox/emoji-db.git"
EMOJI_DB_REPO_PATH = File.join("tmp", "emoji-db")
GENERATED_PATH = File.join(EMOJI_DB_REPO_PATH, "generated")
def search_aliases(emojis)
# Format is search pattern => associated emojis
# eg: "cry" => [ "sob" ]
# for a "cry" query should return: cry and sob
@aliases ||=
begin
aliases = {
"sad" => %w[frowning_face slightly_frowning_face sob crying_cat_face cry],
"cry" => ["sob"],
}
emojis.each do |_, config|
next if config["search_aliases"].blank?
config["search_aliases"].each do |name|
aliases[name] ||= []
aliases[name] << config["name"]
end
end
aliases.map { |_, names| names.uniq! }
aliases
end
end
desc "update emoji images"
task "emoji:update" do
abort("This task can't be run on production.") if Rails.env.production?
copy_emoji_db
json_db = File.read(File.join(GENERATED_PATH, "db.json"))
db = JSON.parse(json_db)
write_db_json(db["emojis"], db["translations"], search_aliases(db["emojis"]))
fix_incomplete_sets(db["emojis"])
write_aliases
groups = generate_emoji_groups(db["emojis"], db["sections"])
write_js_groups(db["emojis"], groups)
optimize_images(Dir.glob(File.join(Rails.root, EMOJI_IMAGES_PATH, "/**/*.png")))
TestEmojiUpdate.run_and_summarize
FileUtils.rm_rf(EMOJI_DB_REPO_PATH)
end
desc "test the emoji generation script"
task "emoji:test" do
ENV["EMOJI_TEST"] = "1"
Rake::Task["emoji:update"].invoke
end
def optimize_images(images)
images.each do |filename|
FileHelper.image_optim(allow_pngquant: true, strip_image_metadata: true).optimize_image!(
filename,
)
end
end
def copy_emoji_db
`rm -rf tmp/emoji-db && git clone -b unicodeorg-as-source-of-truth --depth 1 #{EMOJI_DB_REPO} tmp/emoji-db`
path = "#{EMOJI_IMAGES_PATH}/**/*"
confirm_overwrite(path)
puts "Cleaning emoji folder..."
emoji_assets = Dir.glob(path)
emoji_assets.delete_if { |x| x == "#{EMOJI_IMAGES_PATH}/emoji_one" }
FileUtils.rm_rf(emoji_assets)
EMOJI_SETS.each do |set_name, set_destination|
origin = File.join(GENERATED_PATH, set_name)
destination = File.join(EMOJI_IMAGES_PATH, set_destination)
FileUtils.mv(origin, destination)
end
end
def fix_incomplete_sets(emojis)
emojis.each do |code, config|
EMOJI_SETS.each do |set_name, set_destination|
patch_set =
EMOJI_SETS[EMOJI_IMAGES_PATCH.dig(set_name, config["name"])] ||
EMOJI_SETS[EMOJI_IMAGES_PATCH.dig(set_name, config["category"])]
if patch_set ||
!File.exist?(File.join(EMOJI_IMAGES_PATH, set_destination, "#{config["name"]}.png"))
origin = File.join(EMOJI_IMAGES_PATH, patch_set || EMOJI_SETS[DEFAULT_SET], config["name"])
FileUtils.cp(
"#{origin}.png",
File.join(EMOJI_IMAGES_PATH, set_destination, "#{config["name"]}.png"),
)
if File.directory?(origin)
FileUtils.cp_r(origin, File.join(EMOJI_IMAGES_PATH, set_destination, config["name"]))
end
end
end
end
end
def generate_emoji_groups(keywords, sections)
puts "Generating groups..."
list = URI.parse(EMOJI_ORDERING_URL).read
doc = Nokogiri.HTML5(list)
table = doc.css("table")[0]
EMOJI_GROUPS.map do |group|
group["icons"] ||= []
sub_sections = sections[group["name"]]["sub_sections"]
sub_sections.each do |section|
title_section = table.css("tr th a[@name='#{section}']")
emoji_list_section = title_section.first.parent.parent.next_element
emoji_list_section
.css("a.plain img")
.each do |link|
emoji_code =
link
.attr("title")
.scan(/U\+(.{4,5})\b/)
.flatten
.map { |code| code.downcase.strip }
.join("_")
emoji_char = code_to_emoji(emoji_code)
if emoji = keywords[emoji_char]
group["icons"] << { name: emoji["name"], diversity: emoji["fitzpatrick_scale"] }
end
end
end
group.delete("sections")
group
end
end
def write_aliases
EMOJI_ALIASES.each do |original, aliases|
aliases.each do |emoji_alias|
EMOJI_SETS.each do |set_name, set_destination|
origin_file = File.join(EMOJI_IMAGES_PATH, set_destination, "#{original}.png")
origin_dir = File.join(EMOJI_IMAGES_PATH, set_destination, original)
FileUtils.cp(
origin_file,
File.join(EMOJI_IMAGES_PATH, set_destination, "#{emoji_alias}.png"),
)
if File.directory?(origin_dir)
FileUtils.cp_r(origin_dir, File.join(EMOJI_IMAGES_PATH, set_destination, emoji_alias))
end
end
end
end
end
def write_db_json(emojis, translations, search_aliases)
puts "Writing #{EMOJI_DB_PATH}..."
confirm_overwrite(EMOJI_DB_PATH)
FileUtils.mkdir_p(File.expand_path("..", EMOJI_DB_PATH))
# skin tones variations of emojis shouldn’t appear in autocomplete
emojis_without_tones =
emojis
.select do |char, config|
!FITZPATRICK_SCALE.any? do |scale|
codepoints_to_code(char.codepoints, config["fitzpatrick_scale"])[scale]
end
end
.map do |char, config|
{
"code" => codepoints_to_code(char.codepoints, config["fitzpatrick_scale"]).tr("_", "-"),
"name" => config["name"],
}
end
emoji_with_tones =
emojis
.select { |code, config| config["fitzpatrick_scale"] }
.map { |code, config| config["name"] }
db = {
"emojis" => emojis_without_tones,
"tonableEmojis" => emoji_with_tones,
"aliases" => EMOJI_ALIASES,
"searchAliases" => search_aliases,
"translations" => translations,
}
File.write(EMOJI_DB_PATH, JSON.pretty_generate(db))
end
def write_js_groups(emojis, groups)
puts "Writing #{EMOJI_GROUPS_PATH}..."
confirm_overwrite(EMOJI_GROUPS_PATH)
template = JSON.pretty_generate(groups)
FileUtils.mkdir_p(File.expand_path("..", EMOJI_GROUPS_PATH))
File.write(EMOJI_GROUPS_PATH, template)
end
def code_to_emoji(code)
code.split("_").map { |e| e.to_i(16) }.pack "U*"
end
def codepoints_to_code(codepoints, fitzpatrick_scale)
codepoints = codepoints.map { |c| c.to_s(16).rjust(4, "0") }.join("_").downcase
codepoints.gsub!(/_fe0f\z/, "") if !fitzpatrick_scale
codepoints
end
def confirm_overwrite(path)
return if ENV["EMOJI_TEST"]
STDOUT.puts(
"[!] You are about to overwrite #{path}, are you sure? [CTRL+c] to cancel, [ENTER] to continue",
)
STDIN.gets.chomp
end
class TestEmojiUpdate
def self.run_and_summarize
puts "Running tests..."
instance = TestEmojiUpdate.new
instance.public_methods.each do |method|
next unless method.to_s.start_with? "test_"
print "Running #{method}..."
instance.public_send(method)
puts " ✅"
rescue StandardError => e
puts " ❌"
puts e.message.indent(2)
end
end
def assert_equal(a, b)
raise "Expected #{a.inspect} to equal #{b.inspect}" if a != b
end
def assert(a)
raise "Expected #{a.inspect} to be truthy" if !a
end
def image_path(style, name)
File.join("public", "images", "emoji", style, "#{name}.png")
end
def test_code_to_emoji
assert_equal "😎", code_to_emoji("1f60e")
end
def test_codepoints_to_code
assert_equal "1f6b5_200d_2640", codepoints_to_code([128_693, 8205, 9792, 65_039], false)
end
def test_codepoints_to_code_with_scale
assert_equal "1f6b5_200d_2640_fe0f", codepoints_to_code([128_693, 8205, 9792, 65_039], true)
end
def test_groups_js_es6_creation
assert File.exist?(EMOJI_GROUPS_PATH)
assert File.size?(EMOJI_GROUPS_PATH)
end
def test_db_json_creation
assert File.exist?(EMOJI_DB_PATH)
assert File.size?(EMOJI_DB_PATH)
end
def test_alias_creation
original_image = image_path("apple", "right_anger_bubble")
alias_image = image_path("apple", "anger_right")
assert_equal File.size(original_image), File.size(alias_image)
end
def test_cell_index_patch
original_image = image_path("apple", "snowboarder")
alias_image = image_path("twitter", "snowboarder")
assert_equal File.size(original_image), File.size(alias_image)
end
def test_scales
original_image = image_path("apple", "blonde_woman")
assert File.exist?(original_image)
assert File.size?(original_image)
(2..6).each do |scale|
image = image_path("apple", "blonde_woman/#{scale}")
assert File.exist?(image)
assert File.size?(image)
end
end
def test_default_set
original_image = image_path("twitter", "snowboarder")
alias_image = image_path("apple", "snowboarder")
assert_equal File.size(original_image), File.size(alias_image)
original_image = image_path("twitter", "macau")
alias_image = image_path("win10", "macau")
assert_equal File.size(original_image), File.size(alias_image)
end
end

3
lib/tasks/javascript.rake
View File

@ -184,10 +184,9 @@ task "javascript:update_constants" => :environment do
JS
write_template("pretty-text/addon/emoji/data.js", task_name, <<~JS)
export const emojis = #{Emoji.standard.map(&:name).flatten.inspect};
export const emojis = new Set(#{Emoji.standard.map(&:name).flatten.inspect});
export const tonableEmojis = #{Emoji.tonable_emojis.flatten.inspect};
export const aliases = #{Emoji.aliases.inspect.gsub("=>", ":")};
export const searchAliases = #{Emoji.search_aliases.inspect.gsub("=>", ":")};
export const translations = #{Emoji.translations.inspect.gsub("=>", ":")};
export const replacements = #{Emoji.unicode_replacements_json};
JS