diff --git a/Gemfile b/Gemfile index b7507fed24c..85cabafff2c 100644 --- a/Gemfile +++ b/Gemfile @@ -203,6 +203,8 @@ gem "sassc-rails" gem 'rotp' gem 'rqrcode' +gem 'rubyzip', require: false + gem 'sshkey', require: false gem 'rchardet', require: false diff --git a/Gemfile.lock b/Gemfile.lock index b81b19fa6a7..f0faf0292b0 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -351,6 +351,7 @@ GEM guess_html_encoding (>= 0.0.4) nokogiri (>= 1.6.0) ruby_dep (1.5.0) + rubyzip (1.2.3) safe_yaml (1.0.5) sanitize (5.0.0) crass (~> 1.0.2) @@ -516,6 +517,7 @@ DEPENDENCIES rubocop ruby-prof ruby-readability + rubyzip sanitize sassc sassc-rails diff --git a/app/assets/javascripts/admin/templates/modal/admin-install-theme.hbs b/app/assets/javascripts/admin/templates/modal/admin-install-theme.hbs index 153dbc9e8f1..9a1e18d0e7f 100644 --- a/app/assets/javascripts/admin/templates/modal/admin-install-theme.hbs +++ b/app/assets/javascripts/admin/templates/modal/admin-install-theme.hbs @@ -44,7 +44,7 @@ {{#if local}}
-
+
{{i18n 'admin.customize.theme.import_file_tip'}}
{{/if}} diff --git a/app/controllers/admin/themes_controller.rb b/app/controllers/admin/themes_controller.rb index 5389269af87..4def5bd2149 100644 --- a/app/controllers/admin/themes_controller.rb +++ b/app/controllers/admin/themes_controller.rb @@ -88,7 +88,7 @@ class Admin::ThemesController < Admin::AdminController rescue RemoteTheme::ImportError => e render_json_error e.message end - elsif params[:bundle] || (params[:theme] && ["application/x-gzip", "application/gzip"].include?(params[:theme].content_type)) + elsif params[:bundle] || (params[:theme] && ["application/x-gzip", "application/gzip", "application/zip"].include?(params[:theme].content_type)) # params[:bundle] used by theme CLI. params[:theme] used by admin UI bundle = params[:bundle] || params[:theme] theme_id = params[:theme_id] @@ -252,6 +252,7 @@ class Admin::ThemesController < Admin::AdminController exporter = ThemeStore::TgzExporter.new(@theme) file_path = exporter.package_filename + headers['Content-Length'] = File.size(file_path).to_s send_data File.read(file_path), filename: File.basename(file_path), diff --git a/app/jobs/regular/export_csv_file.rb b/app/jobs/regular/export_csv_file.rb index e1b57398c1a..d8d3a7204ec 100644 --- a/app/jobs/regular/export_csv_file.rb +++ b/app/jobs/regular/export_csv_file.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require 'csv' +require 'zip' require_dependency 'system_message' require_dependency 'upload_creator' @@ -53,18 +54,19 @@ module Jobs # ensure directory exists FileUtils.mkdir_p(UserExport.base_directory) unless Dir.exists?(UserExport.base_directory) - # write to CSV file - CSV.open(absolute_path, "w") do |csv| + # Generate a compressed CSV file + csv_to_export = CSV.generate do |csv| csv << get_header if @entity != "report" public_send(export_method).each { |d| csv << d } end - # compress CSV file - system('gzip', '-5', absolute_path) + compressed_file_path = "#{absolute_path}.zip" + Zip::File.open(compressed_file_path, Zip::File::CREATE) do |zipfile| + zipfile.get_output_stream(file_name) { |f| f.puts csv_to_export } + end # create upload upload = nil - compressed_file_path = "#{absolute_path}.gz" if File.exist?(compressed_file_path) File.open(compressed_file_path) do |file| diff --git a/config/locales/client.en.yml b/config/locales/client.en.yml index aad6399c5f2..d450f80ff4e 100644 --- a/config/locales/client.en.yml +++ b/config/locales/client.en.yml @@ -3514,7 +3514,7 @@ en: delete_upload_confirm: "Delete this upload? (Theme CSS may stop working!)" import_web_tip: "Repository containing theme" import_web_advanced: "Advanced..." - import_file_tip: ".tar.gz or .dcstyle.json file containing theme" + import_file_tip: ".tar.gz, .tar.zip, or .dcstyle.json file containing theme" is_private: "Theme is in a private git repository" remote_branch: "Branch name (optional)" public_key: "Grant the following public key access to the repo:" diff --git a/config/locales/server.en.yml b/config/locales/server.en.yml index 74a2b8ee614..014c84bbb97 100644 --- a/config/locales/server.en.yml +++ b/config/locales/server.en.yml @@ -2668,7 +2668,7 @@ en: The above download link will be valid for 48 hours. - The data is compressed as a gzip archive. If the archive does not extract itself when you open it, use the tools recommended here: https://www.gzip.org/#faq4 + The data is compressed as a zip archive. If the archive does not extract itself when you open it, use the tool recommended here: https://www.7-zip.org/ csv_export_failed: title: "CSV Export Failed" diff --git a/config/site_settings.yml b/config/site_settings.yml index 1a9fbb94dd1..f29155b64e5 100644 --- a/config/site_settings.yml +++ b/config/site_settings.yml @@ -1048,7 +1048,7 @@ files: list_type: compact export_authorized_extensions: hidden: true - default: "gz" + default: "zip" type: list list_type: compact responsive_post_image_sizes: diff --git a/lib/theme_store/tgz_exporter.rb b/lib/theme_store/tgz_exporter.rb index 824a874ab94..d0d8cef27a7 100644 --- a/lib/theme_store/tgz_exporter.rb +++ b/lib/theme_store/tgz_exporter.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require 'zip' + module ThemeStore; end class ThemeStore::TgzExporter @@ -58,11 +60,19 @@ class ThemeStore::TgzExporter private def export_package export_to_folder + Dir.chdir(@temp_folder) do tar_filename = "#{@export_name}.tar" Discourse::Utils.execute_command('tar', '--create', '--file', tar_filename, @export_name, failure_message: "Failed to tar theme.") - Discourse::Utils.execute_command('gzip', '-5', tar_filename, failure_message: "Failed to gzip archive.") - "#{@temp_folder}/#{tar_filename}.gz" + + zip_filename = "#{tar_filename}.zip" + absolute_path = "#{@temp_folder}/#{tar_filename}" + Zip::File.open(zip_filename, Zip::File::CREATE) do |zipfile| + zipfile.add(tar_filename, absolute_path) + zipfile.close + end + + "#{absolute_path}.zip" end end diff --git a/lib/theme_store/tgz_importer.rb b/lib/theme_store/tgz_importer.rb index 5dfb0716e68..e6198ee46e7 100644 --- a/lib/theme_store/tgz_importer.rb +++ b/lib/theme_store/tgz_importer.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require 'zip' + module ThemeStore; end class ThemeStore::TgzImporter @@ -13,8 +15,21 @@ class ThemeStore::TgzImporter def import! FileUtils.mkdir(@temp_folder) - Dir.chdir(@temp_folder) do - Discourse::Utils.execute_command("tar", "-xzvf", @filename, "--strip", "1") + + if @filename.include?('.zip') + name = @filename.split('/').last.gsub('.zip', '') + + Dir.chdir(@temp_folder) do + Zip::File.open(@filename) do |zip_file| + zip_file.each { |entry| entry.extract(name) } + end + + Discourse::Utils.execute_command("tar", "-xvf", name, "--strip", "1") + end + else + Dir.chdir(@temp_folder) do + Discourse::Utils.execute_command("tar", "-xzvf", @filename, "--strip", "1") + end end rescue RuntimeError raise RemoteTheme::ImportError, I18n.t("themes.import_error.unpack_failed") diff --git a/spec/components/theme_store/tgz_exporter_spec.rb b/spec/components/theme_store/tgz_exporter_spec.rb index b66bf6e1b08..34dd280285d 100644 --- a/spec/components/theme_store/tgz_exporter_spec.rb +++ b/spec/components/theme_store/tgz_exporter_spec.rb @@ -2,6 +2,7 @@ require 'rails_helper' require 'theme_store/tgz_exporter' +require 'zip' describe ThemeStore::TgzExporter do let!(:theme) do @@ -55,13 +56,19 @@ describe ThemeStore::TgzExporter do filename = exporter.package_filename FileUtils.cp(filename, dir) exporter.cleanup! - "#{dir}/discourse-header-icons.tar.gz" + "#{dir}/discourse-header-icons.tar.zip" end it "exports the theme correctly" do package + file = 'discourse-header-icons.tar.zip' + dest = 'discourse-header-icons.tar' Dir.chdir("#{dir}") do - `tar -xzf discourse-header-icons.tar.gz` + Zip::File.open(file) do |zip_file| + zip_file.each { |entry| entry.extract(dest) } + end + + `tar -xvf discourse-header-icons.tar 2> /dev/null` end Dir.chdir("#{dir}/discourse-header-icons") do folders = Dir.glob("**/*").reject { |f| File.file?(f) } @@ -121,7 +128,7 @@ describe ThemeStore::TgzExporter do exporter = ThemeStore::TgzExporter.new(theme) filename = exporter.package_filename exporter.cleanup! - expect(filename).to end_with "/discourse-header-icons.tar.gz" + expect(filename).to end_with "/discourse-header-icons.tar.zip" end end diff --git a/spec/components/theme_store/tgz_importer_spec.rb b/spec/components/theme_store/tgz_importer_spec.rb index 2986b1d2c9c..ab2d982da6f 100644 --- a/spec/components/theme_store/tgz_importer_spec.rb +++ b/spec/components/theme_store/tgz_importer_spec.rb @@ -4,26 +4,46 @@ require 'rails_helper' require 'theme_store/tgz_importer' +require 'zip' describe ThemeStore::TgzImporter do before do @temp_folder = "#{Pathname.new(Dir.tmpdir).realpath}/discourse_theme_#{SecureRandom.hex}" + + FileUtils.mkdir(@temp_folder) + Dir.chdir(@temp_folder) do + FileUtils.mkdir('test/') + File.write("test/hello.txt", "hello world") + FileUtils.mkdir('test/a') + File.write("test/a/inner", "hello world inner") + end end after do FileUtils.rm_rf @temp_folder end - it "can import a simple theme" do - - FileUtils.mkdir(@temp_folder) - + it "can import a simple zipped theme" do Dir.chdir(@temp_folder) do - FileUtils.mkdir('test/') - File.write("test/hello.txt", "hello world") - FileUtils.mkdir('test/a') - File.write("test/a/inner", "hello world inner") + `tar -cvf test.tar test/* 2> /dev/null` + Zip::File.open('test.tar.zip', Zip::File::CREATE) do |zipfile| + zipfile.add('test.tar', "#{@temp_folder}/test.tar") + zipfile.close + end + end + + importer = ThemeStore::TgzImporter.new("#{@temp_folder}/test.tar.zip") + importer.import! + + expect(importer["hello.txt"]).to eq("hello world") + expect(importer["a/inner"]).to eq("hello world inner") + + importer.cleanup! + end + + it "can import a simple gzipped theme" do + Dir.chdir(@temp_folder) do `tar -cvzf test.tar.gz test/* 2> /dev/null` end diff --git a/spec/components/validators/upload_validator_spec.rb b/spec/components/validators/upload_validator_spec.rb index e9a3dfe601a..9348c85e79a 100644 --- a/spec/components/validators/upload_validator_spec.rb +++ b/spec/components/validators/upload_validator_spec.rb @@ -22,14 +22,14 @@ describe Validators::UploadValidator do it "allows 'gz' as extension when uploading export file" do SiteSetting.authorized_extensions = "" - expect(UploadCreator.new(csv_file, "#{filename}.gz", for_export: true).create_for(user.id)).to be_valid + expect(UploadCreator.new(csv_file, "#{filename}.zip", for_export: true).create_for(user.id)).to be_valid end it "allows uses max_export_file_size_kb when uploading export file" do SiteSetting.max_attachment_size_kb = "0" - SiteSetting.authorized_extensions = "gz" + SiteSetting.authorized_extensions = "zip" - expect(UploadCreator.new(csv_file, "#{filename}.gz", for_export: true).create_for(user.id)).to be_valid + expect(UploadCreator.new(csv_file, "#{filename}.zip", for_export: true).create_for(user.id)).to be_valid end describe 'when allow_staff_to_upload_any_file_in_pm is true' do diff --git a/spec/requests/admin/themes_controller_spec.rb b/spec/requests/admin/themes_controller_spec.rb index 4f32a63e013..43cc16cd8b8 100644 --- a/spec/requests/admin/themes_controller_spec.rb +++ b/spec/requests/admin/themes_controller_spec.rb @@ -51,10 +51,10 @@ describe Admin::ThemesController do expect(response.status).to eq(200) # Save the output in a temp file (automatically cleaned up) - file = Tempfile.new('archive.tar.gz') + file = Tempfile.new('archive.tar.zip') file.write(response.body) file.rewind - uploaded_file = Rack::Test::UploadedFile.new(file.path, "application/x-gzip") + uploaded_file = Rack::Test::UploadedFile.new(file.path, "application/zip") # Now import it again expect do