From 24cb950432e1fa321d3adf5511543ec57813925a Mon Sep 17 00:00:00 2001 From: Neil Lalonde Date: Wed, 28 Jun 2017 16:56:44 -0400 Subject: [PATCH] FEATURE: Watched Words: when posts contain words, do one of flag, require approval, censor, or block --- .../components/admin-watched-word.js.es6 | 19 ++ .../admin/components/watched-word-form.js.es6 | 49 +++++ .../components/watched-word-uploader.js.es6 | 25 +++ .../admin-watched-words-action.js.es6 | 65 +++++++ .../controllers/admin-watched-words.js.es6 | 51 ++++++ .../admin/models/watched-word.js.es6 | 37 ++++ .../admin/routes/admin-route-map.js.es6 | 5 + .../routes/admin-watched-words-action.js.es6 | 11 ++ .../routes/admin-watched-words-index.js.es6 | 5 + .../admin/routes/admin-watched-words.js.es6 | 15 ++ .../javascripts/admin/templates/admin.hbs | 1 + .../components/watched-word-form.hbs | 7 + .../components/watched-word-uploader.hbs | 7 + .../admin/templates/watched-words-action.hbs | 18 ++ .../admin/templates/watched-words.hbs | 31 ++++ .../javascripts/discourse/lib/text.js.es6 | 4 +- .../javascripts/discourse/models/topic.js.es6 | 6 +- .../discourse-markdown/censored.js.es6 | 1 - .../pretty-text/pretty-text.js.es6 | 4 +- .../stylesheets/common/admin/admin_base.scss | 41 +++++ .../admin/watched_words_controller.rb | 47 +++++ app/jobs/onceoff/migrate_censored_words.rb | 12 ++ app/jobs/regular/process_post.rb | 8 + app/models/watched_word.rb | 54 ++++++ app/serializers/site_serializer.rb | 7 +- .../watched_word_list_serializer.rb | 13 ++ app/serializers/watched_word_serializer.rb | 7 + app/services/word_watcher.rb | 51 ++++++ config/locales/client.en.yml | 25 +++ config/locales/server.en.yml | 5 + config/routes.rb | 6 + config/site_settings.yml | 5 - .../20170628152322_create_watched_words.rb | 11 ++ lib/new_post_manager.rb | 23 ++- lib/post_revisor.rb | 6 + lib/pretty_text.rb | 1 + lib/validators/censored_words_validator.rb | 7 +- .../lib/details-cooked-test.js.es6 | 2 +- spec/components/new_post_manager_spec.rb | 10 +- spec/components/pretty_text_spec.rb | 6 +- spec/fabricators/watched_word_fabricator.rb | 4 + spec/integration/watched_words_spec.rb | 169 ++++++++++++++++++ spec/models/topic_spec.rb | 14 +- spec/models/watched_word_spec.rb | 92 ++++++++++ spec/services/word_watcher_spec.rb | 53 ++++++ .../admin-watched-words-test.js.es6 | 68 +++++++ .../fixtures/watched-words-fixtures.js.es6 | 12 ++ .../helpers/create-pretender.js.es6 | 11 ++ test/javascripts/lib/pretty-text-test.js.es6 | 2 +- 49 files changed, 1096 insertions(+), 37 deletions(-) create mode 100644 app/assets/javascripts/admin/components/admin-watched-word.js.es6 create mode 100644 app/assets/javascripts/admin/components/watched-word-form.js.es6 create mode 100644 app/assets/javascripts/admin/components/watched-word-uploader.js.es6 create mode 100644 app/assets/javascripts/admin/controllers/admin-watched-words-action.js.es6 create mode 100644 app/assets/javascripts/admin/controllers/admin-watched-words.js.es6 create mode 100644 app/assets/javascripts/admin/models/watched-word.js.es6 create mode 100644 app/assets/javascripts/admin/routes/admin-watched-words-action.js.es6 create mode 100644 app/assets/javascripts/admin/routes/admin-watched-words-index.js.es6 create mode 100644 app/assets/javascripts/admin/routes/admin-watched-words.js.es6 create mode 100644 app/assets/javascripts/admin/templates/components/watched-word-form.hbs create mode 100644 app/assets/javascripts/admin/templates/components/watched-word-uploader.hbs create mode 100644 app/assets/javascripts/admin/templates/watched-words-action.hbs create mode 100644 app/assets/javascripts/admin/templates/watched-words.hbs create mode 100644 app/controllers/admin/watched_words_controller.rb create mode 100644 app/jobs/onceoff/migrate_censored_words.rb create mode 100644 app/models/watched_word.rb create mode 100644 app/serializers/watched_word_list_serializer.rb create mode 100644 app/serializers/watched_word_serializer.rb create mode 100644 app/services/word_watcher.rb create mode 100644 db/migrate/20170628152322_create_watched_words.rb create mode 100644 spec/fabricators/watched_word_fabricator.rb create mode 100644 spec/integration/watched_words_spec.rb create mode 100644 spec/models/watched_word_spec.rb create mode 100644 spec/services/word_watcher_spec.rb create mode 100644 test/javascripts/acceptance/admin-watched-words-test.js.es6 create mode 100644 test/javascripts/fixtures/watched-words-fixtures.js.es6 diff --git a/app/assets/javascripts/admin/components/admin-watched-word.js.es6 b/app/assets/javascripts/admin/components/admin-watched-word.js.es6 new file mode 100644 index 00000000000..9a454ad57ee --- /dev/null +++ b/app/assets/javascripts/admin/components/admin-watched-word.js.es6 @@ -0,0 +1,19 @@ +import { iconHTML } from 'discourse-common/helpers/fa-icon'; +import { bufferedRender } from 'discourse-common/lib/buffered-render'; + +export default Ember.Component.extend(bufferedRender({ + classNames: ['watched-word'], + + buildBuffer(buffer) { + buffer.push(iconHTML('times')); + buffer.push(' ' + this.get('word.word')); + }, + + click() { + this.get('word').destroy().then(() => { + this.sendAction('action', this.get('word')); + }).catch(e => { + bootbox.alert(I18n.t("generic_error_with_reason", {error: "http: " + e.status + " - " + e.body})); + });; + } +})); diff --git a/app/assets/javascripts/admin/components/watched-word-form.js.es6 b/app/assets/javascripts/admin/components/watched-word-form.js.es6 new file mode 100644 index 00000000000..95f859abf4b --- /dev/null +++ b/app/assets/javascripts/admin/components/watched-word-form.js.es6 @@ -0,0 +1,49 @@ +import WatchedWord from 'admin/models/watched-word'; +import { on, observes } from 'ember-addons/ember-computed-decorators'; + +export default Ember.Component.extend({ + classNames: ['watched-word-form'], + formSubmitted: false, + actionKey: null, + showSuccessMessage: false, + + @observes('word') + removeSuccessMessage() { + if (this.get('showSuccessMessage') && !Ember.isEmpty(this.get('word'))) { + this.set('showSuccessMessage', false); + } + }, + + actions: { + submit() { + if (!this.get('formSubmitted')) { + this.set('formSubmitted', true); + + const watchedWord = WatchedWord.create({ word: this.get('word'), action: this.get('actionKey') }); + + watchedWord.save().then(result => { + this.setProperties({ word: '', formSubmitted: false, showSuccessMessage: true }); + this.sendAction('action', WatchedWord.create(result)); + Ember.run.schedule('afterRender', () => this.$('.watched-word-input').focus()); + }).catch(e => { + this.set('formSubmitted', false); + const msg = (e.responseJSON && e.responseJSON.errors) ? + I18n.t("generic_error_with_reason", {error: e.responseJSON.errors.join('. ')}) : + I18n.t("generic_error"); + bootbox.alert(msg, () => this.$('.watched-word-input').focus()); + }); + } + } + }, + + @on("didInsertElement") + _init() { + Ember.run.schedule('afterRender', () => { + this.$('.watched-word-input').keydown(e => { + if (e.keyCode === 13) { + this.send('submit'); + } + }); + }); + } +}); diff --git a/app/assets/javascripts/admin/components/watched-word-uploader.js.es6 b/app/assets/javascripts/admin/components/watched-word-uploader.js.es6 new file mode 100644 index 00000000000..c6d39c52a9a --- /dev/null +++ b/app/assets/javascripts/admin/components/watched-word-uploader.js.es6 @@ -0,0 +1,25 @@ +import computed from "ember-addons/ember-computed-decorators"; +import UploadMixin from "discourse/mixins/upload"; + +export default Em.Component.extend(UploadMixin, { + type: 'csv', + classNames: 'watched-words-uploader', + uploadUrl: '/admin/watched_words/upload', + addDisabled: Em.computed.alias("uploading"), + + validateUploadedFilesOptions() { + return { csvOnly: true }; + }, + + @computed('actionKey') + data(actionKey) { + return { action_key: actionKey }; + }, + + uploadDone() { + if (this) { + bootbox.alert(I18n.t("admin.watched_words.form.upload_successful")); + this.sendAction("done"); + } + } +}); diff --git a/app/assets/javascripts/admin/controllers/admin-watched-words-action.js.es6 b/app/assets/javascripts/admin/controllers/admin-watched-words-action.js.es6 new file mode 100644 index 00000000000..bd11f15fcff --- /dev/null +++ b/app/assets/javascripts/admin/controllers/admin-watched-words-action.js.es6 @@ -0,0 +1,65 @@ +import computed from 'ember-addons/ember-computed-decorators'; +import WatchedWord from 'admin/models/watched-word'; + +export default Ember.Controller.extend({ + actionNameKey: null, + adminWatchedWords: Ember.inject.controller(), + showWordsList: Ember.computed.or('adminWatchedWords.filtered', 'adminWatchedWords.showWords'), + + findAction(actionName) { + return (this.get('adminWatchedWords.model') || []).findBy('nameKey', actionName); + }, + + @computed('adminWatchedWords.model', 'actionNameKey') + filteredContent() { + if (!this.get('actionNameKey')) { return []; } + + const a = this.findAction(this.get('actionNameKey')); + return a ? a.words : []; + }, + + @computed('actionNameKey') + actionDescription(actionNameKey) { + return I18n.t('admin.watched_words.action_descriptions.' + actionNameKey); + }, + + actions: { + recordAdded(arg) { + const a = this.findAction(this.get('actionNameKey')); + if (a) { + a.words.unshiftObject(arg); + a.incrementProperty('count'); + Em.run.schedule('afterRender', () => { + // remove from other actions lists + let match = null; + this.get('adminWatchedWords.model').forEach(action => { + if (match) return; + + if (action.nameKey !== this.get('actionNameKey')) { + match = action.words.findBy('id', arg.id); + if (match) { + action.words.removeObject(match); + action.decrementProperty('count'); + } + } + }); + }); + } + }, + + recordRemoved(arg) { + const a = this.findAction(this.get('actionNameKey')); + if (a) { + a.words.removeObject(arg); + a.decrementProperty('count'); + } + }, + + uploadComplete() { + WatchedWord.findAll().then(data => { + this.set('adminWatchedWords.model', data); + }); + } + } + +}); diff --git a/app/assets/javascripts/admin/controllers/admin-watched-words.js.es6 b/app/assets/javascripts/admin/controllers/admin-watched-words.js.es6 new file mode 100644 index 00000000000..07d53fd2301 --- /dev/null +++ b/app/assets/javascripts/admin/controllers/admin-watched-words.js.es6 @@ -0,0 +1,51 @@ +import debounce from 'discourse/lib/debounce'; + +export default Ember.Controller.extend({ + filter: null, + filtered: false, + showWords: false, + disableShowWords: Ember.computed.alias('filtered'), + + filterContentNow() { + + if (!!Ember.isEmpty(this.get('allWatchedWords'))) return; + + let filter; + if (this.get('filter')) { + filter = this.get('filter').toLowerCase(); + } + + if (filter === undefined || filter.length < 1) { + this.set('model', this.get('allWatchedWords')); + return; + } + + const matchesByAction = []; + + this.get('allWatchedWords').forEach(wordsForAction => { + const wordRecords = wordsForAction.words.filter(wordRecord => { + return (wordRecord.word.indexOf(filter) > -1); + }); + matchesByAction.pushObject( Ember.Object.create({ + nameKey: wordsForAction.nameKey, + name: wordsForAction.name, + words: wordRecords, + count: wordRecords.length + }) ); + }); + + this.set('model', matchesByAction); + }, + + filterContent: debounce(function() { + this.filterContentNow(); + this.set('filtered', !Ember.isEmpty(this.get('filter'))); + }, 250).observes('filter'), + + actions: { + clearFilter() { + this.setProperties({ filter: '' }); + } + } + +}); diff --git a/app/assets/javascripts/admin/models/watched-word.js.es6 b/app/assets/javascripts/admin/models/watched-word.js.es6 new file mode 100644 index 00000000000..f5d42df1b92 --- /dev/null +++ b/app/assets/javascripts/admin/models/watched-word.js.es6 @@ -0,0 +1,37 @@ +import { ajax } from 'discourse/lib/ajax'; + +const WatchedWord = Discourse.Model.extend({ + save() { + return ajax("/admin/watched_words" + (this.id ? '/' + this.id : '') + ".json", { + type: this.id ? 'PUT' : 'POST', + data: {word: this.get('word'), action_key: this.get('action')}, + dataType: 'json' + }); + }, + + destroy() { + return ajax("/admin/watched_words/" + this.get('id') + ".json", {type: 'DELETE'}); + } +}); + +WatchedWord.reopenClass({ + findAll() { + return ajax("/admin/watched_words").then(function (list) { + const actions = {}; + list.words.forEach(s => { + if (!actions[s.action]) { actions[s.action] = []; } + actions[s.action].pushObject(WatchedWord.create(s)); + }); + + list.actions.forEach(a => { + if (!actions[a]) { actions[a] = []; } + }); + + return Object.keys(actions).map(function(n) { + return Ember.Object.create({nameKey: n, name: I18n.t('admin.watched_words.actions.' + n), words: actions[n], count: actions[n].length}); + }); + }); + } +}); + +export default WatchedWord; diff --git a/app/assets/javascripts/admin/routes/admin-route-map.js.es6 b/app/assets/javascripts/admin/routes/admin-route-map.js.es6 index dd87207156a..86d624bab79 100644 --- a/app/assets/javascripts/admin/routes/admin-route-map.js.es6 +++ b/app/assets/javascripts/admin/routes/admin-route-map.js.es6 @@ -90,5 +90,10 @@ export default function() { this.route('adminPlugins', { path: '/plugins', resetNamespace: true }, function() { this.route('index', { path: '/' }); }); + + this.route('adminWatchedWords', { path: '/watched_words', resetNamespace: true}, function() { + this.route('index', { path: '/' } ); + this.route('action', { path: '/action/:action_id' } ); + }); }); }; diff --git a/app/assets/javascripts/admin/routes/admin-watched-words-action.js.es6 b/app/assets/javascripts/admin/routes/admin-watched-words-action.js.es6 new file mode 100644 index 00000000000..123884d6fc6 --- /dev/null +++ b/app/assets/javascripts/admin/routes/admin-watched-words-action.js.es6 @@ -0,0 +1,11 @@ +export default Discourse.Route.extend({ + model(params) { + this.controllerFor('adminWatchedWordsAction').set('actionNameKey', params.action_id); + let filteredContent = this.controllerFor('adminWatchedWordsAction').get('filteredContent'); + return Ember.Object.create({ + nameKey: params.action_id, + name: I18n.t('admin.watched_words.actions.' + params.action_id), + words: filteredContent + }); + } +}); diff --git a/app/assets/javascripts/admin/routes/admin-watched-words-index.js.es6 b/app/assets/javascripts/admin/routes/admin-watched-words-index.js.es6 new file mode 100644 index 00000000000..0103744b52d --- /dev/null +++ b/app/assets/javascripts/admin/routes/admin-watched-words-index.js.es6 @@ -0,0 +1,5 @@ +export default Discourse.Route.extend({ + beforeModel() { + this.replaceWith('adminWatchedWords.action', this.modelFor('adminWatchedWords')[0].nameKey); + } +}); diff --git a/app/assets/javascripts/admin/routes/admin-watched-words.js.es6 b/app/assets/javascripts/admin/routes/admin-watched-words.js.es6 new file mode 100644 index 00000000000..77f9082b576 --- /dev/null +++ b/app/assets/javascripts/admin/routes/admin-watched-words.js.es6 @@ -0,0 +1,15 @@ +import WatchedWord from 'admin/models/watched-word'; + +export default Discourse.Route.extend({ + queryParams: { + filter: { replace: true } + }, + + model() { + return WatchedWord.findAll(); + }, + + afterModel(watchedWordsList) { + this.controllerFor('adminWatchedWords').set('allWatchedWords', watchedWordsList); + } +}); diff --git a/app/assets/javascripts/admin/templates/admin.hbs b/app/assets/javascripts/admin/templates/admin.hbs index 780aad41fcf..525a62626e6 100644 --- a/app/assets/javascripts/admin/templates/admin.hbs +++ b/app/assets/javascripts/admin/templates/admin.hbs @@ -22,6 +22,7 @@ {{nav-item route='adminApi' label='admin.api.title'}} {{nav-item route='admin.backups' label='admin.backups.title'}} {{/if}} + {{nav-item route='adminWatchedWords' label='admin.watched_words.title'}} {{nav-item route='adminPlugins' label='admin.plugins.title'}} {{plugin-outlet name="admin-menu" connectorTagName="li"}} diff --git a/app/assets/javascripts/admin/templates/components/watched-word-form.hbs b/app/assets/javascripts/admin/templates/components/watched-word-form.hbs new file mode 100644 index 00000000000..64fbcb6570a --- /dev/null +++ b/app/assets/javascripts/admin/templates/components/watched-word-form.hbs @@ -0,0 +1,7 @@ +{{i18n 'admin.watched_words.form.label'}} +{{text-field value=word disabled=formSubmitted class="watched-word-input" autocorrect="off" autocapitalize="off"}} +{{d-button action="submit" disabled=formSubmitted label="admin.watched_words.form.add"}} + +{{#if showSuccessMessage}} + {{i18n 'admin.watched_words.form.success'}} +{{/if}} diff --git a/app/assets/javascripts/admin/templates/components/watched-word-uploader.hbs b/app/assets/javascripts/admin/templates/components/watched-word-uploader.hbs new file mode 100644 index 00000000000..2582bdcbb0a --- /dev/null +++ b/app/assets/javascripts/admin/templates/components/watched-word-uploader.hbs @@ -0,0 +1,7 @@ + +
+One word per line diff --git a/app/assets/javascripts/admin/templates/watched-words-action.hbs b/app/assets/javascripts/admin/templates/watched-words-action.hbs new file mode 100644 index 00000000000..3a7351d9ed4 --- /dev/null +++ b/app/assets/javascripts/admin/templates/watched-words-action.hbs @@ -0,0 +1,18 @@ +

{{model.name}}

+ +

{{actionDescription}}

+ +{{watched-word-form actionKey=actionNameKey action="recordAdded"}} + +{{watched-word-uploader uploading=uploading actionKey=actionNameKey done="uploadComplete"}} + +
+
+ {{#if showWordsList}} + {{#each filteredContent as |word| }} +
{{admin-watched-word word=word action="recordRemoved"}}
+ {{/each}} + {{else}} + {{i18n 'admin.watched_words.word_count' count=model.words.length}} + {{/if}} +
diff --git a/app/assets/javascripts/admin/templates/watched-words.hbs b/app/assets/javascripts/admin/templates/watched-words.hbs new file mode 100644 index 00000000000..501e4af97b6 --- /dev/null +++ b/app/assets/javascripts/admin/templates/watched-words.hbs @@ -0,0 +1,31 @@ +
+
+ +
+
+ {{text-field value=filter placeholderKey="admin.watched_words.search" class="no-blur"}} + {{d-button action="clearFilter" label="admin.watched_words.clear_filter"}} +
+
+ +
+ +
+ +
+ {{outlet}} +
+ +
diff --git a/app/assets/javascripts/discourse/lib/text.js.es6 b/app/assets/javascripts/discourse/lib/text.js.es6 index 970bbbae5ae..874f519010b 100644 --- a/app/assets/javascripts/discourse/lib/text.js.es6 +++ b/app/assets/javascripts/discourse/lib/text.js.es6 @@ -5,11 +5,13 @@ import { sanitize as textSanitize } from 'pretty-text/sanitizer'; import loadScript from 'discourse/lib/load-script'; function getOpts(opts) { - const siteSettings = Discourse.__container__.lookup('site-settings:main'); + const siteSettings = Discourse.__container__.lookup('site-settings:main'), + site = Discourse.__container__.lookup('site:main'); opts = _.merge({ getURL: Discourse.getURLWithCDN, currentUser: Discourse.__container__.lookup('current-user:main'), + censoredWords: site.censored_words, siteSettings }, opts); diff --git a/app/assets/javascripts/discourse/models/topic.js.es6 b/app/assets/javascripts/discourse/models/topic.js.es6 index 5949766a915..f8e75e517f9 100644 --- a/app/assets/javascripts/discourse/models/topic.js.es6 +++ b/app/assets/javascripts/discourse/models/topic.js.es6 @@ -50,11 +50,7 @@ const Topic = RestModel.extend({ @computed('fancy_title') fancyTitle(title) { - // TODO: `siteSettings` should always be present, but there are places in the code - // that call Discourse.Topic.create instead of using the store. - // When the store is used, remove this. - const siteSettings = this.siteSettings || Discourse.SiteSettings; - return censor(emojiUnescape(title || ""), siteSettings.censored_words); + return censor(emojiUnescape(title || ""), Discourse.Site.currentProp('censored_words')); }, // returns createdAt if there's no bumped date diff --git a/app/assets/javascripts/pretty-text/engines/discourse-markdown/censored.js.es6 b/app/assets/javascripts/pretty-text/engines/discourse-markdown/censored.js.es6 index cd0d7bb7926..91a0d297ba0 100644 --- a/app/assets/javascripts/pretty-text/engines/discourse-markdown/censored.js.es6 +++ b/app/assets/javascripts/pretty-text/engines/discourse-markdown/censored.js.es6 @@ -24,7 +24,6 @@ function censorTree(state, censor) { export function setup(helper) { helper.registerOptions((opts, siteSettings) => { - opts.censoredWords = siteSettings.censored_words; opts.censoredPattern = siteSettings.censored_pattern; }); diff --git a/app/assets/javascripts/pretty-text/pretty-text.js.es6 b/app/assets/javascripts/pretty-text/pretty-text.js.es6 index 21708cbe211..68fd360dc66 100644 --- a/app/assets/javascripts/pretty-text/pretty-text.js.es6 +++ b/app/assets/javascripts/pretty-text/pretty-text.js.es6 @@ -22,7 +22,8 @@ export function buildOptions(state) { emojiUnicodeReplacer, lookupInlineOnebox, previewing, - linkify + linkify, + censoredWords } = state; let features = { @@ -57,6 +58,7 @@ export function buildOptions(state) { mentionLookup: state.mentionLookup, emojiUnicodeReplacer, lookupInlineOnebox, + censoredWords, allowedHrefSchemes: siteSettings.allowed_href_schemes ? siteSettings.allowed_href_schemes.split('|') : null, markdownIt: true, previewing diff --git a/app/assets/stylesheets/common/admin/admin_base.scss b/app/assets/stylesheets/common/admin/admin_base.scss index e462a8328a8..58ee43ef773 100644 --- a/app/assets/stylesheets/common/admin/admin_base.scss +++ b/app/assets/stylesheets/common/admin/admin_base.scss @@ -1847,6 +1847,47 @@ table#user-badges { } } +.watched-word-box { + display: inline-block; + width: 250px; + margin-bottom: 1em; + float: left; +} + +.watched-words-list { + margin-top: 40px; +} +.watched-word { + display: inline-block; + cursor: pointer; + i.fa { + margin-right: 0.25em; + color: dark-light-diff($primary, $secondary, 50%, -50%); + } + &:hover i.fa { + color: $primary; + } +} +.watched-word-form { + display: inline-block; + .success-message { + margin-left: 1em; + } +} +.watched-words-uploader { + float: right; + text-align: right; + .instructions { + font-size: 12px; + } +} +.watched-words-detail { + .about { + margin-top: 24px; + margin-bottom: 40px; + } +} + // Mobile specific styles // Mobile view text-inputs need some padding .mobile-view .admin-contents { diff --git a/app/controllers/admin/watched_words_controller.rb b/app/controllers/admin/watched_words_controller.rb new file mode 100644 index 00000000000..a2f777d8f55 --- /dev/null +++ b/app/controllers/admin/watched_words_controller.rb @@ -0,0 +1,47 @@ +class Admin::WatchedWordsController < Admin::AdminController + + def index + render_json_dump WatchedWordListSerializer.new(WatchedWord.by_action, scope: guardian, root: false) + end + + def create + watched_word = WatchedWord.create_or_update_word(watched_words_params) + if watched_word.valid? + render json: watched_word, root: false + else + render_json_error(watched_word) + end + end + + def destroy + watched_word = WatchedWord.find(params[:id]) + watched_word.destroy + render json: success_json + end + + def upload + file = params[:file] || params[:files].first + action_key = params[:action_key].to_sym + + Scheduler::Defer.later("Upload watched words") do + begin + File.open(file.tempfile, encoding: "ISO-8859-1").each_line do |line| + WatchedWord.create_or_update_word(word: line, action_key: action_key) unless line.empty? + end + data = {url: '/ok'} + rescue => e + data = failed_json.merge(errors: [e.message]) + end + MessageBus.publish("/uploads/csv", data.as_json, client_ids: [params[:client_id]]) + end + + render json: success_json + end + + private + + def watched_words_params + params.permit(:id, :word, :action_key) + end + +end diff --git a/app/jobs/onceoff/migrate_censored_words.rb b/app/jobs/onceoff/migrate_censored_words.rb new file mode 100644 index 00000000000..03ce4b358cb --- /dev/null +++ b/app/jobs/onceoff/migrate_censored_words.rb @@ -0,0 +1,12 @@ +module Jobs + class MigrateCensoredWords < Jobs::Onceoff + def execute_onceoff(args) + row = WatchedWord.exec_sql("SELECT value FROM site_settings WHERE name = 'censored_words'") + if row.count > 0 + row.first["value"].split('|').each do |word| + WatchedWord.create(word: word, action: WatchedWord.actions[:censor]) + end + end + end + end +end diff --git a/app/jobs/regular/process_post.rb b/app/jobs/regular/process_post.rb index ddfb0ff3d58..1230dbc7526 100644 --- a/app/jobs/regular/process_post.rb +++ b/app/jobs/regular/process_post.rb @@ -37,6 +37,14 @@ module Jobs post.publish_change_to_clients! :revised end end + + if !post.user.staff? && !post.user.staged + s = post.cooked + s << " #{post.topic.title}" if post.post_number == 1 + if WordWatcher.new(s).should_flag? + PostAction.act(Discourse.system_user, post, PostActionType.types[:inappropriate]) rescue PostAction::AlreadyActed + end + end end # onebox may have added some links, so extract them now diff --git a/app/models/watched_word.rb b/app/models/watched_word.rb new file mode 100644 index 00000000000..d9cbd4edc19 --- /dev/null +++ b/app/models/watched_word.rb @@ -0,0 +1,54 @@ +require_dependency 'enum' + +class WatchedWord < ActiveRecord::Base + + def self.actions + @actions ||= Enum.new( + block: 1, + censor: 2, + require_approval: 3, + flag: 4 + ) + end + + MAX_WORDS_PER_ACTION = 1000 + + before_validation do + self.word = self.class.normalize_word(self.word) + end + + validates :word, presence: true, uniqueness: true, length: { maximum: 50 } + validates :action, presence: true + validates_each :word do |record, attr, val| + if WatchedWord.where(action: record.action).count >= MAX_WORDS_PER_ACTION + record.errors.add(:word, :too_many) + end + end + + after_save :clear_cache + after_destroy :clear_cache + + scope :by_action, -> { order("action ASC, word ASC") } + + + def self.normalize_word(w) + w.strip.downcase.squeeze('*') + end + + def self.create_or_update_word(params) + w = find_or_initialize_by(word: normalize_word(params[:word])) + w.action_key = params[:action_key] if params[:action_key] + w.action = params[:action] if params[:action] + w.save + w + end + + def action_key=(arg) + self.action = self.class.actions[arg.to_sym] + end + + def clear_cache + WordWatcher.clear_cache! + end + +end diff --git a/app/serializers/site_serializer.rb b/app/serializers/site_serializer.rb index 5ea12e8cbe7..1b474185336 100644 --- a/app/serializers/site_serializer.rb +++ b/app/serializers/site_serializer.rb @@ -25,7 +25,8 @@ class SiteSerializer < ApplicationSerializer :top_tags, :wizard_required, :topic_featured_link_allowed_category_ids, - :user_themes + :user_themes, + :censored_words has_many :categories, serializer: BasicCategorySerializer, embed: :objects has_many :trust_levels, embed: :objects @@ -142,4 +143,8 @@ class SiteSerializer < ApplicationSerializer def topic_featured_link_allowed_category_ids scope.topic_featured_link_allowed_category_ids end + + def censored_words + WordWatcher.words_for_action(:censor).join('|') + end end diff --git a/app/serializers/watched_word_list_serializer.rb b/app/serializers/watched_word_list_serializer.rb new file mode 100644 index 00000000000..e38d56365ca --- /dev/null +++ b/app/serializers/watched_word_list_serializer.rb @@ -0,0 +1,13 @@ +class WatchedWordListSerializer < ApplicationSerializer + attributes :actions, :words + + def actions + WatchedWord.actions.keys + end + + def words + object.map do |word| + WatchedWordSerializer.new(word, root: false) + end + end +end diff --git a/app/serializers/watched_word_serializer.rb b/app/serializers/watched_word_serializer.rb new file mode 100644 index 00000000000..ffa46da5125 --- /dev/null +++ b/app/serializers/watched_word_serializer.rb @@ -0,0 +1,7 @@ +class WatchedWordSerializer < ApplicationSerializer + attributes :id, :word, :action + + def action + WatchedWord.actions[object.action] + end +end diff --git a/app/services/word_watcher.rb b/app/services/word_watcher.rb new file mode 100644 index 00000000000..625e51e999d --- /dev/null +++ b/app/services/word_watcher.rb @@ -0,0 +1,51 @@ +class WordWatcher + + def initialize(raw) + @raw = raw + end + + def self.words_for_action(action) + WatchedWord.where(action: WatchedWord.actions[action.to_sym]).limit(1000).pluck(:word) + end + + def self.words_for_action_exists?(action) + WatchedWord.where(action: WatchedWord.actions[action.to_sym]).exists? + end + + def self.word_matcher_regexp(action) + s = Discourse.cache.fetch(word_matcher_regexp_key(action), expires_in: 1.day) do + words = words_for_action(action) + words.empty? ? nil : '\b(' + words.map { |w| Regexp.escape(w).gsub("\\*", '\S*') }.join('|'.freeze) + ')\b' + end + + s.present? ? Regexp.new(s, Regexp::IGNORECASE) : nil + end + + def self.word_matcher_regexp_key(action) + "watched-words-regexp:#{action}" + end + + def self.clear_cache! + WatchedWord.actions.sum do |a,i| + Discourse.cache.delete word_matcher_regexp_key(a) + end + end + + def requires_approval? + word_matches_for_action?(:require_approval) + end + + def should_flag? + word_matches_for_action?(:flag) + end + + def should_block? + word_matches_for_action?(:block) + end + + def word_matches_for_action?(action) + r = self.class.word_matcher_regexp(action) + r ? r.match(@raw) : false + end + +end diff --git a/config/locales/client.en.yml b/config/locales/client.en.yml index ec2d5398d93..e13e91344d8 100644 --- a/config/locales/client.en.yml +++ b/config/locales/client.en.yml @@ -3148,6 +3148,31 @@ en: logster: title: "Error Logs" + watched_words: + title: "Watched Words" + search: "search" + clear_filter: "Clear" + show_words: "show words" + word_count: + one: "1 word" + other: "%{count} words" + actions: + block: 'Block' + censor: 'Censor' + require_approval: 'Require Approval' + flag: 'Flag' + action_descriptions: + block: 'Prevent posts containing these words from being posted. The user will see an error message when they try to submit their post.' + censor: 'Allow posts containing these words, but replace them with characters that hide the censored words.' + require_approval: 'Posts containing these words will require approval by staff before they can be seen.' + flag: 'Allow posts containing these words, but flag them as inappropriate so moderators can review them.' + form: + label: 'New Word:' + add: 'Add' + success: 'Success' + upload: "Upload" + upload_successful: "Upload successful. Words have been added." + impersonate: title: "Impersonate" help: "Use this tool to impersonate a user account for debugging purposes. You will have to log out once finished." diff --git a/config/locales/server.en.yml b/config/locales/server.en.yml index e05db140c1f..516b639b20f 100644 --- a/config/locales/server.en.yml +++ b/config/locales/server.en.yml @@ -208,6 +208,7 @@ en: too_many_links: one: "Sorry, new users can only put one link in a post." other: "Sorry, new users can only put %{count} links in a post." + contains_blocked_words: "Your post contains words that aren't allowed." spamming_host: "Sorry you cannot post a link to that host." user_is_suspended: "Suspended users are not allowed to post." @@ -414,6 +415,10 @@ en: attributes: value: missing_interpolation_keys: 'The following interpolation key(s) are missing: "%{keys}"' + watched_word: + attributes: + word: + too_many: "Too many words for that action" user_profile: no_info_me: "
the About Me field of your profile is currently blank, would you like to fill it out?
" diff --git a/config/routes.rb b/config/routes.rb index a66dadb4895..b95fff1a39d 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -270,6 +270,12 @@ Discourse::Application.routes.draw do get "dump_heap"=> "diagnostics#dump_heap", constraints: AdminConstraint.new get "dump_statement_cache"=> "diagnostics#dump_statement_cache", constraints: AdminConstraint.new + resources :watched_words, only: [:index, :create, :update, :destroy], constraints: AdminConstraint.new do + collection do + get "action/:id" => "watched_words#index" + end + end + post "watched_words/upload" => "watched_words#upload" end # admin namespace get "email_preferences" => "email#preferences_redirect", :as => "email_preferences_redirect" diff --git a/config/site_settings.yml b/config/site_settings.yml index 53ed6d41ea1..4e0dbdc5caa 100644 --- a/config/site_settings.yml +++ b/config/site_settings.yml @@ -555,11 +555,6 @@ posting: type: list client: true delete_old_hidden_posts: true - censored_words: - client: true - default: '' - refresh: true - type: list censored_pattern: client: true default: '' diff --git a/db/migrate/20170628152322_create_watched_words.rb b/db/migrate/20170628152322_create_watched_words.rb new file mode 100644 index 00000000000..212a3d2b9f6 --- /dev/null +++ b/db/migrate/20170628152322_create_watched_words.rb @@ -0,0 +1,11 @@ +class CreateWatchedWords < ActiveRecord::Migration + def change + create_table :watched_words do |t| + t.string :word, null: false + t.integer :action, null: false + t.timestamps + end + + add_index :watched_words, [:action, :word], unique: true + end +end diff --git a/lib/new_post_manager.rb b/lib/new_post_manager.rb index 537c7faff04..7d99a8b4ed6 100644 --- a/lib/new_post_manager.rb +++ b/lib/new_post_manager.rb @@ -1,6 +1,7 @@ require_dependency 'post_creator' require_dependency 'new_post_result' require_dependency 'post_enqueuer' +require_dependency 'word_watcher' # Determines what actions should be taken with new posts. # @@ -66,21 +67,25 @@ class NewPostManager end - def self.user_needs_approval?(manager) + def self.exempt_user?(user) + user.staff? || user.staged + end + + def self.post_needs_approval?(manager) user = manager.user - return false if user.staff? || user.staged + return false if exempt_user?(user) (user.trust_level <= TrustLevel.levels[:basic] && user.post_count < SiteSetting.approve_post_count) || (user.trust_level < SiteSetting.approve_unless_trust_level.to_i) || (manager.args[:title].present? && user.trust_level < SiteSetting.approve_new_topics_unless_trust_level.to_i) || is_fast_typer?(manager) || - matches_auto_block_regex?(manager) + matches_auto_block_regex?(manager) || + WordWatcher.new("#{manager.args[:title]} #{manager.args[:raw]}").requires_approval? end def self.default_handler(manager) - if user_needs_approval?(manager) - + if post_needs_approval?(manager) validator = Validators::PostValidator.new post = Post.new(raw: manager.args[:raw]) post.user = manager.user @@ -118,6 +123,7 @@ class NewPostManager SiteSetting.approve_post_count > 0 || SiteSetting.approve_unless_trust_level.to_i > 0 || SiteSetting.approve_new_topics_unless_trust_level.to_i > 0 || + WordWatcher.words_for_action_exists?(:require_approval) || handlers.size > 1 end @@ -127,8 +133,15 @@ class NewPostManager end def perform + if !self.class.exempt_user?(@user) && WordWatcher.new("#{@args[:title]} #{@args[:raw]}").should_block? + result = NewPostResult.new(:created_post, false) + result.errors[:base] << I18n.t('contains_blocked_words') + return result + end + # We never queue private messages return perform_create_post if @args[:archetype] == Archetype.private_message + if args[:topic_id] && Topic.where(id: args[:topic_id], archetype: Archetype.private_message).exists? return perform_create_post end diff --git a/lib/post_revisor.rb b/lib/post_revisor.rb index 831faa47b43..dea5948df57 100644 --- a/lib/post_revisor.rb +++ b/lib/post_revisor.rb @@ -273,6 +273,12 @@ class PostRevisor @post.word_count = @fields[:raw].scan(/[[:word:]]+/).size if @fields.has_key?(:raw) @post.self_edits += 1 if self_edit? + if !@post.acting_user.staff? && !@post.acting_user.staged && WordWatcher.new(@post.raw).should_block? + @post.errors[:base] << I18n.t('contains_blocked_words') + @post_successfully_saved = false + return + end + remove_flags_and_unhide_post @post.extract_quoted_post_numbers diff --git a/lib/pretty_text.rb b/lib/pretty_text.rb index 69e60f1bed7..8c74bd6fe1f 100644 --- a/lib/pretty_text.rb +++ b/lib/pretty_text.rb @@ -166,6 +166,7 @@ module PrettyText __optInput.emojiUnicodeReplacer = __emojiUnicodeReplacer; __optInput.lookupInlineOnebox = __lookupInlineOnebox; #{opts[:linkify] == false ? "__optInput.linkify = false;": ""} + __optInput.censoredWords = #{WordWatcher.words_for_action(:censor).join('|').to_json}; JS if opts[:topicId] diff --git a/lib/validators/censored_words_validator.rb b/lib/validators/censored_words_validator.rb index 66e03f5bdd4..2fa5f0e0c0f 100644 --- a/lib/validators/censored_words_validator.rb +++ b/lib/validators/censored_words_validator.rb @@ -1,6 +1,6 @@ class CensoredWordsValidator < ActiveModel::EachValidator def validate_each(record, attribute, value) - if SiteSetting.censored_words.present? && (censored_words = censor_words(value, censored_words_regexp)).present? + if WordWatcher.words_for_action(:censor).present? && (censored_words = censor_words(value, censored_words_regexp)).present? record.errors.add( attribute, :contains_censored_words, censored_words: join_censored_words(censored_words) @@ -32,9 +32,6 @@ class CensoredWordsValidator < ActiveModel::EachValidator end def censored_words_regexp - Regexp.new( - '\b(' + SiteSetting.censored_words.split('|'.freeze).map! { |w| Regexp.escape(w) }.join('|'.freeze) + ')\b', - true - ) + WordWatcher.word_matcher_regexp :censor end end diff --git a/plugins/discourse-details/test/javascripts/lib/details-cooked-test.js.es6 b/plugins/discourse-details/test/javascripts/lib/details-cooked-test.js.es6 index 4087ea22b21..ab49a5a92cb 100644 --- a/plugins/discourse-details/test/javascripts/lib/details-cooked-test.js.es6 +++ b/plugins/discourse-details/test/javascripts/lib/details-cooked-test.js.es6 @@ -8,8 +8,8 @@ const defaultOpts = buildOptions({ emoji_set: 'emoji_one', highlighted_languages: 'json|ruby|javascript', default_code_lang: 'auto', - censored_words: '' }, + censoredWords: 'shucks|whiz|whizzer', getURL: url => url }); diff --git a/spec/components/new_post_manager_spec.rb b/spec/components/new_post_manager_spec.rb index 3b38304d177..79d11707fd9 100644 --- a/spec/components/new_post_manager_spec.rb +++ b/spec/components/new_post_manager_spec.rb @@ -253,22 +253,22 @@ describe NewPostManager do - it "handles user_needs_approval? correctly" do + it "handles post_needs_approval? correctly" do u = user default = NewPostManager.new(u,{}) - expect(NewPostManager.user_needs_approval?(default)).to eq(false) + expect(NewPostManager.post_needs_approval?(default)).to eq(false) with_check = NewPostManager.new(u, first_post_checks: true) - expect(NewPostManager.user_needs_approval?(with_check)).to eq(true) + expect(NewPostManager.post_needs_approval?(with_check)).to eq(true) u.user_stat.post_count = 1 with_check_and_post = NewPostManager.new(u, first_post_checks: true) - expect(NewPostManager.user_needs_approval?(with_check_and_post)).to eq(false) + expect(NewPostManager.post_needs_approval?(with_check_and_post)).to eq(false) u.user_stat.post_count = 0 u.trust_level = 1 with_check_tl1 = NewPostManager.new(u, first_post_checks: true) - expect(NewPostManager.user_needs_approval?(with_check_tl1)).to eq(false) + expect(NewPostManager.post_needs_approval?(with_check_tl1)).to eq(false) end end diff --git a/spec/components/pretty_text_spec.rb b/spec/components/pretty_text_spec.rb index 72beb51030d..6e40972adeb 100644 --- a/spec/components/pretty_text_spec.rb +++ b/spec/components/pretty_text_spec.rb @@ -247,8 +247,9 @@ describe PrettyText do end it 'does censor code fences' do - SiteSetting.censored_words = 'apple|banana' + ['apple', 'banana'].each { |w| Fabricate(:watched_word, word: w, action: WatchedWord.actions[:censor]) } expect(PrettyText.cook("# banana")).not_to include('banana') + $redis.flushall end end @@ -787,11 +788,12 @@ HTML end it 'can censor words correctly' do - SiteSetting.censored_words = 'apple|banana' + ['apple', 'banana'].each { |w| Fabricate(:watched_word, word: w, action: WatchedWord.actions[:censor]) } expect(PrettyText.cook('yay banana yay')).not_to include('banana') expect(PrettyText.cook('yay `banana` yay')).not_to include('banana') expect(PrettyText.cook("# banana")).not_to include('banana') expect(PrettyText.cook("# banana")).to include("\u25a0\u25a0") + $redis.flushall end it 'supports typographer' do diff --git a/spec/fabricators/watched_word_fabricator.rb b/spec/fabricators/watched_word_fabricator.rb new file mode 100644 index 00000000000..f85547d8a2d --- /dev/null +++ b/spec/fabricators/watched_word_fabricator.rb @@ -0,0 +1,4 @@ +Fabricator(:watched_word) do + word { sequence(:word) { |i| "word#{i}"} } + action { WatchedWord.actions[:block] } +end diff --git a/spec/integration/watched_words_spec.rb b/spec/integration/watched_words_spec.rb new file mode 100644 index 00000000000..95787054048 --- /dev/null +++ b/spec/integration/watched_words_spec.rb @@ -0,0 +1,169 @@ +require 'rails_helper' + +describe WatchedWord do + let(:tl2_user) { Fabricate(:user, trust_level: TrustLevel[2]) } + let(:admin) { Fabricate(:admin) } + let(:moderator) { Fabricate(:moderator) } + + let(:topic) { Fabricate(:topic) } + let(:first_post) { Fabricate(:post, topic: topic) } + + let(:require_approval_word) { Fabricate(:watched_word, action: WatchedWord.actions[:require_approval]) } + let(:flag_word) { Fabricate(:watched_word, action: WatchedWord.actions[:flag]) } + let(:block_word) { Fabricate(:watched_word, action: WatchedWord.actions[:block]) } + + context "block" do + def should_block_post(manager) + expect { + result = manager.perform + expect(result).to_not be_success + expect(result.errors[:base]&.first).to eq(I18n.t('contains_blocked_words')) + }.to_not change { Post.count } + end + + it "should prevent the post from being created" do + manager = NewPostManager.new(tl2_user, raw: "Want some #{block_word.word} for cheap?", topic_id: topic.id) + should_block_post(manager) + end + + it "look at title too" do + manager = NewPostManager.new(tl2_user, title: "We sell #{block_word.word} online", raw: "Want some poutine for cheap?", topic_id: topic.id) + should_block_post(manager) + end + + it "should not block the post from admin" do + manager = NewPostManager.new(admin, raw: "Want some #{block_word.word} for cheap?", topic_id: topic.id) + result = manager.perform + expect(result).to be_success + expect(result.action).to eq(:create_post) + end + + it "should not block the post from moderator" do + manager = NewPostManager.new(moderator, raw: "Want some #{block_word.word} for cheap?", topic_id: topic.id) + result = manager.perform + expect(result).to be_success + expect(result.action).to eq(:create_post) + end + + it "should block in a private message too" do + manager = NewPostManager.new( + tl2_user, + raw: "Want some #{block_word.word} for cheap?", + title: 'this is a new title', + archetype: Archetype.private_message, + target_usernames: Fabricate(:user, trust_level: TrustLevel[2]).username + ) + should_block_post(manager) + end + + it "blocks on revisions" do + post = Fabricate(:post, topic: Fabricate(:topic, user: tl2_user), user: tl2_user) + expect { + PostRevisor.new(post).revise!(post.user, { raw: "Want some #{block_word.word} for cheap?" }, revised_at: post.updated_at + 10.seconds) + expect(post.errors).to be_present + post.reload + }.to_not change { post.raw } + end + end + + context "require_approval" do + it "should queue the post for approval" do + manager = NewPostManager.new(tl2_user, raw: "My dog's name is #{require_approval_word.word}.", topic_id: topic.id) + result = manager.perform + expect(result.action).to eq(:enqueued) + end + + it "looks at title too" do + manager = NewPostManager.new(tl2_user, title: "You won't believe these #{require_approval_word.word} dog names!", raw: "My dog's name is Porkins.", topic_id: topic.id) + result = manager.perform + expect(result.action).to eq(:enqueued) + end + + it "should not queue posts from admin" do + manager = NewPostManager.new(admin, raw: "My dog's name is #{require_approval_word.word}.", topic_id: topic.id) + result = manager.perform + expect(result).to be_success + expect(result.action).to eq(:create_post) + end + + it "should not queue posts from moderator" do + manager = NewPostManager.new(moderator, raw: "My dog's name is #{require_approval_word.word}.", topic_id: topic.id) + result = manager.perform + expect(result).to be_success + expect(result.action).to eq(:create_post) + end + + it "doesn't need approval in a private message" do + manager = NewPostManager.new( + tl2_user, + raw: "Want some #{require_approval_word.word} for cheap?", + title: 'this is a new title', + archetype: Archetype.private_message, + target_usernames: Fabricate(:user, trust_level: TrustLevel[2]).username + ) + result = manager.perform + expect(result).to be_success + expect(result.action).to eq(:create_post) + end + end + + context "flag" do + def should_flag_post(author, raw, topic) + post = Fabricate(:post, raw: raw, topic: topic, user: author) + expect { + Jobs::ProcessPost.new.execute(post_id: post.id) + }.to change { PostAction.count }.by(1) + expect(PostAction.where(post_id: post.id, post_action_type_id: PostActionType.types[:inappropriate]).exists?).to eq(true) + end + + def should_not_flag_post(author, raw, topic) + post = Fabricate(:post, raw: raw, topic: topic, user: author) + expect { + Jobs::ProcessPost.new.execute(post_id: post.id) + }.to_not change { PostAction.count } + end + + it "should flag the post as inappropriate" do + should_flag_post(tl2_user, "I thought the #{flag_word.word} was bad.", Fabricate(:topic, user: tl2_user)) + end + + it "should look at the title too" do + should_flag_post(tl2_user, "I thought the movie was not bad actually.", Fabricate(:topic, user: tl2_user, title: "Read my #{flag_word.word} review!")) + end + + it "shouldn't flag posts by admin" do + should_not_flag_post(admin, "I thought the #{flag_word.word} was bad.", Fabricate(:topic, user: admin)) + end + + it "shouldn't flag posts by moderator" do + should_not_flag_post(moderator, "I thought the #{flag_word.word} was bad.", Fabricate(:topic, user: moderator)) + end + + it "is compatible with flag_sockpuppets" do + # e.g., handle PostAction::AlreadyActed + SiteSetting.flag_sockpuppets = true + ip_address = '182.189.119.174' + user1 = Fabricate(:user, ip_address: ip_address, created_at: 2.days.ago) + user2 = Fabricate(:user, ip_address: ip_address) + first = create_post(user: user1, created_at: 2.days.ago) + sockpuppet_post = create_post(user: user2, topic: first.topic, raw: "I thought the #{flag_word.word} was bad.") + expect(PostAction.where(post_id: sockpuppet_post.id).count).to eq(1) + end + + it "flags in private message too" do + post = Fabricate(:private_message_post, raw: "Want some #{flag_word.word} for cheap?", user: tl2_user) + expect { + Jobs::ProcessPost.new.execute(post_id: post.id) + }.to change { PostAction.count }.by(1) + expect(PostAction.where(post_id: post.id, post_action_type_id: PostActionType.types[:inappropriate]).exists?).to eq(true) + end + + it "flags on revisions" do + post = Fabricate(:post, topic: Fabricate(:topic, user: tl2_user), user: tl2_user) + expect { + PostRevisor.new(post).revise!(post.user, { raw: "Want some #{flag_word.word} for cheap?" }, revised_at: post.updated_at + 10.seconds) + }.to change { PostAction.count }.by(1) + expect(PostAction.where(post_id: post.id, post_action_type_id: PostActionType.types[:inappropriate]).exists?).to eq(true) + end + end +end diff --git a/spec/models/topic_spec.rb b/spec/models/topic_spec.rb index c93d6d3e393..b4b43759df4 100644 --- a/spec/models/topic_spec.rb +++ b/spec/models/topic_spec.rb @@ -30,9 +30,13 @@ describe Topic do end describe 'censored words' do + after do + $redis.flushall + end + describe 'when title contains censored words' do it 'should not be valid' do - SiteSetting.censored_words = 'pineapple|pen' + ['pineapple', 'pen'].each { |w| Fabricate(:watched_word, word: w, action: WatchedWord.actions[:censor]) } topic.title = 'pen PinEapple apple pen is a complete sentence' @@ -46,7 +50,7 @@ describe Topic do describe 'titles with censored words not on boundaries' do it "should be valid" do - SiteSetting.censored_words = 'apple' + Fabricate(:watched_word, word: 'apple', action: WatchedWord.actions[:censor]) topic.title = "Pineapples are great fruit! Applebee's is a great restaurant" expect(topic).to be_valid end @@ -62,10 +66,12 @@ describe Topic do describe 'escape special characters in censored words' do before do - SiteSetting.censored_words = 'co(onut|coconut|a**le' + ['co(onut', 'coconut', 'a**le'].each do |w| + Fabricate(:watched_word, word: w, action: WatchedWord.actions[:censor]) + end end - it 'should not valid' do + it 'should not be valid' do topic.title = "I have a co(onut a**le" expect(topic.valid?).to eq(false) diff --git a/spec/models/watched_word_spec.rb b/spec/models/watched_word_spec.rb new file mode 100644 index 00000000000..7bc52d47e5e --- /dev/null +++ b/spec/models/watched_word_spec.rb @@ -0,0 +1,92 @@ +require 'rails_helper' + +describe WatchedWord do + + it "can't have duplicate words" do + Fabricate(:watched_word, word: "darn", action: described_class.actions[:block]) + w = Fabricate.build(:watched_word, word: "darn", action: described_class.actions[:block]) + expect(w.save).to eq(false) + w = Fabricate.build(:watched_word, word: "darn", action: described_class.actions[:flag]) + expect(w.save).to eq(false) + expect(described_class.count).to eq(1) + end + + it "downcases words" do + expect(described_class.create(word: "ShooT").word).to eq('shoot') + end + + it "strips leading and trailing spaces" do + expect(described_class.create(word: " poutine ").word).to eq('poutine') + end + + it "squeezes multiple asterisks" do + expect(described_class.create(word: "a**les").word).to eq('a*les') + end + + describe "action_key=" do + let(:w) { WatchedWord.new(word: "troll") } + + it "sets action attr from symbol" do + described_class.actions.keys.each do |k| + w.action_key = k + expect(w.action).to eq(described_class.actions[k]) + end + end + + it "sets action attr from string" do + described_class.actions.keys.each do |k| + w.action_key = k.to_s + expect(w.action).to eq(described_class.actions[k]) + end + end + + it "sets error for invalid key" do + w.action_key = "shame" + expect(w).to_not be_valid + expect(w.errors[:action]).to be_present + end + end + + describe '#create_or_update_word' do + it "can create a new record" do + expect { + w = described_class.create_or_update_word(word: 'nickelback', action_key: :block) + expect(w.reload.action).to eq(described_class.actions[:block]) + }.to change { described_class.count }.by(1) + end + + it "can update an existing record with different action" do + existing = Fabricate(:watched_word, action: described_class.actions[:flag]) + expect { + w = described_class.create_or_update_word(word: existing.word, action_key: :block) + expect(w.reload.action).to eq(described_class.actions[:block]) + expect(w.id).to eq(existing.id) + }.to_not change { described_class.count } + end + + it "doesn't error for existing record with same action" do + existing = Fabricate(:watched_word, action: described_class.actions[:flag], created_at: 1.day.ago, updated_at: 1.day.ago) + expect { + w = described_class.create_or_update_word(word: existing.word, action_key: :flag) + expect(w.id).to eq(existing.id) + expect(w.updated_at).to eq(w.updated_at) + }.to_not change { described_class.count } + end + + it "allows action param instead of action_key" do + expect { + w = described_class.create_or_update_word(word: 'nickelback', action: described_class.actions[:block]) + expect(w.reload.action).to eq(described_class.actions[:block]) + }.to change { described_class.count }.by(1) + end + + it "normalizes input" do + existing = Fabricate(:watched_word, action: described_class.actions[:flag]) + expect { + w = described_class.create_or_update_word(word: " #{existing.word.upcase} ", action_key: :block) + expect(w.reload.action).to eq(described_class.actions[:block]) + expect(w.id).to eq(existing.id) + }.to_not change { described_class.count } + end + end +end diff --git a/spec/services/word_watcher_spec.rb b/spec/services/word_watcher_spec.rb new file mode 100644 index 00000000000..98c2776d8ee --- /dev/null +++ b/spec/services/word_watcher_spec.rb @@ -0,0 +1,53 @@ +require 'rails_helper' + +describe WordWatcher do + + let(:raw) { "Do you like liquorice?\n\nI really like them. One could even say that I am *addicted* to liquorice. Anf if\nyou can mix it up with some anise, then I'm in heaven ;)" } + + after do + $redis.flushall + end + + describe "word_matches_for_action?" do + it "is falsey when there are no watched words" do + expect(WordWatcher.new(raw).word_matches_for_action?(:require_approval)).to be_falsey + end + + context "with watched words" do + let!(:anise) { Fabricate(:watched_word, word: "anise", action: WatchedWord.actions[:require_approval]) } + + it "is falsey without a match" do + expect(WordWatcher.new("No liquorice for me, thanks...").word_matches_for_action?(:require_approval)).to be_falsey + end + + it "is returns matched words if there's a match" do + m = WordWatcher.new(raw).word_matches_for_action?(:require_approval) + expect(m).to be_truthy + expect(m[1]).to eq(anise.word) + end + + it "finds at start of string" do + m = WordWatcher.new("#{anise.word} is garbage").word_matches_for_action?(:require_approval) + expect(m[1]).to eq(anise.word) + end + + it "finds at end of string" do + m = WordWatcher.new("who likes #{anise.word}").word_matches_for_action?(:require_approval) + expect(m[1]).to eq(anise.word) + end + + it "finds non-letters in place of letters" do + Fabricate(:watched_word, word: "co(onut", action: WatchedWord.actions[:require_approval]) + m = WordWatcher.new("This co(onut is delicious.").word_matches_for_action?(:require_approval) + expect(m[1]).to eq("co(onut") + end + + it "handles * for wildcards" do + Fabricate(:watched_word, word: "a**le*", action: WatchedWord.actions[:require_approval]) + m = WordWatcher.new("I acknowledge you.").word_matches_for_action?(:require_approval) + expect(m[1]).to eq("acknowledge") + end + end + end + +end diff --git a/test/javascripts/acceptance/admin-watched-words-test.js.es6 b/test/javascripts/acceptance/admin-watched-words-test.js.es6 new file mode 100644 index 00000000000..ac24215f56f --- /dev/null +++ b/test/javascripts/acceptance/admin-watched-words-test.js.es6 @@ -0,0 +1,68 @@ +import { acceptance } from "helpers/qunit-helpers"; +acceptance("Admin - Watched Words", { loggedIn: true }); + +QUnit.test("list words in groups", assert => { + visit("/admin/watched_words/action/block"); + andThen(() => { + assert.ok(exists('.watched-words-list')); + assert.ok(!exists('.watched-words-list .watched-word'), "Don't show bad words by default."); + }); + + fillIn('.admin-controls .controls input[type=text]', 'li'); + andThen(() => { + assert.equal(find('.watched-words-list .watched-word').length, 1, "When filtering, show words even if checkbox is unchecked."); + }); + + fillIn('.admin-controls .controls input[type=text]', ''); + andThen(() => { + assert.ok(!exists('.watched-words-list .watched-word'), "Clearing the filter hides words again."); + }); + + click('.show-words-checkbox'); + andThen(() => { + assert.ok(exists('.watched-words-list .watched-word'), "Always show the words when checkbox is checked."); + }); + + click('.nav-stacked .censor'); + andThen(() => { + assert.ok(exists('.watched-words-list')); + assert.ok(!exists('.watched-words-list .watched-word'), "Empty word list."); + }); +}); + +QUnit.test("add words", assert => { + visit("/admin/watched_words/action/block"); + andThen(() => { + click('.show-words-checkbox'); + fillIn('.watched-word-form input', 'poutine'); + }); + click('.watched-word-form button'); + andThen(() => { + let found = []; + _.each(find('.watched-words-list .watched-word'), i => { + if ($(i).text().trim() === 'poutine') { + found.push(true); + } + }); + assert.equal(found.length, 1); + }); +}); + +QUnit.test("remove words", assert => { + visit("/admin/watched_words/action/block"); + click('.show-words-checkbox'); + + let word = null; + andThen(() => { + _.each(find('.watched-words-list .watched-word'), i => { + if ($(i).text().trim() === 'anise') { + word = i; + } + }); + click('#' + $(word).attr('id')); + }); + andThen(() => { + assert.equal(find('.watched-words-list .watched-word').length, 1); + }); +}); + diff --git a/test/javascripts/fixtures/watched-words-fixtures.js.es6 b/test/javascripts/fixtures/watched-words-fixtures.js.es6 new file mode 100644 index 00000000000..e73505f1851 --- /dev/null +++ b/test/javascripts/fixtures/watched-words-fixtures.js.es6 @@ -0,0 +1,12 @@ +export default { + "/admin/watched_words.json": { + "actions": ["block", "censor", "require_approval", "flag"], + "words": [ + {id: 1, word: "liquorice", action: "block"}, + {id: 2, word: "anise", action: "block"}, + {id: 3, word: "pyramid", action: "flag"}, + {id: 4, word: "scheme", action: "flag"}, + {id: 5, word: "coupon", action: "require_approval"} + ] + } +}; diff --git a/test/javascripts/helpers/create-pretender.js.es6 b/test/javascripts/helpers/create-pretender.js.es6 index bfa32ba4af4..2b5141ce9ed 100644 --- a/test/javascripts/helpers/create-pretender.js.es6 +++ b/test/javascripts/helpers/create-pretender.js.es6 @@ -334,6 +334,17 @@ export default function() { this.post('/admin/badges', success); this.delete('/admin/badges/:id', success); + this.get('/admin/watched_words', () => { + return response(200, fixturesByUrl['/admin/watched_words.json']); + }); + this.delete('/admin/watched_words/:id.json', success); + + this.post('/admin/watched_words.json', request => { + const result = parsePostData(request.requestBody); + result.id = new Date().getTime(); + return response(200, result); + }); + this.get('/onebox', request => { if (request.queryParams.url === 'http://www.example.com/has-title.html' || request.queryParams.url === 'http://www.example.com/has-title-and-a-url-that-is-more-than-80-characters-because-thats-good-for-seo-i-guess.html') { diff --git a/test/javascripts/lib/pretty-text-test.js.es6 b/test/javascripts/lib/pretty-text-test.js.es6 index 8594976b73a..d53fcc346e1 100644 --- a/test/javascripts/lib/pretty-text-test.js.es6 +++ b/test/javascripts/lib/pretty-text-test.js.es6 @@ -11,9 +11,9 @@ const rawOpts = { emoji_set: 'emoji_one', highlighted_languages: 'json|ruby|javascript', default_code_lang: 'auto', - censored_words: 'shucks|whiz|whizzer|a**le', censored_pattern: '\\d{3}-\\d{4}|tech\\w*' }, + censoredWords: 'shucks|whiz|whizzer|a**le', getURL: url => url };