<commit_before>/*
 * Copyright (C) 2012-2017  Online-Go.com
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

import * as preferences from "preferences";
import {current_language} from "translate";

let filters: any = {};
export let profanity_regex: any = {};

export function profanity_filter(str) {
    for (let lang in filters) {
        str = str.replace(profanity_regex[lang], "$*!%", str);
    }
    return str;
}
profanity_regex["en"] = new RegExp(atob("<KEY>"), "ig");
profanity_regex["de"] = new RegExp(atob("<KEY>), "ig");
profanity_regex["es"] = new RegExp(atob("<KEY>), "ig");
profanity_regex["fr"] = new RegExp(atob("<KEY>), "ig");
profanity_regex["it"] = new RegExp(atob("<KEY>), "ig");
profanity_regex["ja"] = new RegExp(atob("<KEY>"), "ig");
profanity_regex["nl"] = new RegExp(atob("<KEY>), "ig");
profanity_regex["pl"] = new RegExp(atob("<KEY>), "ig");
profanity_regex["pt"] = new RegExp(atob("<KEY>), "ig");
profanity_regex["ro"] = new RegExp(atob("<KEY>), "ig");
profanity_regex["ru"] = new RegExp(atob("<KEY>"), "ig");
profanity_regex["zh_CN"] = new RegExp(atob("<KEY>), "ig");


/* this needs to be declared after profanity_regex has been populated */
preferences.watch("profanity-filter", (_filters) => {
    filters = {};
    for (let lang in _filters) {
        if (_filters[lang] && lang in profanity_regex) {
            filters[lang] = true;
        }
    }
});<commit_msg>Updated profanity_filter.ts<commit_after>/*
 * Copyright (C) 2012-2017  <EMAIL>
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

import * as preferences from "preferences";
import {current_language} from "translate";

let filters: any = {};
export let profanity_regex: any = {};

export function profanity_filter(str) {
    for (let lang in filters) {
        str = str.replace(profanity_regex[lang], "$*!%", str);
    }
    return str;
}
profanity_regex["en"] = new RegExp(atob("<KEY>"), "ig");
profanity_regex["de"] = new RegExp(atob("<KEY>"), "ig");
profanity_regex["es"] = new RegExp(atob("<KEY>), "ig");
profanity_regex["fr"] = new RegExp(atob("<KEY>), "ig");
profanity_regex["it"] = new RegExp(atob("<KEY>), "ig");
profanity_regex["ja"] = new RegExp(atob("KF<KEY>"), "ig");
profanity_regex["nl"] = new RegExp(atob("<KEY>), "ig");
profanity_regex["pl"] = new RegExp(atob("<KEY>), "ig");
profanity_regex["pt"] = new RegExp(atob("<KEY>"), "igu");
profanity_regex["ro"] = new RegExp(atob("<KEY>), "ig");
profanity_regex["ru"] = new RegExp(atob("<KEY>"), "ig");
profanity_regex["zh_CN"] = new RegExp(atob("<KEY>), "ig");


/* this needs to be declared after profanity_regex has been populated */
preferences.watch("profanity-filter", (_filters) => {
    filters = {};
    for (let lang in _filters) {
        if (_filters[lang] && lang in profanity_regex) {
            filters[lang] = true;
        }
    }
});
{'fraction_non_alphanumeric': 0.16071868077775042, 'fraction_numerical': 0.004430223972434162, 'mean_word_length': 4.574759945130316, 'pattern_counts': {'":': 0, '<': 30, '<?xml version=': 0, '>': 32, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '896400', 'n_tokens_mistral': 1322, 'n_tokens_neox': 1290, 'n_words': 419}