<commit_before>// Exports a wordfilter singleton

var Datastore = require('nedb');
var db = new Datastore({ filename: './db/wordfilter.db', autoload: true });
var Q = require('q');

// Automatically compact the DB every 5 minutes
db.persistence.setAutocompactionInterval(300000);

function Wordfilter() {}

Wordfilter.prototype.blacklisted = function (string) {
    var deferred = Q.defer();
    this.getList()
        .then(function (blacklist) {
            if (!blacklist || !string) {
                deferred.resolve(false);
            }

            for (var i = 0; i < blacklist.length; i++) {
                if (string.toLowerCase().indexOf(blacklist[i]) >= 0) {
                    return true;
                }
            }

            deferred.resolve(false);
        }, function (reason) {
            deferred.reject(new Error(reason));
        });
    return deferred.promise;
};

Wordfilter.prototype.addWords = function (words) {
    // Upserting the words
    var deferred = Q.defer();
    db.update({ _id: 'blacklist' }, { $addToSet: { words: { $each: words } } }, { upsert: true }, function (err, numAdded) {
        if (err) {
            deferred.reject(new Error(err));
        } else {
            deferred.resolve(numAdded);
        }
    });
    return deferred.promise;
};

Wordfilter.prototype.removeWords = function (words) {
    var deferred = Q.defer();
    db.update({ _id: 'blacklist' }, { $pull: { words: { $in: words } } }, { upsert: true }, function (err, numRemoved) {
        if (err) {
            deferred.reject(new Error(err));
        } else {
            deferred.resolve(numRemoved);
        }
    });
    return deferred.promise;
};

Wordfilter.prototype.clearList = function () {
    // Delete all words in the datastore
    var deferred = Q.defer();
    db.remove({}, { multi: true }, function (err, numRemoved) {
        if (err) {
            deferred.reject(new Error(err));
        } else {
            deferred.resolve(numRemoved);
        }
    });
    return deferred.promise;
};

Wordfilter.prototype.getList = function() {
    // Find all words in the datastore
    var deferred = Q.defer();
    db.findOne({ _id: 'blacklist' }, function (err, blacklist) {
        if (err) {
            deferred.reject(new Error(err));
        } else {
            deferred.resolve(blacklist
                ? blacklist.words.sort()
                : []
            );
        }
    });
    return deferred.promise;
};

module.exports = exports = new Wordfilter();<commit_msg>Force all blacklisted words to lowercase<commit_after>// Exports a wordfilter singleton

var Datastore = require('nedb');
var db = new Datastore({ filename: './db/wordfilter.db', autoload: true });
var Q = require('q');

// Automatically compact the DB every 5 minutes
db.persistence.setAutocompactionInterval(300000);

function Wordfilter() {}

Wordfilter.prototype.blacklisted = function (string) {
    var deferred = Q.defer();
    this.getList()
        .then(function (blacklist) {
            if (!blacklist || !string) {
                deferred.resolve(false);
            }

            for (var i = 0; i < blacklist.length; i++) {
                if (string.toLowerCase().indexOf(blacklist[i]) >= 0) {
                    return true;
                }
            }

            deferred.resolve(false);
        }, function (reason) {
            deferred.reject(new Error(reason));
        });
    return deferred.promise;
};

Wordfilter.prototype.addWords = function (words) {
    // Force all the words to lowercase
    for (var i = 0; i < words.length; i++) {
        words[i] = words[i].toLowerCase();
    }

    // Upserting the words
    var deferred = Q.defer();
    db.update({ _id: 'blacklist' }, { $addToSet: { words: { $each: words } } }, { upsert: true }, function (err, numAdded) {
        if (err) {
            deferred.reject(new Error(err));
        } else {
            deferred.resolve(numAdded);
        }
    });
    return deferred.promise;
};

Wordfilter.prototype.removeWords = function (words) {
    var deferred = Q.defer();
    db.update({ _id: 'blacklist' }, { $pull: { words: { $in: words } } }, { upsert: true }, function (err, numRemoved) {
        if (err) {
            deferred.reject(new Error(err));
        } else {
            deferred.resolve(numRemoved);
        }
    });
    return deferred.promise;
};

Wordfilter.prototype.clearList = function () {
    // Delete all words in the datastore
    var deferred = Q.defer();
    db.remove({}, { multi: true }, function (err, numRemoved) {
        if (err) {
            deferred.reject(new Error(err));
        } else {
            deferred.resolve(numRemoved);
        }
    });
    return deferred.promise;
};

Wordfilter.prototype.getList = function() {
    // Find all words in the datastore
    var deferred = Q.defer();
    db.findOne({ _id: 'blacklist' }, function (err, blacklist) {
        if (err) {
            deferred.reject(new Error(err));
        } else {
            deferred.resolve(blacklist
                ? blacklist.words.sort()
                : []
            );
        }
    });
    return deferred.promise;
};

module.exports = exports = new Wordfilter();
{'fraction_non_alphanumeric': 0.14214368036880523, 'fraction_numerical': 0.0036496350364963502, 'mean_word_length': 2.0665488810365136, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6841041', 'n_tokens_mistral': 1578, 'n_tokens_neox': 1529, 'n_words': 361}