<commit_before>var Filter = (function () {

function filter_term(operator, operand) {
    // For legacy reasons we must represent filter_terms as tuples
    // until we phase out all the code that assumes tuples.
    var term = [];
    term[0] = operator;
    term[1] = operand;

    // This is the new style we are phasing in.  (Yes, the same
    // object can be treated like either a tuple or a struct.)
    term.operator = operator;
    term.operand = operand;

    return term;
}

function mit_edu_stream_name_match(message, operand) {
    // MIT users expect narrowing to "social" to also show messages to /^(un)*social(.d)*$/
    // (unsocial, ununsocial, social.d, etc)
    // TODO: hoist the regex compiling out of the closure
    var m = /^(?:un)*(.+?)(?:\.d)*$/i.exec(operand);
    var base_stream_name = operand;
    if (m !== null && m[1] !== undefined) {
        base_stream_name = m[1];
    }
    var related_regexp = new RegExp(/^(un)*/.source + util.escape_regexp(base_stream_name) + /(\.d)*$/.source, 'i');
    return related_regexp.test(message.stream);
}

function mit_edu_topic_name_match(message, operand) {
    // MIT users expect narrowing to topic "foo" to also show messages to /^foo(.d)*$/
    // (foo, foo.d, foo.d.d, etc)
    // TODO: hoist the regex compiling out of the closure
    var m = /^(.*?)(?:\.d)*$/i.exec(operand);
    var base_topic = m[1];
    var related_regexp;

    // Additionally, MIT users expect the empty instance and
    // instance "personal" to be the same.
    if (base_topic === ''
        || base_topic.toLowerCase() === 'personal'
        || base_topic.toLowerCase() === '(instance "")')
    {
        related_regexp = /^(|personal|\(instance ""\))(\.d)*$/i;
    } else {
        related_regexp = new RegExp(/^/.source + util.escape_regexp(base_topic) + /(\.d)*$/.source, 'i');
    }

    return related_regexp.test(message.subject);
}

function message_in_home(message) {
    if (message.type === "private" || message.mentioned ||
        (page_params.narrow_stream !== undefined &&
         message.stream.toLowerCase() === page_params.narrow_stream.toLowerCase())) {
        return true;
    }

    return stream_data.in_home_view(message.stream);
}

function message_matches_search_term(message, operator, operand) {
    switch (operator) {
    case 'is':
        if (operand === 'private') {
            if (message.type !== 'private') {
                return false;
            }
        } else if (operand === 'starred') {
            if (!message.starred) {
                return false;
            }
        } else if (operand === 'mentioned') {
            if (!message.mentioned) {
                return false;
            }
        } else if (operand === 'alerted') {
            if (!message.alerted) {
                return false;
            }
        }

        break;

    case 'in':
        if (operand === 'home') {
            return message_in_home(message);
        }
        else if (operand === 'all') {
            break;
        }
        break;

    case 'near':
        break;

    case 'id':
        if (message.id.toString() !== operand) {
            return false;
        }
        break;

    case 'stream':
        if (message.type !== 'stream') {
            return false;
        }

        operand = operand.toLowerCase();
        if (page_params.domain === "mit.edu") {
            if (!mit_edu_stream_name_match(message, operand)) {
                return false;
            }
        } else if (message.stream.toLowerCase() !== operand) {
            return false;
        }
        break;

    case 'topic':
        if (message.type !== 'stream') {
            return false;
        }

        operand = operand.toLowerCase();
        if (page_params.domain === "mit.edu") {
            if (!mit_edu_topic_name_match(message, operand)) {
                return false;
            }
        } else if (message.subject.toLowerCase() !== operand) {
            return false;
        }
        break;

    case 'sender':
        if ((message.sender_email.toLowerCase() !== operand)) {
            return false;
        }
        break;

    case 'pm-with':
        if ((message.type !== 'private') ||
            message.reply_to.toLowerCase() !== operand.split(',').sort().join(',')) {
            return false;
        }
        break;
    }

    return true;
}


function Filter(operators) {
    if (operators === undefined) {
        this._operators = [];
    } else {
        this._operators = this._canonicalize_operators(operators);
    }
}

var canonical_operators = {"from": "sender", "subject": "topic"};

Filter.canonicalize_operator = function (operator) {
    operator = operator.toLowerCase();
    if (canonical_operators.hasOwnProperty(operator)) {
        return canonical_operators[operator];
    } else {
        return operator;
    }<commit_msg>Clean up double negatives in message_matches_search_term<commit_after>var Filter = (function () {

function filter_term(operator, operand) {
    // For legacy reasons we must represent filter_terms as tuples
    // until we phase out all the code that assumes tuples.
    var term = [];
    term[0] = operator;
    term[1] = operand;

    // This is the new style we are phasing in.  (Yes, the same
    // object can be treated like either a tuple or a struct.)
    term.operator = operator;
    term.operand = operand;

    return term;
}

function mit_edu_stream_name_match(message, operand) {
    // MIT users expect narrowing to "social" to also show messages to /^(un)*social(.d)*$/
    // (unsocial, ununsocial, social.d, etc)
    // TODO: hoist the regex compiling out of the closure
    var m = /^(?:un)*(.+?)(?:\.d)*$/i.exec(operand);
    var base_stream_name = operand;
    if (m !== null && m[1] !== undefined) {
        base_stream_name = m[1];
    }
    var related_regexp = new RegExp(/^(un)*/.source + util.escape_regexp(base_stream_name) + /(\.d)*$/.source, 'i');
    return related_regexp.test(message.stream);
}

function mit_edu_topic_name_match(message, operand) {
    // MIT users expect narrowing to topic "foo" to also show messages to /^foo(.d)*$/
    // (foo, foo.d, foo.d.d, etc)
    // TODO: hoist the regex compiling out of the closure
    var m = /^(.*?)(?:\.d)*$/i.exec(operand);
    var base_topic = m[1];
    var related_regexp;

    // Additionally, MIT users expect the empty instance and
    // instance "personal" to be the same.
    if (base_topic === ''
        || base_topic.toLowerCase() === 'personal'
        || base_topic.toLowerCase() === '(instance "")')
    {
        related_regexp = /^(|personal|\(instance ""\))(\.d)*$/i;
    } else {
        related_regexp = new RegExp(/^/.source + util.escape_regexp(base_topic) + /(\.d)*$/.source, 'i');
    }

    return related_regexp.test(message.subject);
}

function message_in_home(message) {
    if (message.type === "private" || message.mentioned ||
        (page_params.narrow_stream !== undefined &&
         message.stream.toLowerCase() === page_params.narrow_stream.toLowerCase())) {
        return true;
    }

    return stream_data.in_home_view(message.stream);
}

function message_matches_search_term(message, operator, operand) {
    switch (operator) {
    case 'is':
        if (operand === 'private') {
            return (message.type === 'private');
        } else if (operand === 'starred') {
            return message.starred;
        } else if (operand === 'mentioned') {
            return message.mentioned;
        } else if (operand === 'alerted') {
            return message.alerted;
        }
        return true; // is:whatever returns true

    case 'in':
        if (operand === 'home') {
            return message_in_home(message);
        }
        else if (operand === 'all') {
            return true;
        }
        return true; // in:whatever returns true

    case 'near':
        // this is all handled server side
        return true;

    case 'id':
        return (message.id.toString() === operand);

    case 'stream':
        if (message.type !== 'stream') {
            return false;
        }

        operand = operand.toLowerCase();
        if (page_params.domain === "mit.edu") {
            return mit_edu_stream_name_match(message, operand);
        } else {
            return (message.stream.toLowerCase() === operand);
        }

    case 'topic':
        if (message.type !== 'stream') {
            return false;
        }

        operand = operand.toLowerCase();
        if (page_params.domain === "mit.edu") {
            return mit_edu_topic_name_match(message, operand);
        } else {
            return (message.subject.toLowerCase() === operand);
        }

    case 'sender':
        return (message.sender_email.toLowerCase() === operand);

    case 'pm-with':
        return (message.type === 'private') &&
            (message.reply_to.toLowerCase() === operand.split(',').sort().join(','));
    }

    return true; // unknown operators return true (effectively ignored)
}


function Filter(operators) {
    if (operators === undefined) {
        this._operators = [];
    } else {
        this._operators = this._canonicalize_operators(operators);
    }
}

var canonical_operators = {"from": "sender", "subject": "topic"};

Filter.canonicalize_operator = function (operator) {
    operator = operator.toLowerCase();
    if (canonical_operators.hasOwnProperty(operator)) {
        return canonical_operators[operator];
    } else {
        return operator;
    }
{'fraction_non_alphanumeric': 0.14561163207248973, 'fraction_numerical': 0.0010536297545042672, 'mean_word_length': 2.235173824130879, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7444916', 'n_tokens_mistral': 2836, 'n_tokens_neox': 2823, 'n_words': 706}