<commit_before>/*
 * Parses a search query. Returns an object that looks like this:
 *    .sr_term_sets: list of [true/false, term] subreddit names to match.
 *    .tag_term_sets: list of [true/false, tags ...] tag sets to match.
 *    .name_terms: list of emote name terms to match.
 * or null, if there was no query.
 */
function parse_search_query(terms) {
    var query = {sr_term_sets: [], tag_term_sets: [], name_terms: []};

    /*
     * Adds a list of matching ids as one term. Cancels out earlier
     * opposites where appropriate.
     */
    function add_cancelable_id_list(sets, positive, ids) {
        // Search from right to left, looking for sets of an opposite type
        for(var set_i = sets.length - 1; set_i >= 0; set_i--) {
            var set = sets[set_i];
            if(set[0] !== positive) {
                // Look for matching ids, and remove them
                for(var id_i = ids.length - 1; id_i >= 0; id_i--) {
                    var index = set.indexOf(ids[id_i]);
                    if(index > -1) {
                        // When a tag and an antitag collide...
                        set.splice(index, 1);
                        ids.splice(id_i, 1);
                        // It makes a great big mess of my search code is what
                    }
                }
                // This set was cancelled out completely, so remove it
                if(set.length <= 1) {
                    sets.splice(set_i, 1);
                }
            }
        }
        // If there's still anything left, add this new set
        if(ids.length) {
            ids.unshift(positive);
            sets.push(ids);
        }
    }

    /*
     * Adds an id set term, by either adding it exactly or adding all
     * matching tags.
     */
    function add_id_set(sets, name2id, positive, exact, query) {
        var id = name2id[exact];
        if(id) {
            add_cancelable_id_list(sets, positive, [id]); // Exact name match
        } else {
            // Search through all tags for one that looks like the term.
            var matches = [];
            for(var name in name2id) {
                id = name2id[name];
                if(name.indexOf(query) > -1 && matches.indexOf(id) < 0) {
                    matches.push(id);
                }
            }
            // If we found anything at all, append it
            if(matches.length) {
                add_cancelable_id_list(sets, positive, matches);
            }
        }
    }

    // Parse query
    for(var t = 0; t < terms.length; t++) {
        var term = terms[t];
        var is_tag = false; // Whether it started with "+"/"-" (which could actually be a subreddit!!)
        var positive = true;
        if(term[0] === "+" || term[0] === "-") {
            // It's a thing that can be negated, which means either subreddit
            // or a tag.
            is_tag = true;
            positive = term[0] === "+";
            term = term.slice(1);
            if(!term) {
                continue;
            }
        }
        if(term.slice(0, 3) === "sr:") {
            if(term.length > 3) {
                // Chop off sr:
                add_id_set(query.sr_term_sets, sr_name2id, positive, term.slice(3), term.slice(3));
            }
        } else if(term.slice(0, 2) === "r/") {
            if(term.length > 2) {
                // Leave the r/ on
                add_id_set(query.sr_term_sets, sr_name2id, positive, term, term);
            }
        } else if(is_tag) {
            // A tag-like thing that isn't a subreddit = tag term
            add_id_set(query.tag_term_sets, tag_name2id, positive, "+" + term, term);
        } else {
            query.name_terms.push(term); // Anything else
        }
    }

    if(query.sr_term_sets.length || query.tag_term_sets.length || query.name_terms.length) {
        return query;
    } else {
        return null;
    }
}

/*
 * Executes a search query. Returns an object with two properties:
 *    .results: a sorted list of emotes
 */
function execute_search(store, query) {
    var results = [];
    no_match:
    for(var emote_name in emote_map) {
        var emote_info = store.lookup_core_emote(emote_name, true);
        var lc_emote_name = emote_name.toLowerCase();
<commit_msg>Make search query parsing code a little more consistent with literal +<commit_after>/*
 * Parses a search query. Returns an object that looks like this:
 *    .sr_term_sets: list of [true/false, term] subreddit names to match.
 *    .tag_term_sets: list of [true/false, tags ...] tag sets to match.
 *    .name_terms: list of emote name terms to match.
 * or null, if there was no query.
 */
function parse_search_query(terms) {
    var query = {sr_term_sets: [], tag_term_sets: [], name_terms: []};

    /*
     * Adds a list of matching ids as one term. Cancels out earlier
     * opposites where appropriate.
     */
    function add_cancelable_id_list(sets, positive, ids) {
        // Search from right to left, looking for sets of an opposite type
        for(var set_i = sets.length - 1; set_i >= 0; set_i--) {
            var set = sets[set_i];
            if(set[0] !== positive) {
                // Look for matching ids, and remove them
                for(var id_i = ids.length - 1; id_i >= 0; id_i--) {
                    var index = set.indexOf(ids[id_i]);
                    if(index > -1) {
                        // When a tag and an antitag collide...
                        set.splice(index, 1);
                        ids.splice(id_i, 1);
                        // It makes a great big mess of my search code is what
                    }
                }
                // This set was cancelled out completely, so remove it
                if(set.length <= 1) {
                    sets.splice(set_i, 1);
                }
            }
        }
        // If there's still anything left, add this new set
        if(ids.length) {
            ids.unshift(positive);
            sets.push(ids);
        }
    }

    /*
     * Adds an id set term, by either adding it exactly or adding all
     * matching tags.
     */
    function add_id_set(sets, name2id, positive, exact, query) {
        var id = name2id[exact];
        if(id) {
            add_cancelable_id_list(sets, positive, [id]); // Exact name match
        } else {
            // Search through all tags for one that looks like the term.
            var matches = [];
            for(var name in name2id) {
                id = name2id[name];
                if(name.indexOf(query) > -1 && matches.indexOf(id) < 0) {
                    matches.push(id);
                }
            }
            // If we found anything at all, append it
            if(matches.length) {
                add_cancelable_id_list(sets, positive, matches);
            }
        }
    }

    // Parse query
    for(var t = 0; t < terms.length; t++) {
        var term = terms[t];
        var is_tag = false; // Whether it started with "+"/"-" (which could actually be a subreddit!!)
        var positive = true;
        if(term[0] === "+" || term[0] === "-") {
            // It's a thing that can be negated, which means either subreddit
            // or a tag.
            is_tag = true;
            positive = term[0] === "+";
            term = term.slice(1);
            if(!term) {
                continue;
            }
        }
        if(term.slice(0, 3) === "sr:") {
            if(term.length > 3) {
                // Chop off sr:
                add_id_set(query.sr_term_sets, sr_name2id, positive, term.slice(3), term.slice(3));
            }
        } else if(term.slice(0, 2) === "r/") {
            if(term.length > 2) {
                // Leave the r/ on
                add_id_set(query.sr_term_sets, sr_name2id, positive, term, term);
            }
        } else if(is_tag) {
            if(term.length > 1) {
                // A tag-like thing that isn't a subreddit = tag term
                add_id_set(query.tag_term_sets, tag_name2id, positive, "+" + term, term);
            }
        } else {
            query.name_terms.push(term); // Anything else
        }
    }

    if(query.sr_term_sets.length || query.tag_term_sets.length || query.name_terms.length) {
        return query;
    } else {
        return null;
    }
}

/*
 * Executes a search query. Returns an object with two properties:
 *    .results: a sorted list of emotes
 */
function execute_search(store, query) {
    var results = [];
    no_match:
    for(var emote_name in emote_map) {
        var emote_info = store.lookup_core_emote(emote_name, true);
        var lc_emote_name = emote_name.toLowerCase();

{'fraction_non_alphanumeric': 0.11467836932671209, 'fraction_numerical': 0.007506640489663934, 'mean_word_length': 1.5174418604651163, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3580864', 'n_tokens_mistral': 2650, 'n_tokens_neox': 2547, 'n_words': 753}