<commit_before>/**
 *  Schema aware Lucene Wrapper
 */
const OP_NOT = '-';
const OP_MUST = '+';
const OP_CAN = '';

/**
 *  The lucene class takes a Schema and creates a query parser.
 *
 *  It can be used without the Schema, and then it "identity" encodes all values/property names.
 *
 *  This is super powerful, the guide to all the syntax options is in the documentation
 *  for the {@link LuceneQuery} type.
 *
 *  @param {?LuceneQuery} query - the query to parse
 *  @param {?Entity} entity - an optional schema entity to use to translate the keys and values
 *  @return {{ querystring: string, placeholders: object }} - the return value contains the string to be used
 *          as the query body and the placeholders to go with it (if any)
 */
export default function createLuceneQuerystring(
    query = {},
    entity = nativeEntity, // the nativeEntity does no mapping of prop names, or encoding of values
) {
    return convertLuceneQuery(entity, query);
}
/**
 * Create a placeholder key
 */
const createPlaceholder = (placeholders, term) => {
    placeholders.push(term);

    return placeholderAliasInQuery(placeholders.length - 1);
};

const placeholderAliasInQuery = (i) => `$ph_${i}`;
const placeholderKeyInRequest = (i) => `ph_${i}`;

// this is only exported to all for use in testing
export const getPlaceholderKeyForIndex = placeholderKeyInRequest;

/**
 *  The initial context object for the query building
 */
const initialContext = (entity) => {
    return {
        querystring: '',
        placeholders: [],
        op: OP_MUST,
        entity,
    };
};

/**
 *  This is a helper for this lib if you want to use it without the ORM property mappings.
 */
const nativeEntity = {
    internal: true,
    prop: (name) => {
        const transform = /^ogit\/_/.test(name) ? (x) => x : (x) => '' + x;

        return {
            src: name,
            dst: name,
            encode: transform,
            decode: transform,
        };
    },
};

/**
 * using the entity given, map the query object to a Lucene query string + placeholders.
 */
function convertLuceneQuery(entity, query) {
    const context = initialContext(entity);
    const querystring = createQuerySegment(context, normaliseQuery(query));
    const parsed = {
        querystring,
        placeholders: context.placeholders.reduce(
            (opts, placeholder, index) => {
                opts[placeholderKeyInRequest(index)] = placeholder;

                return opts;
            },
            {},
        ),
    };

    if (!entity.internal) {
        //this is not an internal-only entity (i.e. a fake one that just translates
        // ogit internal attributes. This means we should insert a "ogit/_type" filter.
        parsed.querystring = `+${slashForward('ogit/_type')}:${quote(
            entity.ogit,
        )} ${parsed.querystring}`;
    }

    return parsed;
}

//turns a single value into an array if not already
const ensureArray = (value) => {
    return Array.isArray(value) ? value : [value];
};

//the "$" keys which do not recurse
const noRecurseKeys = ['$search', '$range', '$missing'];

//forces all properties to be arrays.
//knows how to recurse and when not to.
const normaliseQuery = (queryObject, isAnyOperator = false) => {
    return Object.keys(queryObject).map((key) => {
        const value = queryObject[key];

        if (key[0] === '$' && noRecurseKeys.indexOf(key) === -1) {
            //we should recurse
            return { key, values: normaliseQuery(value, true) };
        }

        if (key.match(/\.ngram$/)) {
            // ngram search doesn't work with quoted phrases, like a "Single value", "Run machine".
            // Because of that we need to split our value by whitespace
            if (!Array.isArray(value)) {
                return { key, values: value.split(' '), isAnyOperator };
            } else {
                const values = value.reduce(
                    (acc, val) => acc.concat(val.split(' ')),
                    [],
                );

                return { key, values, isAnyOperator };
            }
        }

        return { key, values: ensureArray(value), isAnyOperator };
    });
};<commit_msg>Improve NGRAM Filter<commit_after>/**
 *  Schema aware Lucene Wrapper
 */
const OP_NOT = '-';
const OP_MUST = '+';
const OP_CAN = '';

/**
 *  The lucene class takes a Schema and creates a query parser.
 *
 *  It can be used without the Schema, and then it "identity" encodes all values/property names.
 *
 *  This is super powerful, the guide to all the syntax options is in the documentation
 *  for the {@link LuceneQuery} type.
 *
 *  @param {?LuceneQuery} query - the query to parse
 *  @param {?Entity} entity - an optional schema entity to use to translate the keys and values
 *  @return {{ querystring: string, placeholders: object }} - the return value contains the string to be used
 *          as the query body and the placeholders to go with it (if any)
 */
export default function createLuceneQuerystring(
    query = {},
    entity = nativeEntity, // the nativeEntity does no mapping of prop names, or encoding of values
) {
    return convertLuceneQuery(entity, query);
}
/**
 * Create a placeholder key
 */
const createPlaceholder = (placeholders, term) => {
    placeholders.push(term);

    return placeholderAliasInQuery(placeholders.length - 1);
};

const placeholderAliasInQuery = (i) => `$ph_${i}`;
const placeholderKeyInRequest = (i) => `ph_${i}`;

// this is only exported to all for use in testing
export const getPlaceholderKeyForIndex = placeholderKeyInRequest;

/**
 *  The initial context object for the query building
 */
const initialContext = (entity) => {
    return {
        querystring: '',
        placeholders: [],
        op: OP_MUST,
        entity,
    };
};

/**
 *  This is a helper for this lib if you want to use it without the ORM property mappings.
 */
const nativeEntity = {
    internal: true,
    prop: (name) => {
        const transform = /^ogit\/_/.test(name) ? (x) => x : (x) => '' + x;

        return {
            src: name,
            dst: name,
            encode: transform,
            decode: transform,
        };
    },
};

/**
 * using the entity given, map the query object to a Lucene query string + placeholders.
 */
function convertLuceneQuery(entity, query) {
    const context = initialContext(entity);
    const querystring = createQuerySegment(context, normaliseQuery(query));
    const parsed = {
        querystring,
        placeholders: context.placeholders.reduce(
            (opts, placeholder, index) => {
                opts[placeholderKeyInRequest(index)] = placeholder;

                return opts;
            },
            {},
        ),
    };

    if (!entity.internal) {
        //this is not an internal-only entity (i.e. a fake one that just translates
        // ogit internal attributes. This means we should insert a "ogit/_type" filter.
        parsed.querystring = `+${slashForward('ogit/_type')}:${quote(
            entity.ogit,
        )} ${parsed.querystring}`;
    }

    return parsed;
}

//turns a single value into an array if not already
const ensureArray = (value) => {
    return Array.isArray(value) ? value : [value];
};

//the "$" keys which do not recurse
const noRecurseKeys = ['$search', '$range', '$missing'];

// split value into ngrams min length 2, max length 10
const ngramChunker = (value) => value.match(/.{2,10}/g) || [];

const ngramArray = (values) => {
    return values
        .reduce((acc, val) => {
            const vals = val
                .toLowerCase()
                .split(/[^a-z0-9]/)
                .reduce((acc2, val2) => [...acc2, ...ngramChunker(val2)], []);

            return [...acc, ...vals];
        }, [])
        .filter(Boolean);
};

//forces all properties to be arrays.
//knows how to recurse and when not to.
const normaliseQuery = (queryObject, isAnyOperator = false) => {
    return Object.keys(queryObject).map((key) => {
        const value = queryObject[key];

        if (key[0] === '$' && noRecurseKeys.indexOf(key) === -1) {
            //we should recurse
            return { key, values: normaliseQuery(value, true) };
        }

        let values = ensureArray(value);

        if (key.match(/\.ngram$/)) {
            // ngram search doesn't work with quoted phrases, like a "Single value", "Run machine".
            // Because of that we need to split our value by non-alphanumeric characters
            values = ngramArray(values);
        }

        return { key, values, isAnyOperator };
    });
};
{'fraction_non_alphanumeric': 0.12399299474605954, 'fraction_numerical': 0.0021015761821366026, 'mean_word_length': 2.407319013524264, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '963213', 'n_tokens_mistral': 2448, 'n_tokens_neox': 2387, 'n_words': 823}