<commit_before>import Ajv = require('ajv');
import ajvKeywords = require('ajv-keywords');
import metaSchema6 = require('ajv/lib/refs/json-schema-draft-06.json');
import { JSONSchema6 } from 'json-schema';
import cloneDeep = require('lodash/cloneDeep');
import defaults = require('lodash/defaults');
import every = require('lodash/every');
import findIndex = require('lodash/findIndex');
import findKey = require('lodash/findKey');
import isArray = require('lodash/isArray');
import isBoolean = require('lodash/isBoolean');
import map = require('lodash/map');
import pickBy = require('lodash/pickBy');
import reduce = require('lodash/reduce');
import startsWith = require('lodash/startsWith');
import trimStart = require('lodash/trimStart');
import { FilterSignature } from 'rendition';
import * as utils from '../../utils';
import { getDataModel } from '../DataTypes';

const ajv = new Ajv();
ajvKeywords(ajv, ['regexp', 'formatMaximum', 'formatMinimum']);
ajv.addMetaSchema(metaSchema6);

export const FULL_TEXT_SLUG = 'full_text_search';
const DEFAULT_DELIMITER = '___';

export const filter = (
	filters: JSONSchema6 | JSONSchema6[],
	collection: any[],
) => {
	// Remove all schemas that may have been compiled already
	ajv.removeSchema(/^.*$/);

	const validators = isArray(filters)
		? filters.map(s => ajv.compile(s))
		: [ajv.compile(filters)];

	if (isArray(collection)) {
		return collection.filter(m => every(validators, v => v(m)));
	}

	return pickBy(collection, m => every(validators, v => v(m)));
};

export const createFullTextSearchFilter = (
	schema: JSONSchema6,
	term: string,
) => {
	// A schema that matches applies the pattern to each known schema field
	const filter = {
		$id: utils.randomString(),
		title: FULL_TEXT_SLUG,
		anyOf: [
			{
				description: `Full text search is ${term}`,
				anyOf: Object.keys(schema.properties!).map(key => ({
					properties: {
						[key]: {
							type: 'string',
							regexp: {
								pattern: utils.regexEscape(term),
								flags: 'i',
							},
						},
					},
					required: [key],<commit_msg>fix(Filters): Make full text search only filter on string type fields<commit_after>import Ajv = require('ajv');
import ajvKeywords = require('ajv-keywords');
import metaSchema6 = require('ajv/lib/refs/json-schema-draft-06.json');
import { JSONSchema6 } from 'json-schema';
import cloneDeep = require('lodash/cloneDeep');
import defaults = require('lodash/defaults');
import every = require('lodash/every');
import findIndex = require('lodash/findIndex');
import findKey = require('lodash/findKey');
import isArray = require('lodash/isArray');
import isBoolean = require('lodash/isBoolean');
import map = require('lodash/map');
import pickBy = require('lodash/pickBy');
import reduce = require('lodash/reduce');
import startsWith = require('lodash/startsWith');
import trimStart = require('lodash/trimStart');
import { FilterSignature } from 'rendition';
import * as utils from '../../utils';
import { getDataModel } from '../DataTypes';

const ajv = new Ajv();
ajvKeywords(ajv, ['regexp', 'formatMaximum', 'formatMinimum']);
ajv.addMetaSchema(metaSchema6);

export const FULL_TEXT_SLUG = 'full_text_search';
const DEFAULT_DELIMITER = '___';

export const filter = (
	filters: JSONSchema6 | JSONSchema6[],
	collection: any[],
) => {
	// Remove all schemas that may have been compiled already
	ajv.removeSchema(/^.*$/);

	const validators = isArray(filters)
		? filters.map(s => ajv.compile(s))
		: [ajv.compile(filters)];

	if (isArray(collection)) {
		return collection.filter(m => every(validators, v => v(m)));
	}

	return pickBy(collection, m => every(validators, v => v(m)));
};

export const createFullTextSearchFilter = (
	schema: JSONSchema6,
	term: string,
) => {
	const stringKeys = reduce(
		schema.properties,
		(carry, item: JSONSchema6, key) => {
			if (item.type === 'string') {
				carry.push(key);
			}

			return carry;
		},
		[] as string[],
	);

	// A schema that matches applies the pattern to each schema field with a type
	// of 'string'
	const filter = {
		$id: utils.randomString(),
		title: FULL_TEXT_SLUG,
		anyOf: [
			{
				description: `Any field contains ${term}`,
				anyOf: stringKeys.map(key => ({
					properties: {
						[key]: {
							type: 'string',
							regexp: {
								pattern: utils.regexEscape(term),
								flags: 'i',
							},
						},
					},
					required: [key],
{'fraction_non_alphanumeric': 0.15795480483907784, 'fraction_numerical': 0.0038803926044282126, 'mean_word_length': 4.363525091799265, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2962028', 'n_tokens_mistral': 1566, 'n_tokens_neox': 1385, 'n_words': 346}