<commit_before>import all from 'lodash.every'

export default function SearchMatcher(text) {
	text = text.trim().toLowerCase()
	if (text === "") {
		return song => true;
	}

	/*
	These lists contain the substrings to be matched. In order to be a match,
	a song's title must have ALL of the title parts as substrings, all of the
	artist parts as substrings, and all of the generic parts in either the
	title or artist
	*/
	const generics = []
	const artists = []
	const titles = []

	const tokenRegex = /([^\s"]+)|"([^"]+)"/g
	const artistRegex = /^artist:\s*(([^\s].*)?)$/
	const titleRegex = /^title:\s*(([^\s].*)?)$/

	let regexMatch;

	while((regexMatch = tokenRegex.exec(text)) !== null) {
		let matchText = regexMatch[1] || regexMatch[2];

		if (matchText) {
			if ((regexMatch = artistRegex.exec(matchText)) !== null) {
				if (regexMatch[2]) artists.push(regexMatch[2]);
			} else if ((regexMatch = titleRegex.exec(matchText)) !== null) {
				if (regexMatch[2]) titles.push(regexMatch[1]);
			} else {
				generics.push(matchText);
			}
		}
	}

	const makeMatcher = function(tokens, token_matcher, prev_matcher=null) {
		return tokens.length ?
			prev_matcher === null ?
				song => all(tokens, token => token_matcher(song, token)) :
				song => prev_matcher(song) && all(tokens, token => token_matcher(song, token)) :
			prev_matcher
	}

	let matcher = null
	matcher = makeMatcher(generics, ((song, blob) =>
		song.title.includes(blob) || song.artist.includes(blob)), matcher)
	matcher = makeMatcher(artists, ((song, artist) =>
		song.artist.includes(artist)), matcher)
	matcher = makeMatcher(titles, ((song, title) =>
		song.title.includes(title)), matcher)

	return matcher === null ?
		song => true :
		song => matcher({
			title: song.title.toLowerCase(),
			artist: song.artist.toLowerCase(),
		})
}


<commit_msg>Update searchMatcher.jsx<commit_after>import all from 'lodash.every'

export default function SearchMatcher(text) {
	text = text.trim().toLowerCase()
	if (text === "") {
		return song => true;
	}

	/*
	These lists contain the substrings to be matched. In order to be a match,
	a song's title must have ALL of the title parts as substrings, all of the
	artist parts as substrings, and all of the generic parts in either the
	title or artist
	*/
	const generics = []
	const artists = []
	const titles = []

	const tokenRegex = /([^\s"]+)|"([^"]+)"/g
	const artistRegex = /^artist:\s*(([^\s].*)?)$/
	const titleRegex = /^title:\s*(([^\s].*)?)$/

	let regexMatch;

	while((regexMatch = tokenRegex.exec(text)) !== null) {
		let matchText = regexMatch[1] || regexMatch[2];

		if (matchText) {
			if ((regexMatch = artistRegex.exec(matchText)) !== null) {
				if (regexMatch[2]) artists.push(regexMatch[2]);
			} else if ((regexMatch = titleRegex.exec(matchText)) !== null) {
				if (regexMatch[2]) titles.push(regexMatch[1]);
			} else {
				generics.push(matchText);
			}
		}
	}

	const makeMatcher = (tokens, token_matcher, prev_matcher=null) =>
		tokens.length ?
			prev_matcher === null ?
				song => all(tokens, token => token_matcher(song, token)) :
				song => prev_matcher(song) && all(tokens, token => token_matcher(song, token)) :
			prev_matcher

	let matcher = null
	matcher = makeMatcher(generics, ((song, blob) =>
		song.title.includes(blob) || song.artist.includes(blob)), matcher)
	matcher = makeMatcher(artists, ((song, artist) =>
		song.artist.includes(artist)), matcher)
	matcher = makeMatcher(titles, ((song, title) =>
		song.title.includes(title)), matcher)

	return matcher === null ?
		song => true :
		song => matcher({
			title: song.title.toLowerCase(),
			artist: song.artist.toLowerCase(),
		})
}



{'fraction_non_alphanumeric': 0.1669407894736842, 'fraction_numerical': 0.003289473684210526, 'mean_word_length': 4.054016620498615, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '197954', 'n_tokens_mistral': 1355, 'n_tokens_neox': 1227, 'n_words': 314}