<commit_before>#include "stdafx.h"
#include "path.h"
#include "utils.h"

prefix_matcher::prefix_matcher(std::initializer_list<wstr> patterns)
	: trie(), done(false), pos(0) {
	trie.resize(1);
	for (crwstr s : patterns) {
		size_t p = 0;
		for (size_t i = 0; i < s.size() - 1; i++) {
			auto &m = trie[p];
			auto it = m.find(s[i]);
			if (it == m.end()) {
				it = m.insert(std::make_pair(s[i], trie.size())).first;
				trie.resize(trie.size() + 1);
			}
			p = it->second;
		}
		trie[p][s.back()] = 0;
	}
}

match_result prefix_matcher::move(wchar_t c) {
	if (done) return match_unknown;
	auto &m = trie[pos];
	auto it = m.find(c);
	if (it == m.end()) {
		done = true;
		return match_failed;
	}<commit_msg>Fix possible dangling iterator.<commit_after>#include "stdafx.h"
#include "path.h"
#include "utils.h"

prefix_matcher::prefix_matcher(std::initializer_list<wstr> patterns)
	: trie(), done(false), pos(0) {
	trie.resize(1);
	for (crwstr s : patterns) {
		size_t p = 0;
		for (size_t i = 0; i < s.size() - 1; i++) {
			auto &m = trie[p];
			auto it = m.find(s[i]);
			if (it == m.end()) {
				p = m[s[i]] = trie.size();
				trie.resize(trie.size() + 1);
			} else p = it->second;
		}
		trie[p][s.back()] = 0;
	}
}

match_result prefix_matcher::move(wchar_t c) {
	if (done) return match_unknown;
	auto &m = trie[pos];
	auto it = m.find(c);
	if (it == m.end()) {
		done = true;
		return match_failed;
	}
{'fraction_non_alphanumeric': 0.20569395017793593, 'fraction_numerical': 0.0099644128113879, 'mean_word_length': 3.5064102564102564, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1317428', 'n_tokens_mistral': 673, 'n_tokens_neox': 620, 'n_words': 134}