<commit_before>/**
 * Hashtag
 */

export type TextElementHashtag = {
	type: 'hashtag'
	content: string
	hashtag: string
};

export default function(text: string, i: number) {
	if (!(/^\s#[^\s\.,!\?]+/.test(text) || (i == 0 && /^#[^\s\.,!\?]+/.test(text)))) return null;
	const isHead = text.startsWith('#');
	const hashtag = text.match(/^\s?#[^\s\.,!\?]+/)[0];
	const res: any[] = !isHead ? [{
		type: 'text',
		content: text[0]
	}] : [];
	res.push({
		type: 'hashtag',
		content: isHead ? hashtag : hashtag.substr(1),
		hashtag: isHead ? hashtag.substr(1) : hashtag.substr(2)
	});
	return res as TextElementHashtag[];
}
<commit_msg>[MFM] Better hashtag detection<commit_after>/**
 * Hashtag
 */

export type TextElementHashtag = {
	type: 'hashtag'
	content: string
	hashtag: string
};

export default function(text: string, i: number) {
	if (!(/^\s#[^\s\.,!\?#]+/.test(text) || (i == 0 && /^#[^\s\.,!\?#]+/.test(text)))) return null;
	const isHead = text.startsWith('#');
	const hashtag = text.match(/^\s?#[^\s\.,!\?#]+/)[0];
	const res: any[] = !isHead ? [{
		type: 'text',
		content: text[0]
	}] : [];
	res.push({
		type: 'hashtag',
		content: isHead ? hashtag : hashtag.substr(1),
		hashtag: isHead ? hashtag.substr(1) : hashtag.substr(2)
	});
	return res as TextElementHashtag[];
}

{'fraction_non_alphanumeric': 0.24922360248447206, 'fraction_numerical': 0.009316770186335404, 'mean_word_length': 3.845864661654135, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4492473', 'n_tokens_mistral': 538, 'n_tokens_neox': 523, 'n_words': 109}