<commit_before>{
	"Lorems": {
		"Latin": "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.",
		"Cyrillic": "Римский император Константин I Великий по достоинству оценил выгодное местоположение приморского Византия, расположенного на стыке Европы и Азии. Кроме того, на решение Константина повлияла неспокойная обстановка в самом Риме: недовольство знати и постоянные распри в борьбе за трон. Император хотел увенчать свою реформаторскую деятельность созданием нового административного центра огромной державы. Закладка города состоялась осенью 324 года, и Константин лично решил обозначить его границы.",
		"Arabic (RTL)": "وضع ابن الهيثم تصور واضح للعلاقة بين النموذج الرياضي المثالي ومنظومة الظواهر الملحوظة.",
		"Chinese": "北京位於華北平原的西北边缘，背靠燕山，有永定河流经老城西南，毗邻天津市、河北省，是一座有三千余年建城历史、八百六十余年建都史的历史文化名城，历史上有金、元、明、清、中华民国（北洋政府时期）等五个朝代在此定都，以及数个政权建政于此，荟萃了自元明清以来的中华文化，拥有众多历史名胜古迹和人文景观。《不列颠百科全书》将北京形容为全球最伟大的城市之一，而且断言，“这座城市是中国历史上最重要的组成部分。在中国过去的八个世纪里，不论历史是否悠久，几乎北京所有主要建筑都拥有着不可磨灭的民族和历史意义”。北京古迹众多，著名的有故宫、天坛、颐和园、圆明园、北海公园等。",
		"Mixed charsets": "Lorem ipsum dolor sit amet, Римский император Константин I Великий, 北京位於華北平原的西北边缘"
	},
	"Text size": {
		"With spaces": {
			"128b": { "_type": "size", "size": "128", "template": "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua." },
			"129b": { "_type": "size", "size": "129", "template": "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua." },
			"256b": { "_type": "size", "size": "256", "template": "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua." },
			"257b": { "_type": "size", "size": "257", "template": "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua." },
			"32K": { "_type": "size", "size": "32768", "template": "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua." },
			"32K + 1": { "_type": "size", "size": "32769", "template": "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua." },
			"64K": { "_type": "size", "size": "65536", "template": "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua." },
			"64K + 1": { "_type": "size", "size": "65537", "template": "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua." }
		},
		"Without spaces": {
			"128b": { "_type": "size", "size": "128", "template": "0123456789" },
			"129b": { "_type": "size", "size": "129", "template": "0123456789" },
			"256b": { "_type": "size", "size": "256", "template": "0123456789" },
			"257b": { "_type": "size", "size": "257", "template": "0123456789" },
			"32K": { "_type": "size", "size": "32768", "template": "0123456789" },
			"32K + 1": { "_type": "size", "size": "32769", "template": "0123456789" },
			"64K": { "_type": "size", "size": "65536", "template": "0123456789" },
			"64K + 1": { "_type": "size", "size": "65537", "template": "0123456789" }
		}
	},
	"Names": [ 
		"<NAME>",
		"<NAME>",
		"<NAME> \"Jimmy\" O'Grady",
		"<NAME>",
		"<NAME>",
		"<NAME>",
		"NULL",
		"GoVeg.com",
		"田中太郎",
		"Le<NAME>ius Tollemache-Tollemache de Orellana Plantagenet Tollemache-Tollemache"
	],
	"E-mail addresses": {
		"Valid" :{ 
			"Simple": "<EMAIL>",
			"Dot in the address": "<EMAIL>",
			"Subdomain": "<EMAIL>",
			"Plus in address": "<EMAIL>",
			"Numeric domain": "email@192.168.3.11",
			"Square bracket around IP address": "email@[192.168.3.11]",
			"Quotes around address": "\"<EMAIL>",
			"Numeric address": "1234567<EMAIL>",
			"Dash in domain": "<EMAIL>",
			"Underscore": "_______<EMAIL>",
			">3 char TLD": "<EMAIL>",
			"2 char TLD": "<EMAIL>",
			"Dash in address": "<EMAIL>",
			"Intranet": "name@localhost"
		},
		"Invalid": {
			"No @ or domain": "plainaddress",
			"Missing @": "<EMAIL>.<EMAIL>.com",
			"Missing address": "@domain.com",
			"Garbage": "#@%^%#$@#$@#.com",
			"Copy/paste from address book with name": "<NAME> <<EMAIL>>",
			"Superfluous text": "<EMAIL> (<NAME>)",
			"Two @": "<EMAIL>",
			"Leading dot in address": <EMAIL>",
			"Trailing dot in address": "<EMAIL>",
			"Multiple dots": "email..<EMAIL>",
			"Unicode chars in address": "<EMAIL>",
			"Leading dash in domain": "<EMAIL>",
			"Leading dot in domain": "<EMAIL>",
			"Invalid IP format": "email@111.222.333.44444",
			"Multiple dots in the domain": "email@domain..com"
		}
	},
	"Numbers": [
		"0", "32768", "32769", "65536", "65537", "2147483648", "2147483649", "4294967296", "4294967297", "1E-16", "-1", "0.0001", 
		"1,234,567", "1.234.567,89"
	],
	"Whitespace": {
		"Tabs and newlines": "Contains\tTabs\nAnd\tNewlines",
		"Leading spaces": "  leading spaces",
		"Mixing tabs and spaces": "\t leading tabs and spaces",
		"spaces on both sides":" Space on both sides ",
		"Just whitespace": "\t \t\n \n " 
	},
	"Format exploits": {
		"SQL Injection": "Robert'); DROP TABLE Students;--",
		"JS Script Injection": "Nice site,  I think I'll take it. <script>alert('Executing JS')</script>",
		"HTML parsing": "<blink>Hello there</blink>",
		"Broken HTML": "<i><b>Bold</i></b>"
	}
}
<commit_msg>Add valid email address with necessary quotes<commit_after>{
	"Lorems": {
		"Latin": "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.",
		"Cyrillic": "Римский император Константин I Великий по достоинству оценил выгодное местоположение приморского Византия, расположенного на стыке Европы и Азии. Кроме того, на решение Константина повлияла неспокойная обстановка в самом Риме: недовольство знати и постоянные распри в борьбе за трон. Император хотел увенчать свою реформаторскую деятельность созданием нового административного центра огромной державы. Закладка города состоялась осенью 324 года, и Константин лично решил обозначить его границы.",
		"Arabic (RTL)": "وضع ابن الهيثم تصور واضح للعلاقة بين النموذج الرياضي المثالي ومنظومة الظواهر الملحوظة.",
		"Chinese": "北京位於華北平原的西北边缘，背靠燕山，有永定河流经老城西南，毗邻天津市、河北省，是一座有三千余年建城历史、八百六十余年建都史的历史文化名城，历史上有金、元、明、清、中华民国（北洋政府时期）等五个朝代在此定都，以及数个政权建政于此，荟萃了自元明清以来的中华文化，拥有众多历史名胜古迹和人文景观。《不列颠百科全书》将北京形容为全球最伟大的城市之一，而且断言，“这座城市是中国历史上最重要的组成部分。在中国过去的八个世纪里，不论历史是否悠久，几乎北京所有主要建筑都拥有着不可磨灭的民族和历史意义”。北京古迹众多，著名的有故宫、天坛、颐和园、圆明园、北海公园等。",
		"Mixed charsets": "Lorem ipsum dolor sit amet, Римский император Константин I Великий, 北京位於華北平原的西北边缘"
	},
	"Text size": {
		"With spaces": {
			"128b": { "_type": "size", "size": "128", "template": "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua." },
			"129b": { "_type": "size", "size": "129", "template": "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua." },
			"256b": { "_type": "size", "size": "256", "template": "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua." },
			"257b": { "_type": "size", "size": "257", "template": "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua." },
			"32K": { "_type": "size", "size": "32768", "template": "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua." },
			"32K + 1": { "_type": "size", "size": "32769", "template": "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua." },
			"64K": { "_type": "size", "size": "65536", "template": "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua." },
			"64K + 1": { "_type": "size", "size": "65537", "template": "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua." }
		},
		"Without spaces": {
			"128b": { "_type": "size", "size": "128", "template": "0123456789" },
			"129b": { "_type": "size", "size": "129", "template": "0123456789" },
			"256b": { "_type": "size", "size": "256", "template": "0123456789" },
			"257b": { "_type": "size", "size": "257", "template": "0123456789" },
			"32K": { "_type": "size", "size": "32768", "template": "0123456789" },
			"32K + 1": { "_type": "size", "size": "32769", "template": "0123456789" },
			"64K": { "_type": "size", "size": "65536", "template": "0123456789" },
			"64K + 1": { "_type": "size", "size": "65537", "template": "0123456789" }
		}
	},
	"Names": [ 
		"<NAME>",
		"<NAME>",
		"<NAME> \"Jimmy\" O'Grady",
		"<NAME>",
		"<NAME>",
		"<NAME>",
		"NULL",
		"GoVeg.com",
		"田中太郎",
		"Leone Sextus Denys Oswolf Fraudatifilius Tollemache-Tollemache de Orellana Plantagenet Tollemache-Tollemache"
	],
	"E-mail addresses": {
		"Valid" :{ 
			"Simple": "<EMAIL>",
			"Dot in the address": "<EMAIL>",
			"Subdomain": "<EMAIL>",
			"Plus in address": "<EMAIL>",
			"Numeric domain": "email@123.123.123.123",
			"Square bracket around IP address": "email@[123.123.123.123]",
			"Unnecessary quotes around address": "\"<EMAIL>\"<EMAIL>",
			"Necessary quotes around address": "\"email..<EMAIL>\"<EMAIL>",
			"Numeric address": "<EMAIL>",
			"Dash in domain": "<EMAIL>",
			"Underscore": "_______<EMAIL>",
			">3 char TLD": "<EMAIL>",
			"2 char TLD": "<EMAIL>",
			"Dash in address": "<EMAIL>",
			"Intranet": "name@localhost"
		},
		"Invalid": {
			"No @ or domain": "plainaddress",
			"Missing @": "email.domain.com",
			"Missing address": "@domain.com",
			"Garbage": "#@%^%#$@#$@#.com",
			"Copy/paste from address book with name": "<NAME> <<EMAIL>>",
			"Superfluous text": "<EMAIL> (<NAME>)",
			"Two @": "<EMAIL>",
			"Leading dot in address": <EMAIL>",
			"Trailing dot in address": "<EMAIL>",
			"Multiple dots": "email..<EMAIL>",
			"Unicode chars in address": "<EMAIL>",
			"Leading dash in domain": "<EMAIL>",
			"Leading dot in domain": "<EMAIL>",
			"Invalid IP format": "email@111.222.333.44444",
			"Multiple dots in the domain": "email@domain..com"
		}
	},
	"Numbers": [
		"0", "32768", "32769", "65536", "65537", "2147483648", "2147483649", "4294967296", "4294967297", "1E-16", "-1", "0.0001", 
		"1,234,567", "1.234.567,89"
	],
	"Whitespace": {
		"Tabs and newlines": "Contains\tTabs\nAnd\tNewlines",
		"Leading spaces": "  leading spaces",
		"Mixing tabs and spaces": "\t leading tabs and spaces",
		"spaces on both sides":" Space on both sides ",
		"Just whitespace": "\t \t\n \n " 
	},
	"Format exploits": {
		"SQL Injection": "Robert'); DROP TABLE Students;--",
		"JS Script Injection": "Nice site,  I think I'll take it. <script>alert('Executing JS')</script>",
		"HTML parsing": "<blink>Hello there</blink>",
		"Broken HTML": "<i><b>Bold</i></b>"
	}
}

{'fraction_non_alphanumeric': 0.17249893571732652, 'fraction_numerical': 0.05406555981268625, 'mean_word_length': 4.864203694458313, 'pattern_counts': {'":': 235, '<': 83, '<?xml version=': 0, '>': 85, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 6, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 30, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3047970', 'n_tokens_mistral': 5593, 'n_tokens_neox': 5211, 'n_words': 1294}