<commit_before>function tXml(b){function g(){for(var f=[];b[a];){if(62==b.charCodeAt(a)){if(47===b.charCodeAt(a+1)){a=b.indexOf(">",a);break}else if("!"===b[a+1]){if(b.charCodeAt(a+2)==minusCC){for(;62!==b.charCodeAt(a)||b.charCodeAt(a-1)!=minusCC||b.charCodeAt(a-2)!=minusCC||-1==a;)a=b.indexOf(">",a+1);-1===a&&(a=b.length)}else for(a+=2;">"!==b[a];)a++;a++;continue}var c={};if(60!==b.charCodeAt(a))throw"";a++;c.tagName=h();for(var e=!1;62!==b.charCodeAt(a);){var d=b.charCodeAt(a);if(64<d&&91>d||96<d&&123>d){for(var k=
h();'"'!==b[a]&&"'"!==b[a];)a++;var d=b[a],l=++a;a=b.indexOf(d,l);d=b.slice(l,a);e||(c.attributes={},e=!0);c.attributes[k]=d}a++}47!==b.charCodeAt(a-1)&&("script"==c.tagName?(e=a,a=b.indexOf("\x3c/script>",a),c.children=[b.slice(e,a-1)],a+=8):"style"==c.tagName?(e=a,a=b.indexOf("</style>",a),c.children=[b.slice(e,a-1)],a+=7):-1==m.indexOf(c.tagName)&&(a++,c.children=g(k)));f.push(c)}else c=a,a=b.indexOf("<",a),0>a&&(a=b.length-1),c=b.slice(c,a+1),0<c.trim().length&&f.push(c);a++}return f}function h(){for(var f=
a;-1===n.indexOf(b[a]);)a++;return b.slice(f,a)}var n="\n\t>/= ",m=["img","br","input"],a=0;return g()};<commit_msg>meta and link don't get closed in html<commit_after>function tXml(b){function g(){for(var f=[];b[a];){if(62==b.charCodeAt(a)){if(47===b.charCodeAt(a+1)){a=b.indexOf(">",a);break}else if("!"===b[a+1]){if(b.charCodeAt(a+2)==minusCC){for(;62!==b.charCodeAt(a)||b.charCodeAt(a-1)!=minusCC||b.charCodeAt(a-2)!=minusCC||-1==a;)a=b.indexOf(">",a+1);-1===a&&(a=b.length)}else for(a+=2;">"!==b[a];)a++;a++;continue}var c={};if(60!==b.charCodeAt(a))throw"";a++;c.tagName=h();for(var e=!1;62!==b.charCodeAt(a);){var d=b.charCodeAt(a);if(64<d&&91>d||96<d&&123>d){for(var k=
h();'"'!==b[a]&&"'"!==b[a];)a++;var d=b[a],l=++a;a=b.indexOf(d,l);d=b.slice(l,a);e||(c.attributes={},e=!0);c.attributes[k]=d}a++}47!==b.charCodeAt(a-1)&&("script"==c.tagName?(e=a,a=b.indexOf("\x3c/script>",a),c.children=[b.slice(e,a-1)],a+=8):"style"==c.tagName?(e=a,a=b.indexOf("</style>",a),c.children=[b.slice(e,a-1)],a+=7):-1==m.indexOf(c.tagName)&&(a++,c.children=g(k)));f.push(c)}else c=a,a=b.indexOf("<",a),0>a&&(a=b.length-1),c=b.slice(c,a+1),0<c.trim().length&&f.push(c);a++}return f}function h(){for(var f=
a;-1===n.indexOf(b[a]);)a++;return b.slice(f,a)}var n="\n\t>/= ",m=["img","br","input","meta","link"],a=0;return g()};
{'fraction_non_alphanumeric': 0.39278131634819535, 'fraction_numerical': 0.03821656050955414, 'mean_word_length': 14.102564102564102, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1396219', 'n_tokens_mistral': 1242, 'n_tokens_neox': 1216, 'n_words': 48}