<commit_before>Codec = new function() {

	var UrlEncoded = Packages.helma.util.UrlEncoded,
		MD5Encoder = Packages.helma.util.MD5Encoder,
		Base64 = Packages.helma.util.Base64,
		Translate = Packages.org.htmlparser.util.Translate,
		MessageDigest = java.security.MessageDigest,
		JavaString = java.lang.String;

	function encodeJs(str) {
		return str ? (str = uneval(str)).substring(1, str.length - 1) : str;
	}

	function encoder(encoder, str) {
		return function(str) {
			// format() ignores trailing line break characters, so add
			// periods to make them render, then remove the periods again:
			var begin = /^[\n\r]/.test(str);
			var end = /[\n\r]$/.test(str);
			if (begin)
				str = '.' + str;
			if (end)
				str = str + '.';
			str = encoder(str);
			if (begin || end)
				str = str.substring(begin ? 1 : 0, str.length - (end ? 1 : 0));
			if (str && !Html.XHTML) {
			 	// Helma falsy uses <br /> even for non XHTML, so fix this here<commit_msg>Add comment about potential optimisations.<commit_after>Codec = new function() {

	var UrlEncoded = Packages.helma.util.UrlEncoded,
		MD5Encoder = Packages.helma.util.MD5Encoder,
		Base64 = Packages.helma.util.Base64,
		Translate = Packages.org.htmlparser.util.Translate,
		MessageDigest = java.security.MessageDigest,
		JavaString = java.lang.String;

	function encodeJs(str) {
		return str ? (str = uneval(str)).substring(1, str.length - 1) : str;
	}

	function encoder(encoder, str) {
		// TODO: This is used so much, it would make sense to fix Helma instead
		return function(str) {
			// format() ignores trailing line break characters, so add
			// periods to make them render, then remove the periods again:
			var begin = /^[\n\r]/.test(str);
			var end = /[\n\r]$/.test(str);
			if (begin)
				str = '.' + str;
			if (end)
				str = str + '.';
			str = encoder(str);
			if (begin || end)
				str = str.substring(begin ? 1 : 0, str.length - (end ? 1 : 0));
			if (str && !Html.XHTML) {
			 	// Helma falsy uses <br /> even for non XHTML, so fix this here
{'fraction_non_alphanumeric': 0.14596734289955468, 'fraction_numerical': 0.011875309252845126, 'mean_word_length': 3.084848484848485, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6581533', 'n_tokens_mistral': 794, 'n_tokens_neox': 695, 'n_words': 197}