<commit_before>/*global console define require window*/
var registerPlugin;
require(['orion/plugin', 'orion/EventTarget', 'orion/textview/textModel',
		'orion-cm/mirrorTextModel', 'orion-cm/highlighter', 'orion-cm/globalCodeMirror',
		'cm/codemirror-compressed', 'requirejs/domReady!'],
	function(PluginProvider, EventTarget, mTextModel, mMirrorTextModel, mHighlighter, globalCodeMirror, _, document) {
		// Invert 1:1 map
		function invert(obj) {
			var result = {};
			for (var key in obj) {
				if (obj.hasOwnProperty(key)) {
					var arr = obj[key];
					if (arr instanceof Array) {
						for (var i=0; i < arr.length; i++) {
							var val = arr[i];
							result[val] = key;
						}
					} else {
						result[arr] = key;
					}
				}
			}
			return result;
		}

		var modes = "clike clojure coffeescript css diff ecl erlang gfm go groovy haskell haxe htmlembedded htmlmixed javascript jinja2 less lua markdown mysql ntriples ocaml pascal perl php pig plsql properties python r rpm rst ruby rust scheme shell smalltalk smarty sparql stex tiddlywiki tiki vb vbscript velocity verilog xml xquery yaml".split(" ");
		// Orion's content type IDs are MIME-like but don't match CodeMirror's exactly, so we map them.
		var mime2ContentType = {
			"application/xml": "application/xml", // orion
			"text/x-java": "text/x-java-source", // orion
			"text/javascript": "application/javascript" // orion
		};
		var contentType2Mime = invert(mime2ContentType);
		var mime2Ext = {
			"text/x-csrc": ["c", "C", "h"],
			"text/x-c++src": ["cc", "cpp", "c++"],
			"text/x-clojure": ["clj"],
			"text/x-coffeescript": ["coffee"],
			"text/x-csharp": ["cs"],
			"text/css": ["css"],
			"text/x-diff": ["diff", "patch"],
			"text/x-erlang": ["erlang"],
			"text/x-go": ["go"],
			"text/x-groovy": ["groovy"],
			"text/x-haskell": ["hs"],
			"text/html": ["html", "htm"],
			"text/x-java": ["java"],
			"text/javascript": ["js"],
			"application/json": ["json"],
			"jinja2": [],
			"text/x-lua": ["lua"],
			"text/x-markdown": ["md"],
			"text/n-triples": ["nt"],
			"text/x-ocaml": ["ocaml"],
			"text/x-pascal": ["pascal", "p"],
			"text/x-perl": ["pl"],
			"text/x-php": ["php", "php3", "php4", "php5"],
			"application/x-httpd-php": ["phtml"],
			"text/x-properties": ["properties", "ini", "conf"],
			"text/x-python": ["py"],
			"text/x-rsrc": ["r"],
			"text/x-rst": ["rst"],
			"text/x-ruby": ["rb"],
			"text/x-rust": ["rs", "rc"],
			"text/x-scheme": ["scm", "ss"],
			"text/x-sh": ["sh"],
			"text/x-stsrc": ["sm"],
			"application/x-sparql-query": ["spk"],
			"text/x-plsql": ["sql"],
			"text/stex": [],
			"text/x-tiddlywiki": [],
			"text/x-vb": ["vb"],
			"text/vbscript": ["vbs"],
			"text/velocity": [],
	//		"text/xml": ["xml"],
			"application/xml": ["xml"],
			"text/x-yaml": ["yaml", "yml"]<commit_msg>Fix .sh and .conf highlighting<commit_after>/*global console define require window*/
var registerPlugin;
require(['orion/plugin', 'orion/EventTarget', 'orion/textview/textModel',
		'orion-cm/mirrorTextModel', 'orion-cm/highlighter', 'orion-cm/globalCodeMirror',
		'cm/codemirror-compressed', 'requirejs/domReady!'],
	function(PluginProvider, EventTarget, mTextModel, mMirrorTextModel, mHighlighter, globalCodeMirror, _, document) {
		// Invert 1:1 map
		function invert(obj) {
			var result = {};
			for (var key in obj) {
				if (obj.hasOwnProperty(key)) {
					var arr = obj[key];
					if (arr instanceof Array) {
						for (var i=0; i < arr.length; i++) {
							var val = arr[i];
							result[val] = key;
						}
					} else {
						result[arr] = key;
					}
				}
			}
			return result;
		}

		var modes = "clike clojure coffeescript css diff ecl erlang gfm go groovy haskell haxe htmlembedded htmlmixed javascript jinja2 less lua markdown mysql ntriples ocaml pascal perl php pig plsql properties python r rpm rst ruby rust scheme shell smalltalk smarty sparql stex tiddlywiki tiki vb vbscript velocity verilog xml xquery yaml".split(" ");
		// Orion's content type IDs are MIME-like but don't match CodeMirror's exactly, so we map them.
		var mime2ContentType = {
			"application/xml": "application/xml", // orion
			"text/x-java": "text/x-java-source", // orion
			"text/javascript": "application/javascript", // orion
			"text/x-sh": "text/sh", // orion
			"text/x-properties": "text/conf" // orion
		};
		var contentType2Mime = invert(mime2ContentType);
		var mime2Ext = {
			"text/x-csrc": ["c", "C", "h"],
			"text/x-c++src": ["cc", "cpp", "c++"],
			"text/x-clojure": ["clj"],
			"text/x-coffeescript": ["coffee"],
			"text/x-csharp": ["cs"],
			"text/css": ["css"],
			"text/x-diff": ["diff", "patch"],
			"text/x-erlang": ["erlang"],
			"text/x-go": ["go"],
			"text/x-groovy": ["groovy"],
			"text/x-haskell": ["hs"],
			"text/html": ["html", "htm"],
			"text/x-java": ["java"],
			"text/javascript": ["js"],
			"application/json": ["json"],
			"jinja2": [],
			"text/x-lua": ["lua"],
			"text/x-markdown": ["md"],
			"text/n-triples": ["nt"],
			"text/x-ocaml": ["ocaml"],
			"text/x-pascal": ["pascal", "p"],
			"text/x-perl": ["pl"],
			"text/x-php": ["php", "php3", "php4", "php5"],
			"application/x-httpd-php": ["phtml"],
			"text/x-properties": ["properties", "ini"/*, "conf"*/],
			"text/x-python": ["py"],
			"text/x-rsrc": ["r"],
			"text/x-rst": ["rst"],
			"text/x-ruby": ["rb"],
			"text/x-rust": ["rs", "rc"],
			"text/x-scheme": ["scm", "ss"],
			"text/x-sh": ["sh"],
			"text/x-stsrc": ["sm"],
			"application/x-sparql-query": ["spk"],
			"text/x-plsql": ["sql"],
			"text/stex": [],
			"text/x-tiddlywiki": [],
			"text/x-vb": ["vb"],
			"text/vbscript": ["vbs"],
			"text/velocity": [],
	//		"text/xml": ["xml"],
			"application/xml": ["xml"],
			"text/x-yaml": ["yaml", "yml"]
{'fraction_non_alphanumeric': 0.22758259045621396, 'fraction_numerical': 0.004195070791819612, 'mean_word_length': 3.445998445998446, 'pattern_counts': {'":': 94, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 14}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1974766', 'n_tokens_mistral': 2570, 'n_tokens_neox': 2175, 'n_words': 485}