<commit_before>CodeMirror.defineMode('tikiwiki', function(config, parserConfig) {
	function inBlock(style, terminator, returnTokenizer) {
		return function(stream, state) {
			while (!stream.eol()) {
				if (stream.match(terminator)) {
					state.tokenize = inText;
					break;
				}
				stream.next();
			}
			
			if (returnTokenizer) state.tokenize = returnTokenizer;
			
			return style;
		};
	}
	
	function inLine(style, terminator) {
		return function(stream, state) {
			while(!stream.eol()) {
				stream.next()
			}
			state.tokenize = inText;
			return style;
		};
	}
	
	function inText(stream, state) {
		function chain(parser) {
			state.tokenize = parser;
			return parser(stream, state);
		}
		
		var sol = stream.sol();
		var ch = stream.next();
		
		//non start of line
		switch (ch) { //switch is generally much faster than if, so it is used here
			case "{": //plugin
				type = stream.eat("/") ? "closeTag" : "openTag";
				stream.eatSpace();
				tagName = "";
				var c;
				while ((c = stream.eat(/[^\s\u00a0=\"\'\/?(}]/))) tagName += c;
				state.tokenize = inPlugin;
				return "tag";
				break;
			case "_": //bold
				if (stream.eat("_")) {
					return chain(inBlock("tw-bold", "__", inText));
				}
				break;
			case "'": //italics
				if (stream.eat("'")) {
					// Italic text
					return chain(inBlock("tw-italic", "''", inText));
				}
				break;
			case "(":// Wiki Link
				if (stream.eat("(")) {
					return chain(inBlock("tw-link", "))", inText));
				}
				break;
			case "[":// Weblink
				return chain(inBlock("tw-weblink", "]", inText));
				break;
			case "|": //table
				if (stream.eat("|")) {
					return chain(inBlock("tw-table", "||"));
				}
				break;
			case "-": 
				if (stream.eat("=")) {//titleBar
					return chain(inBlock("tw-titlebar", "=-", inText));
				} else if (stream.eat("-")) {//deleted
					return chain(inBlock("tw-deleted", "--", inText));
				}
				break;
			case "=": //underline
				if (stream.match("==")) {
					return chain(inBlock("tw-underline", "===", inText));
				}
				break;
			case ":":
				if (stream.eat(":")) {
					return chain(inBlock("tw-center", "::"));
				}
				break;
			case "^": //box
				return chain(inBlock("tw-box", "^"));
				break;
		}
		
		//start of line types
		if (sol) {
			switch (ch) {
				case "!": //header at start of line
					if (stream.match('!!!!!')) {
						return chain(inLine("tw-header5"));
					} else if (stream.match('!!!!')) {
						return chain(inLine("tw-header4"));
					} else if (stream.match('!!!')) {
						return chain(inLine("tw-header3"));
					} else if (stream.match('!!')) {
						return chain(inLine("tw-header2"));
					} else {
						return chain(inLine("tw-header1"));
					}
					break;
				case "*": //unordered list line item, or <li /> at start of line
				case "#": //ordered list line item, or <li /> at start of line
					return chain(inLine("tw-listitem"));
					break;
			}
		}
		
		stream.eatWhile(/[&{]/);
		return null;
	}
	
	var indentUnit = config.indentUnit;

	// Return variables for tokenizers
	var pluginName, type;
	function inPlugin(stream, state) {
		var ch = stream.next();
		var peek = stream.peek();
		
		if (ch == "}") {
			state.tokenize = inText;
			//type = ch == ")" ? "endPlugin" : "selfclosePlugin"; inPlugin
			return "tag";
		} else if (ch == "(" || ch == ")") {
			return "plugin";
		} else if (ch == "=") {
			type = "equals";
			if (!/[\'\"]/.test(peek)) {
				state.tokenize = inAttributeNoQuote();
			}
			return null;
		} else if (/[\'\"]/.test(ch)) {
			state.tokenize = inAttribute(ch);
			return state.tokenize(stream, state);
		} else {
			stream.eatWhile(/[^\s\u00a0=\"\'\/?]/);
			return "word";
		}
	}<commit_msg>[FIX] plugin attributes with now quotes highlighting, forgot to return tokenize<commit_after>CodeMirror.defineMode('tikiwiki', function(config, parserConfig) {
	function inBlock(style, terminator, returnTokenizer) {
		return function(stream, state) {
			while (!stream.eol()) {
				if (stream.match(terminator)) {
					state.tokenize = inText;
					break;
				}
				stream.next();
			}
			
			if (returnTokenizer) state.tokenize = returnTokenizer;
			
			return style;
		};
	}
	
	function inLine(style, terminator) {
		return function(stream, state) {
			while(!stream.eol()) {
				stream.next()
			}
			state.tokenize = inText;
			return style;
		};
	}
	
	function inText(stream, state) {
		function chain(parser) {
			state.tokenize = parser;
			return parser(stream, state);
		}
		
		var sol = stream.sol();
		var ch = stream.next();
		
		//non start of line
		switch (ch) { //switch is generally much faster than if, so it is used here
			case "{": //plugin
				type = stream.eat("/") ? "closeTag" : "openTag";
				stream.eatSpace();
				tagName = "";
				var c;
				while ((c = stream.eat(/[^\s\u00a0=\"\'\/?(}]/))) tagName += c;
				state.tokenize = inPlugin;
				return "tag";
				break;
			case "_": //bold
				if (stream.eat("_")) {
					return chain(inBlock("tw-bold", "__", inText));
				}
				break;
			case "'": //italics
				if (stream.eat("'")) {
					// Italic text
					return chain(inBlock("tw-italic", "''", inText));
				}
				break;
			case "(":// Wiki Link
				if (stream.eat("(")) {
					return chain(inBlock("tw-link", "))", inText));
				}
				break;
			case "[":// Weblink
				return chain(inBlock("tw-weblink", "]", inText));
				break;
			case "|": //table
				if (stream.eat("|")) {
					return chain(inBlock("tw-table", "||"));
				}
				break;
			case "-": 
				if (stream.eat("=")) {//titleBar
					return chain(inBlock("tw-titlebar", "=-", inText));
				} else if (stream.eat("-")) {//deleted
					return chain(inBlock("tw-deleted", "--", inText));
				}
				break;
			case "=": //underline
				if (stream.match("==")) {
					return chain(inBlock("tw-underline", "===", inText));
				}
				break;
			case ":":
				if (stream.eat(":")) {
					return chain(inBlock("tw-center", "::"));
				}
				break;
			case "^": //box
				return chain(inBlock("tw-box", "^"));
				break;
		}
		
		//start of line types
		if (sol) {
			switch (ch) {
				case "!": //header at start of line
					if (stream.match('!!!!!')) {
						return chain(inLine("tw-header5"));
					} else if (stream.match('!!!!')) {
						return chain(inLine("tw-header4"));
					} else if (stream.match('!!!')) {
						return chain(inLine("tw-header3"));
					} else if (stream.match('!!')) {
						return chain(inLine("tw-header2"));
					} else {
						return chain(inLine("tw-header1"));
					}
					break;
				case "*": //unordered list line item, or <li /> at start of line
				case "#": //ordered list line item, or <li /> at start of line
					return chain(inLine("tw-listitem"));
					break;
			}
		}
		
		stream.eatWhile(/[&{]/);
		return null;
	}
	
	var indentUnit = config.indentUnit;

	// Return variables for tokenizers
	var pluginName, type;
	function inPlugin(stream, state) {
		var ch = stream.next();
		var peek = stream.peek();
		
		if (ch == "}") {
			state.tokenize = inText;
			//type = ch == ")" ? "endPlugin" : "selfclosePlugin"; inPlugin
			return "tag";
		} else if (ch == "(" || ch == ")") {
			return "plugin";
		} else if (ch == "=") {
			type = "equals";
			
			//here we detect values directly after equal character with no quotes
			if (!/[\'\"]/.test(peek)) {
				state.tokenize = inAttributeNoQuote();
				return state.tokenize(stream, state);
			}
			//end detect values
			
			return null;
		} else if (/[\'\"]/.test(ch)) {
			state.tokenize = inAttribute(ch);
			return state.tokenize(stream, state);
		} else {
			stream.eatWhile(/[^\s\u00a0=\"\'\/?]/);
			return "word";
		}
	}
{'fraction_non_alphanumeric': 0.20157894736842105, 'fraction_numerical': 0.0028947368421052633, 'mean_word_length': 2.6040777619724986, 'pattern_counts': {'":': 32, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5002845', 'n_tokens_mistral': 3349, 'n_tokens_neox': 2602, 'n_words': 603}