<commit_before>(function(ced) {

	function Highlighter(editor) {
		var escape = ced.Utils.escape;

		function highlight(section) {
			var text = escape(section.text);
			text = ced.Prism.highlight(text, editor.options.language);
			text = text.replace(/\n/gm, '<span>\n</span>');
			/*
			 var frontMatter = section.textWithFrontMatter.substring(0, section.textWithFrontMatter.length - section.text.length);
			 if(frontMatter.length) {
			 // Front matter highlighting
			 frontMatter = escape(frontMatter);
			 frontMatter = frontMatter.replace(/\n/g, '<span class="token lf">\n</span>');
			 text = '<span class="token md">' + frontMatter + '</span>' + text;
			 }
			 */
			var sectionElt = document.createElement('span');
			sectionElt.id = 'classeur-editor-section-' + section.id;
			sectionElt.className = 'classeur-editor-section';
			sectionElt.generated = true;
			sectionElt.innerHTML = text;
			section.elt = sectionElt;
		}

		var contentElt = editor.$contentElt;
		var sectionCounter = 0;
		this.trailingLfElt;
		this.isComposing = 0;

		var sectionList = [];
		var sectionsToRemove = [];
		var modifiedSections = [];
		var insertBeforeSection;

		this.addTrailingLfElt = function() {
			this.trailingLfElt = document.createElement('span');
			this.trailingLfElt.textContent = '\n';
			contentElt.appendChild(this.trailingLfElt);
		};

		this.parseSections = function(content, isInit) {
			var tmpText = content + "\n\n";
			var newSectionList = [];
			var offset = 0;

			function addSection(startOffset, endOffset) {
				var sectionText = tmpText.substring(offset, endOffset);
				newSectionList.push({
					id: ++sectionCounter,
					text: sectionText
				});
			}

			// Look for delimiters
			editor.options.sectionDelimiter && tmpText.replace(editor.options.sectionDelimiter, function(match, matchOffset) {
				// Create a new section with the text preceding the delimiter
				addSection(offset, matchOffset);
				offset = matchOffset;
			});

			// Last section
			addSection(offset, content.length);

			modifiedSections = [];
			sectionsToRemove = [];
			insertBeforeSection = undefined;

			if(isInit) {
				// Render everything if isInit
				sectionsToRemove = sectionList;
				sectionList = newSectionList;
				modifiedSections = newSectionList;
			}
			else {
				// Find modified section starting from top
				var leftIndex = sectionList.length;
				sectionList.some(function(section, index) {
					var newSection = newSectionList[index];
					if(index >= newSectionList.length ||
							// Check text modification
						section.text != newSection.text ||
							// Check that section has not been detached or moved
						section.elt.parentNode !== contentElt ||
							// Check also the content since nodes can be injected in sections via copy/paste
						section.elt.textContent != newSection.text) {
						leftIndex = index;
						return true;
					}
				});

				// Find modified section starting from bottom
				var rightIndex = -sectionList.length;
				sectionList.slice().reverse().some(function(section, index) {
					var newSection = newSectionList[newSectionList.length - index - 1];
					if(index >= newSectionList.length ||
							// Check modified
						section.text != newSection.text ||
							// Check that section has not been detached or moved
						section.elt.parentNode !== contentElt ||
							// Check also the content since nodes can be injected in sections via copy/paste
						section.elt.textContent != newSection.text) {
						rightIndex = -index;
						return true;
					}
				});

				if(leftIndex - rightIndex > sectionList.length) {
					// Prevent overlap
					rightIndex = leftIndex - sectionList.length;
				}

				// Create an array composed of left unmodified, modified, right
				// unmodified sections
				var leftSections = sectionList.slice(0, leftIndex);
				modifiedSections = newSectionList.slice(leftIndex, newSectionList.length + rightIndex);
				var rightSections = sectionList.slice(sectionList.length + rightIndex, sectionList.length);
				insertBeforeSection = rightSections[0];
				sectionsToRemove = sectionList.slice(leftIndex, sectionList.length + rightIndex);
				sectionList = leftSections.concat(modifiedSections).concat(rightSections);
			}

			if(this.isComposing) {
				return sectionList;
			}

			var newSectionEltList = document.createDocumentFragment();
			modifiedSections.forEach(function(section) {
				highlight(section);
				newSectionEltList.appendChild(section.elt);
			});
			editor.watcher.noWatch((function() {
				if(isInit) {
					contentElt.innerHTML = '';
					contentElt.appendChild(newSectionEltList);
				}
				else {
					// Remove outdated sections
					sectionsToRemove.forEach(function(section) {
						// section can be already removed
						section.elt.parentNode === contentElt && contentElt.removeChild(section.elt);
					});

					if(insertBeforeSection !== undefined) {
						contentElt.insertBefore(newSectionEltList, insertBeforeSection.elt);
					}
					else {
						contentElt.appendChild(newSectionEltList);
					}

					// Remove unauthorized nodes (text nodes outside of sections or duplicated sections via copy/paste)
					var childNode = contentElt.firstChild;
					while(childNode) {
						var nextNode = childNode.nextSibling;
						if(!childNode.generated) {
							contentElt.removeChild(childNode);
						}
						childNode = nextNode;
					}
				}
				this.addTrailingLfElt();
				editor.selectionMgr.updateSelectionRange();
				editor.selectionMgr.updateCursorCoordinates();
			}).bind(this));

			return sectionList;
		};
	}

	ced.Highlighter = Highlighter;

})(window.ced);
<commit_msg>Fixed auto focus on init<commit_after>(function(ced) {

	function Highlighter(editor) {
		var escape = ced.Utils.escape;

		function highlight(section) {
			var text = escape(section.text);
			text = ced.Prism.highlight(text, editor.options.language);
			text = text.replace(/\n/gm, '<span>\n</span>');
			/*
			 var frontMatter = section.textWithFrontMatter.substring(0, section.textWithFrontMatter.length - section.text.length);
			 if(frontMatter.length) {
			 // Front matter highlighting
			 frontMatter = escape(frontMatter);
			 frontMatter = frontMatter.replace(/\n/g, '<span class="token lf">\n</span>');
			 text = '<span class="token md">' + frontMatter + '</span>' + text;
			 }
			 */
			var sectionElt = document.createElement('span');
			sectionElt.id = 'classeur-editor-section-' + section.id;
			sectionElt.className = 'classeur-editor-section';
			sectionElt.generated = true;
			sectionElt.innerHTML = text;
			section.elt = sectionElt;
		}

		var contentElt = editor.$contentElt;
		var sectionCounter = 0;
		this.trailingLfElt;
		this.isComposing = 0;

		var sectionList = [];
		var sectionsToRemove = [];
		var modifiedSections = [];
		var insertBeforeSection;

		this.addTrailingLfElt = function() {
			this.trailingLfElt = document.createElement('span');
			this.trailingLfElt.textContent = '\n';
			contentElt.appendChild(this.trailingLfElt);
		};

		this.parseSections = function(content, isInit) {
			var tmpText = content + "\n\n";
			var newSectionList = [];
			var offset = 0;

			function addSection(startOffset, endOffset) {
				var sectionText = tmpText.substring(offset, endOffset);
				newSectionList.push({
					id: ++sectionCounter,
					text: sectionText
				});
			}

			// Look for delimiters
			editor.options.sectionDelimiter && tmpText.replace(editor.options.sectionDelimiter, function(match, matchOffset) {
				// Create a new section with the text preceding the delimiter
				addSection(offset, matchOffset);
				offset = matchOffset;
			});

			// Last section
			addSection(offset, content.length);

			modifiedSections = [];
			sectionsToRemove = [];
			insertBeforeSection = undefined;

			if(isInit) {
				// Render everything if isInit
				sectionsToRemove = sectionList;
				sectionList = newSectionList;
				modifiedSections = newSectionList;
			}
			else {
				// Find modified section starting from top
				var leftIndex = sectionList.length;
				sectionList.some(function(section, index) {
					var newSection = newSectionList[index];
					if(index >= newSectionList.length ||
							// Check text modification
						section.text != newSection.text ||
							// Check that section has not been detached or moved
						section.elt.parentNode !== contentElt ||
							// Check also the content since nodes can be injected in sections via copy/paste
						section.elt.textContent != newSection.text) {
						leftIndex = index;
						return true;
					}
				});

				// Find modified section starting from bottom
				var rightIndex = -sectionList.length;
				sectionList.slice().reverse().some(function(section, index) {
					var newSection = newSectionList[newSectionList.length - index - 1];
					if(index >= newSectionList.length ||
							// Check modified
						section.text != newSection.text ||
							// Check that section has not been detached or moved
						section.elt.parentNode !== contentElt ||
							// Check also the content since nodes can be injected in sections via copy/paste
						section.elt.textContent != newSection.text) {
						rightIndex = -index;
						return true;
					}
				});

				if(leftIndex - rightIndex > sectionList.length) {
					// Prevent overlap
					rightIndex = leftIndex - sectionList.length;
				}

				// Create an array composed of left unmodified, modified, right
				// unmodified sections
				var leftSections = sectionList.slice(0, leftIndex);
				modifiedSections = newSectionList.slice(leftIndex, newSectionList.length + rightIndex);
				var rightSections = sectionList.slice(sectionList.length + rightIndex, sectionList.length);
				insertBeforeSection = rightSections[0];
				sectionsToRemove = sectionList.slice(leftIndex, sectionList.length + rightIndex);
				sectionList = leftSections.concat(modifiedSections).concat(rightSections);
			}

			if(this.isComposing) {
				return sectionList;
			}

			var newSectionEltList = document.createDocumentFragment();
			modifiedSections.forEach(function(section) {
				highlight(section);
				newSectionEltList.appendChild(section.elt);
			});
			editor.watcher.noWatch((function() {
				if(isInit) {
					contentElt.innerHTML = '';
					contentElt.appendChild(newSectionEltList);
					return this.addTrailingLfElt();
				}

				// Remove outdated sections
				sectionsToRemove.forEach(function(section) {
					// section can be already removed
					section.elt.parentNode === contentElt && contentElt.removeChild(section.elt);
				});

				if(insertBeforeSection !== undefined) {
					contentElt.insertBefore(newSectionEltList, insertBeforeSection.elt);
				}
				else {
					contentElt.appendChild(newSectionEltList);
				}

				// Remove unauthorized nodes (text nodes outside of sections or duplicated sections via copy/paste)
				var childNode = contentElt.firstChild;
				while(childNode) {
					var nextNode = childNode.nextSibling;
					if(!childNode.generated) {
						contentElt.removeChild(childNode);
					}
					childNode = nextNode;
				}
				this.addTrailingLfElt();
				editor.selectionMgr.updateSelectionRange();
				editor.selectionMgr.updateCursorCoordinates();
			}).bind(this));

			return sectionList;
		};
	}

	ced.Highlighter = Highlighter;

})(window.ced);

{'fraction_non_alphanumeric': 0.1122956645344705, 'fraction_numerical': 0.0012437810945273632, 'mean_word_length': 3.314679954005366, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4707238', 'n_tokens_mistral': 4172, 'n_tokens_neox': 3220, 'n_words': 707}