<commit_before>/**
 * WordFilter.js
 *
 * Copyright, Moxiecode Systems AB
 * Released under LGPL License.
 *
 * License: http://www.tinymce.com/license
 * Contributing: http://www.tinymce.com/contributing
 */

/**
 * This class parses word HTML into proper TinyMCE markup.
 *
 * @class tinymce.pasteplugin.Quirks
 * @private
 */
define("tinymce/pasteplugin/WordFilter", [
	"tinymce/util/Tools",
	"tinymce/html/DomParser",
	"tinymce/html/Schema",
	"tinymce/html/Serializer",
	"tinymce/html/Node",
	"tinymce/pasteplugin/Utils"
], function(Tools, DomParser, Schema, Serializer, Node, Utils) {
	/**
	 * Checks if the specified content is from any of the following sources: MS Word/Office 365/Google docs.
	 */
	function isWordContent(content) {
		return (
			(/<font face="Times New Roman"|class="?Mso|style="[^"]*\bmso-|style='[^'']*\bmso-|w:WordDocument/i).test(content) ||
			(/class="OutlineElement/).test(content) ||
			(/id="?docs\-internal\-guid\-/.test(content))
		);
	}

	function WordFilter(editor) {
		var settings = editor.settings;

		editor.on('BeforePastePreProcess', function(e) {
			var content = e.content, retainStyleProperties, validStyles;

			retainStyleProperties = settings.paste_retain_style_properties;
			if (retainStyleProperties) {
				validStyles = Tools.makeMap(retainStyleProperties.split(/[, ]/));
			}

			/**
			 * Converts fake bullet and numbered lists to real semantic OL/UL.
			 *
			 * @param {tinymce.html.Node} node Root node to convert children of.
			 */
			function convertFakeListsToProperLists(node) {
				var currentListNode, prevListNode, lastLevel = 1;

				function getText(node) {
					var txt = '';

					if (node.type === 3) {
						return node.value;
					}

					if ((node = node.firstChild)) {
						do {
							txt += getText(node);
						} while ((node = node.next));
					}

					return txt;
				}

				function trimListStart(node, regExp) {
					if (node.type === 3) {
						if (regExp.test(node.value)) {
							node.value = node.value.replace(regExp, '');
							return false;
						}
					}

					if ((node = node.firstChild)) {
						do {
							if (!trimListStart(node, regExp)) {
								return false;
							}
						} while ((node = node.next));
					}

					return true;
				}

				function removeIgnoredNodes(node) {
					if (node._listIgnore) {
						node.remove();
						return;
					}

					if ((node = node.firstChild)) {
						do {
							removeIgnoredNodes(node);
						} while ((node = node.next));
					}
				}

				function convertParagraphToLi(paragraphNode, listName, start) {
					var level = paragraphNode._listLevel || lastLevel;

					// Handle list nesting
					if (level != lastLevel) {
						if (level < lastLevel) {
							// Move to parent list
							if (currentListNode) {
								currentListNode = currentListNode.parent.parent;
							}
						} else {
							// Create new list
							prevListNode = currentListNode;
							currentListNode = null;
						}
					}

					if (!currentListNode || currentListNode.name != listName) {
						prevListNode = prevListNode || currentListNode;
						currentListNode = new Node(listName, 1);

						if (start > 1) {
							currentListNode.attr('start', '' + start);
						}

						paragraphNode.wrap(currentListNode);
					} else {
						currentListNode.append(paragraphNode);
					}

					paragraphNode.name = 'li';

					// Append list to previous list if it exists
					if (level > lastLevel && prevListNode) {
						prevListNode.lastChild.append(currentListNode);
					}

					lastLevel = level;

					// Remove start of list item "1. " or "&middot; " etc
					removeIgnoredNodes(paragraphNode);
					trimListStart(paragraphNode, /^\u00a0+/);
					trimListStart(paragraphNode, /^\s*([\u2022\u00b7\u00a7\u00d8\u25CF]|\w+\.)/);
					trimListStart(paragraphNode, /^\u00a0+/);
				}

				var paragraphs = node.getAll('p');

				for (var i = 0; i < paragraphs.length; i++) {
					node = paragraphs[i];

					if (node.name == 'p' && node.firstChild) {
						// Find first text node in paragraph
						var nodeText = getText(node);
						var listStartTextNode = node.firstChild;

						// Detect unordered lists look for bullets
						if (/^\s*[\u2022\u00b7\u00a7\u00d8\u25CF]\s*/.test(nodeText)) {
							convertParagraphToLi(node, 'ul');
							continue;
						}

						// Detect ordered lists 1., a. or ixv.
						if (/^\s*\w+\./.test(nodeText) && !/^\s*\w+\.\s*[^\s]+/.test(listStartTextNode.value)) {
							// Parse OL start number
							var matches = /([0-9])\./.exec(nodeText);
							var start = 1;
							if (matches) {
								start = parseInt(matches[1], 10);
							}

							convertParagraphToLi(node, 'ol', start);
							continue;
						}

						// Convert paragraphs marked as lists but doesn't look like anything
						if (node._listLevel) {
							convertParagraphToLi(node, 'ul', 1);
							continue;
						}

						currentListNode = null;
					}
				}<commit_msg>Fixed issue with list detection in paste plugin on older IEs.<commit_after>/**
 * WordFilter.js
 *
 * Copyright, Moxiecode Systems AB
 * Released under LGPL License.
 *
 * License: http://www.tinymce.com/license
 * Contributing: http://www.tinymce.com/contributing
 */

/**
 * This class parses word HTML into proper TinyMCE markup.
 *
 * @class tinymce.pasteplugin.Quirks
 * @private
 */
define("tinymce/pasteplugin/WordFilter", [
	"tinymce/util/Tools",
	"tinymce/html/DomParser",
	"tinymce/html/Schema",
	"tinymce/html/Serializer",
	"tinymce/html/Node",
	"tinymce/pasteplugin/Utils"
], function(Tools, DomParser, Schema, Serializer, Node, Utils) {
	/**
	 * Checks if the specified content is from any of the following sources: MS Word/Office 365/Google docs.
	 */
	function isWordContent(content) {
		return (
			(/<font face="Times New Roman"|class="?Mso|style="[^"]*\bmso-|style='[^'']*\bmso-|w:WordDocument/i).test(content) ||
			(/class="OutlineElement/).test(content) ||
			(/id="?docs\-internal\-guid\-/.test(content))
		);
	}

	function WordFilter(editor) {
		var settings = editor.settings;

		editor.on('BeforePastePreProcess', function(e) {
			var content = e.content, retainStyleProperties, validStyles;

			retainStyleProperties = settings.paste_retain_style_properties;
			if (retainStyleProperties) {
				validStyles = Tools.makeMap(retainStyleProperties.split(/[, ]/));
			}

			/**
			 * Converts fake bullet and numbered lists to real semantic OL/UL.
			 *
			 * @param {tinymce.html.Node} node Root node to convert children of.
			 */
			function convertFakeListsToProperLists(node) {
				var currentListNode, prevListNode, lastLevel = 1;

				function getText(node) {
					var txt = '';

					if (node.type === 3) {
						return node.value;
					}

					if ((node = node.firstChild)) {
						do {
							txt += getText(node);
						} while ((node = node.next));
					}

					return txt;
				}

				function trimListStart(node, regExp) {
					if (node.type === 3) {
						if (regExp.test(node.value)) {
							node.value = node.value.replace(regExp, '');
							return false;
						}
					}

					if ((node = node.firstChild)) {
						do {
							if (!trimListStart(node, regExp)) {
								return false;
							}
						} while ((node = node.next));
					}

					return true;
				}

				function removeIgnoredNodes(node) {
					if (node._listIgnore) {
						node.remove();
						return;
					}

					if ((node = node.firstChild)) {
						do {
							removeIgnoredNodes(node);
						} while ((node = node.next));
					}
				}

				function convertParagraphToLi(paragraphNode, listName, start) {
					var level = paragraphNode._listLevel || lastLevel;

					// Handle list nesting
					if (level != lastLevel) {
						if (level < lastLevel) {
							// Move to parent list
							if (currentListNode) {
								currentListNode = currentListNode.parent.parent;
							}
						} else {
							// Create new list
							prevListNode = currentListNode;
							currentListNode = null;
						}
					}

					if (!currentListNode || currentListNode.name != listName) {
						prevListNode = prevListNode || currentListNode;
						currentListNode = new Node(listName, 1);

						if (start > 1) {
							currentListNode.attr('start', '' + start);
						}

						paragraphNode.wrap(currentListNode);
					} else {
						currentListNode.append(paragraphNode);
					}

					paragraphNode.name = 'li';

					// Append list to previous list if it exists
					if (level > lastLevel && prevListNode) {
						prevListNode.lastChild.append(currentListNode);
					}

					lastLevel = level;

					// Remove start of list item "1. " or "&middot; " etc
					removeIgnoredNodes(paragraphNode);
					trimListStart(paragraphNode, /^\u00a0+/);
					trimListStart(paragraphNode, /^\s*([\u2022\u00b7\u00a7\u00d8\u25CF]|\w+\.)/);
					trimListStart(paragraphNode, /^\u00a0+/);
				}

				var paragraphs = node.getAll('p');

				for (var i = 0; i < paragraphs.length; i++) {
					node = paragraphs[i];

					if (node.name == 'p' && node.firstChild) {
						// Find first text node in paragraph
						var nodeText = getText(node);
						var listStartTextNode = node.firstChild;

						// Detect unordered lists look for bullets
						if (/^[\s\u00a0]*[\u2022\u00b7\u00a7\u00d8\u25CF]\s*/.test(nodeText)) {
							convertParagraphToLi(node, 'ul');
							continue;
						}

						// Detect ordered lists 1., a. or ixv.
						if (/^[\s\u00a0]*\w+\./.test(nodeText) && !/^[\s\u00a0]*\w+\.\s*[^\s]+/.test(listStartTextNode.value)) {
							// Parse OL start number
							var matches = /([0-9])\./.exec(nodeText);
							var start = 1;
							if (matches) {
								start = parseInt(matches[1], 10);
							}

							convertParagraphToLi(node, 'ol', start);
							continue;
						}

						// Convert paragraphs marked as lists but doesn't look like anything
						if (node._listLevel) {
							convertParagraphToLi(node, 'ul', 1);
							continue;
						}

						currentListNode = null;
					}
				}
{'fraction_non_alphanumeric': 0.14355034371209058, 'fraction_numerical': 0.011827739587545492, 'mean_word_length': 2.501946902654867, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2971278', 'n_tokens_mistral': 4449, 'n_tokens_neox': 3170, 'n_words': 688}