<commit_before>angular.module('classeur.extensions.markdown', [])
	.directive('clMarkdown',
		function($window, clEditorSvc) {
			var options = {};
			var coreBaseRules = [
					'normalize',
					'block',
					'inline',
					'linkify',
					'replacements',
					'smartquotes',
				],
				blockBaseRules = [
					'code',
					'blockquote',
					'hr',
					'list',
					'reference',
					'heading',
					'lheading',
					'html_block',
					'paragraph',
				],
				inlineBaseRules = [
					'text',
					'newline',
					'escape',
					'backticks',
					'strikethrough',
					'emphasis',
					'link',
					'image',
					'autolink',
					'html_inline',
					'entity',
				],
				blockRules = [
					'fence',
					'table'
				];

			clEditorSvc.onMarkdownInit(0, function(markdown) {
				markdown.set({
					html: true,
					breaks: !!options.breaks,
					linkify: !!options.linkify,
					typographer: !!options.typographer,
					langPrefix: 'prism language-'
				});

				markdown.core.ruler.enable(coreBaseRules);
				markdown.block.ruler.enable(options.cl_reduce(function(rules, value, key) {
					return rules.concat(value && blockRules.indexOf(key) !== -1 ? key : []);
				}, blockBaseRules));
				markdown.inline.ruler.enable(inlineBaseRules);
				options.abbr && markdown.use($window.markdownitAbbr);
				options.deflist && markdown.use($window.markdownitDeflist);
				options.footnote && markdown.use($window.markdownitFootnote);
				options.sub && markdown.use($window.markdownitSub);
				options.sup && markdown.use($window.markdownitSup);

				markdown.core.ruler.before('replacements', 'anchors', function(state) {
					var anchorHash = {};
					var headingOpenToken, headingContent;
					state.tokens.cl_each(function(token) {
						if (token.type === 'heading_open') {
							headingContent = '';
							headingOpenToken = token;
						} else if (token.type === 'heading_close') {
							headingOpenToken.headingContent = headingContent;

							// Slugify according to http://pandoc.org/README.html#extension-auto_identifiers
							var slug = headingContent
								.replace(/\s/g, '-') // Replace all spaces and newlines with hyphens
								.replace(/[\0-,\/:-@[-^`{-~]/g, '') // Remove all punctuation, except underscores, hyphens, and periods
								.toLowerCase(); // Convert all alphabetic characters to lowercase

							// Remove everything up to the first letter
							for (var i = 0; i < slug.length; i++) {
								var charCode = slug.charCodeAt(i);
								if ((charCode >= 0x61 && charCode <= 0x7A) || charCode > 0x7E) {
									break;
								}
							}
							
							// If nothing is left after this, use the identifier `section`
							slug = slug.slice(i) || 'section';

							var anchor = slug;
							var index = 1;
							while (anchorHash.hasOwnProperty(anchor)) {
								anchor = slug + '-' + (index++);
							}
							anchorHash[anchor] = true;
							headingOpenToken.headingAnchor = anchor;
							headingOpenToken = undefined;
						} else if (headingOpenToken) {
							headingContent += token.children.cl_reduce(function(result, child) {
								if (child.type !== 'footnote_ref') {
									result += child.content;
								}
								return result;
							}, '');
						}
					});
				});

				var originalHeadingOpen = markdown.renderer.rules.heading_open;
				markdown.renderer.rules.heading_open = function(tokens, idx) {
					var token = tokens[idx];
					(token.attrs = token.attrs || []).push(['id', token.headingAnchor]);
					if (originalHeadingOpen) {
						return originalHeadingOpen.apply(this, arguments);
					} else {
						return markdown.renderer.renderToken.apply(markdown.renderer, arguments);
					}
				};<commit_msg>Transform table style into align attribute. Fixes #25<commit_after>angular.module('classeur.extensions.markdown', [])
	.directive('clMarkdown',
		function($window, clEditorSvc) {
			var options = {};
			var coreBaseRules = [
					'normalize',
					'block',
					'inline',
					'linkify',
					'replacements',
					'smartquotes',
				],
				blockBaseRules = [
					'code',
					'blockquote',
					'hr',
					'list',
					'reference',
					'heading',
					'lheading',
					'html_block',
					'paragraph',
				],
				inlineBaseRules = [
					'text',
					'newline',
					'escape',
					'backticks',
					'strikethrough',
					'emphasis',
					'link',
					'image',
					'autolink',
					'html_inline',
					'entity',
				],
				blockRules = [
					'fence',
					'table'
				];

			clEditorSvc.onMarkdownInit(0, function(markdown) {
				markdown.set({
					html: true,
					breaks: !!options.breaks,
					linkify: !!options.linkify,
					typographer: !!options.typographer,
					langPrefix: 'prism language-'
				});

				markdown.core.ruler.enable(coreBaseRules);
				markdown.block.ruler.enable(options.cl_reduce(function(rules, value, key) {
					return rules.concat(value && blockRules.indexOf(key) !== -1 ? key : []);
				}, blockBaseRules));
				markdown.inline.ruler.enable(inlineBaseRules);
				options.abbr && markdown.use($window.markdownitAbbr);
				options.deflist && markdown.use($window.markdownitDeflist);
				options.footnote && markdown.use($window.markdownitFootnote);
				options.sub && markdown.use($window.markdownitSub);
				options.sup && markdown.use($window.markdownitSup);

				markdown.core.ruler.before('replacements', 'anchors', function(state) {
					var anchorHash = {};
					var headingOpenToken, headingContent;
					state.tokens.cl_each(function(token) {
						if (token.type === 'heading_open') {
							headingContent = '';
							headingOpenToken = token;
						} else if (token.type === 'heading_close') {
							headingOpenToken.headingContent = headingContent;

							// Slugify according to http://pandoc.org/README.html#extension-auto_identifiers
							var slug = headingContent
								.replace(/\s/g, '-') // Replace all spaces and newlines with hyphens
								.replace(/[\0-,\/:-@[-^`{-~]/g, '') // Remove all punctuation, except underscores, hyphens, and periods
								.toLowerCase(); // Convert all alphabetic characters to lowercase

							// Remove everything up to the first letter
							for (var i = 0; i < slug.length; i++) {
								var charCode = slug.charCodeAt(i);
								if ((charCode >= 0x61 && charCode <= 0x7A) || charCode > 0x7E) {
									break;
								}
							}
							
							// If nothing is left after this, use the identifier `section`
							slug = slug.slice(i) || 'section';

							var anchor = slug;
							var index = 1;
							while (anchorHash.hasOwnProperty(anchor)) {
								anchor = slug + '-' + (index++);
							}
							anchorHash[anchor] = true;
							headingOpenToken.headingAnchor = anchor;
							headingOpenToken = undefined;
						} else if (headingOpenToken) {
							headingContent += token.children.cl_reduce(function(result, child) {
								if (child.type !== 'footnote_ref') {
									result += child.content;
								}
								return result;
							}, '');
						}
					});
				});

				var originalHeadingOpen = markdown.renderer.rules.heading_open;
				markdown.renderer.rules.heading_open = function(tokens, idx, options) {
					var token = tokens[idx];
					(token.attrs = token.attrs || []).push(['id', token.headingAnchor]);
					if (originalHeadingOpen) {
						return originalHeadingOpen.call(this, tokens, idx, options);
					} else {
						return markdown.renderer.renderToken.call(markdown.renderer, tokens, idx, options);
					}
				};

				// Transform style into align attribute to pass the HTML sanitizer
				var textAlignLength = 'text-align:'.length;
				markdown.renderer.rules.th_open = markdown.renderer.rules.td_open = function(tokens, idx, options) {
					var token = tokens[idx];
					if(token.attrs) {
						token.attrs = [ [ 'align', token.attrs[0][1].slice(textAlignLength) ] ];
					}
					return markdown.renderer.renderToken.call(markdown.renderer, tokens, idx, options);
				};
{'fraction_non_alphanumeric': 0.1449385531483593, 'fraction_numerical': 0.003547447105029773, 'mean_word_length': 2.649560795191863, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '956181', 'n_tokens_mistral': 3455, 'n_tokens_neox': 2395, 'n_words': 474}