<commit_before>var _ = require('lodash');
var hljs = require('highlight.js');
var createRenderer = require('../src/utils/markdown.js');

var md = createRenderer();

var evalPlaceholder = '<%{#eval#}%>';
var codePlaceholder = '<%{#code#}%>';

// Need to supply the regex test as a string for reuse in unit tests
// Currently, trying to change flags throws a TypeError
// Slated for change in ES6, but not possible now:
// https://developer.mozilla.org/en/docs/Web/JavaScript/Reference/Global_Objects/RegExp#Description
var requireAnythingTest = 'require\\s*\\(([^)]+)\\)';
var requireAnythingRegex = new RegExp(requireAnythingTest, 'g');
var simpleStringRegex = /^"([^"]+)"$|^'([^']+)'$/;

function readExamples(markdown) {
	var codeChunks = [];

	// Collect code blocks and replace them with placeholders
	md.renderer.rules.code_block = md.renderer.rules.fence = function(tokens, idx) {
		var token = tokens[idx];
		var code = tokens[idx].content.trim();
		if (token.type === 'fence' && token.info) {
			// Render fenced blocks with language flag as regular Markdown code snippets
			var highlighted;
			try {
				highlighted = hljs.highlight(token.info, code).value;
			}
			catch(e) {
				highlighted = e.message;
			}
			return '```' + token.info + '\n' + highlighted + '\n```';
		}
		codeChunks.push(code);
		return codePlaceholder;
	};

	var rendered = md.render(markdown);

	var chunks = [];
	var textChunks = rendered.split(codePlaceholder);
	textChunks.forEach(function(chunk) {
		if (chunk) {
			chunks.push({type: 'markdown', content: chunk});
		}
		var code = codeChunks.shift();
		if (code) {
			chunks.push({type: 'code', content: code, evalInContext: evalPlaceholder});
		}
	});<commit_msg>Fix code style.<commit_after>var _ = require('lodash');
var hljs = require('highlight.js');
var createRenderer = require('../src/utils/markdown.js');

var md = createRenderer();

var evalPlaceholder = '<%{#eval#}%>';
var codePlaceholder = '<%{#code#}%>';

// Need to supply the regex test as a string for reuse in unit tests
// Currently, trying to change flags throws a TypeError
// Slated for change in ES6, but not possible now:
// https://developer.mozilla.org/en/docs/Web/JavaScript/Reference/Global_Objects/RegExp#Description
var requireAnythingTest = 'require\\s*\\(([^)]+)\\)';
var requireAnythingRegex = new RegExp(requireAnythingTest, 'g');
var simpleStringRegex = /^"([^"]+)"$|^'([^']+)'$/;

function readExamples(markdown) {
	var codeChunks = [];

	// Collect code blocks and replace them with placeholders
	md.renderer.rules.code_block = md.renderer.rules.fence = function(tokens, idx) {
		var token = tokens[idx];
		var code = tokens[idx].content.trim();
		if (token.type === 'fence' && token.info) {
			// Render fenced blocks with language flag as regular Markdown code snippets
			var highlighted;
			try {
				highlighted = hljs.highlight(token.info, code).value;
			}
			catch (e) {
				highlighted = e.message;
			}
			return '```' + token.info + '\n' + highlighted + '\n```';
		}
		codeChunks.push(code);
		return codePlaceholder;
	};

	var rendered = md.render(markdown);

	var chunks = [];
	var textChunks = rendered.split(codePlaceholder);
	textChunks.forEach(function(chunk) {
		if (chunk) {
			chunks.push({type: 'markdown', content: chunk});
		}
		var code = codeChunks.shift();
		if (code) {
			chunks.push({type: 'code', content: code, evalInContext: evalPlaceholder});
		}
	});
{'fraction_non_alphanumeric': 0.167985927880387, 'fraction_numerical': 0.0005863383172090296, 'mean_word_length': 4.209160305343511, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2457507', 'n_tokens_mistral': 1207, 'n_tokens_neox': 1084, 'n_words': 281}