<commit_before>import lowlight from 'lowlight/lib/core';
import React from 'react';

const createTextElement = (text, key) => <span key={key}>{text.value}</span>;

const createStyleObject = (classNames, style) => {
	return classNames.reduce((styleObject, className) => {
		return {...styleObject, ...style[className]};
	}, {});
}

function createChildren(style) {
	let childrenCount = 0;
	return children => {
		childrenCount += 1;
		return children.map((child, i) => createElement(child, style, `code-segement-${childrenCount}-${i}`));
	}
}
function createElement(node, style, key) {
	if (node.type === "text") {
		return createTextElement(node, key);
	}
	else if (node.tagName) {
		const TagName = node.tagName;
		const { properties } = node;
		const childrenCreator = createChildren(style);
		const nodeStyle = createStyleObject(properties.className, style);
		const children = childrenCreator(node.children);
		return <TagName key={key} style={nodeStyle}>{children}</TagName>;
	}
}

export default function SyntaxHighlighter(props) {
	const {language, children, stylesheet = 'default'} = props;
	const style = require(`./styles/${stylesheet}`).default;
	const lang = require(`highlight.js/lib/languages/${language}`);
	lowlight.registerLanguage(language, lang);
	const codeTree = lowlight.highlight(language, children);

	return <pre {...props}>{codeTree.value.map((node, i) => createElement(node, style, `code-segement${i}`))}</pre>;
}<commit_msg>wrap in code element as well<commit_after>import lowlight from 'lowlight/lib/core';
import React from 'react';

const createTextElement = (text, key) => <span key={key}>{text.value}</span>;

const createStyleObject = (classNames, style) => {
	return classNames.reduce((styleObject, className) => {
		return {...styleObject, ...style[className]};
	}, {});
}

function createChildren(style) {
	let childrenCount = 0;
	return children => {
		childrenCount += 1;
		return children.map((child, i) => createElement(child, style, `code-segement-${childrenCount}-${i}`));
	}
}
function createElement(node, style, key) {
	if (node.type === "text") {
		return createTextElement(node, key);
	}
	else if (node.tagName) {
		const TagName = node.tagName;
		const { properties } = node;
		const childrenCreator = createChildren(style);
		const nodeStyle = createStyleObject(properties.className, style);
		const children = childrenCreator(node.children);
		return <TagName key={key} style={nodeStyle}>{children}</TagName>;
	}
}

export default function SyntaxHighlighter(props) {
	const {language, children, stylesheet = 'default'} = props;
	const style = require(`./styles/${stylesheet}`).default;
	const lang = require(`highlight.js/lib/languages/${language}`);
	lowlight.registerLanguage(language, lang);
	const codeTree = lowlight.highlight(language, children);

	return (
		<pre {...props}>
			<code>
				{codeTree.value.map((node, i) => createElement(node, style, `code-segement${i}`))}
			</code>
		</pre>
	);
}
{'fraction_non_alphanumeric': 0.16683587140439932, 'fraction_numerical': 0.0013536379018612521, 'mean_word_length': 5.370689655172414, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2676279', 'n_tokens_mistral': 952, 'n_tokens_neox': 923, 'n_words': 211}