<commit_before>import fs from 'fs';
import path from 'path';
import glob from 'glob';
import yaml from 'js-yaml';
import fastmatter from 'fastmatter';
import _ from 'lodash';

import renderMarkdown from './markdown';

export function removeExtension(filename) {
	return filename.replace(/\.\w+$/, '');
}

export function filepathToUrl(filename) {
	return '/' + removeExtension(filename);
}

export function parsePage(source, filepath) {
	let { attributes, body } = fastmatter(source);
	let extension = path.extname(filepath);
	if (extension === '.md') {
		body = renderMarkdown(body);
	}
	return _.merge(attributes, {
		sourcePath: filepath,
		url: filepathToUrl(filepath),
		content: body
	});
}

export function getSourceFilesList(folder) {
	return glob.sync(path.join(folder, '**/*.{md,html}'));
}

export function readFile(filepath) {
	return fs.readFileSync(filepath, {encoding: 'utf8'});
}

export function readYamlFile(filepath) {
	try {
		return yaml.safeLoad(readFile(filepath));
	}
	catch (e) {
		console.log(`Cannot read YAML file ${filepath}:`, e);
	}
}

export function readFiles(files) {
	return files.map(readFile);
}

export function getConfigFilesList(folder) {
	return glob.sync(path.join(folder, '*.yml'));
}

export function readConfigFiles(files) {
	return files.reduce((configs, filepath) => {
		let name = removeExtension(path.basename(filepath));
		if (name === 'default') {
			configs.default = readYamlFile(filepath);
		}
		else {
			configs.langs[name] = readYamlFile(filepath);
		}
		return configs;
	}, {default: {}, langs: {}});  // @todo use really default config
}

export function mergeConfigs(configs) {
	let { langs } = configs;
	if (_.isEmpty(langs)) {
		return {
			default: configs.default
		};
	}

	return Object.keys(langs).reduce((merged, lang) => {
		merged[lang] = _.merge({}, configs.default, langs[lang]);
		return merged;
	}, {});
}

export function readConfig(folder) {
	let files = getConfigFilesList(folder);
	let configs = readConfigFiles(files);
	return mergeConfigs(configs);
}

export function makeContext(page, config, helpers) {
	// Combine page data and config
	let context = _.merge({}, { config }, page);

	// Bind all helpers to context
	return Object.keys(helpers).reduce((ctx, name) => {
		ctx[name] = helpers[name].bind(ctx);
		return ctx;
	}, context);
}<commit_msg>Simplify context creation.<commit_after>import fs from 'fs';
import path from 'path';
import glob from 'glob';
import yaml from 'js-yaml';
import fastmatter from 'fastmatter';
import _ from 'lodash';

import renderMarkdown from './markdown';

export function removeExtension(filename) {
	return filename.replace(/\.\w+$/, '');
}

export function filepathToUrl(filename) {
	return '/' + removeExtension(filename);
}

export function parsePage(source, filepath) {
	let { attributes, body } = fastmatter(source);
	let extension = path.extname(filepath);
	if (extension === '.md') {
		body = renderMarkdown(body);
	}
	return _.merge(attributes, {
		sourcePath: filepath,
		url: filepathToUrl(filepath),
		content: body
	});
}

export function getSourceFilesList(folder) {
	return glob.sync(path.join(folder, '**/*.{md,html}'));
}

export function readFile(filepath) {
	return fs.readFileSync(filepath, {encoding: 'utf8'});
}

export function readYamlFile(filepath) {
	try {
		return yaml.safeLoad(readFile(filepath));
	}
	catch (e) {
		console.log(`Cannot read YAML file ${filepath}:`, e);
	}
}

export function readFiles(files) {
	return files.map(readFile);
}

export function getConfigFilesList(folder) {
	return glob.sync(path.join(folder, '*.yml'));
}

export function readConfigFiles(files) {
	return files.reduce((configs, filepath) => {
		let name = removeExtension(path.basename(filepath));
		if (name === 'default') {
			configs.default = readYamlFile(filepath);
		}
		else {
			configs.langs[name] = readYamlFile(filepath);
		}
		return configs;
	}, {default: {}, langs: {}});  // @todo use really default config
}

export function mergeConfigs(configs) {
	let { langs } = configs;
	if (_.isEmpty(langs)) {
		return {
			default: configs.default
		};
	}

	return Object.keys(langs).reduce((merged, lang) => {
		merged[lang] = _.merge({}, configs.default, langs[lang]);
		return merged;
	}, {});
}

export function readConfig(folder) {
	let files = getConfigFilesList(folder);
	let configs = readConfigFiles(files);
	return mergeConfigs(configs);
}

export function makeContext(page, config, helpers) {
	return _.merge({}, helpers, { config }, page);
}
{'fraction_non_alphanumeric': 0.15422331178961443, 'fraction_numerical': 0.0004457321149988857, 'mean_word_length': 4.913043478260869, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3039577', 'n_tokens_mistral': 1561, 'n_tokens_neox': 1484, 'n_words': 352}