<commit_before>require('shelljs/global')

var fs = require('fs-extra')
var ora = require('ora')
var slug = require('slug')
var path = require('path')
var colors = require('colors')
var mkdirp = require('mkdirp')
var jsonfile = require('jsonfile')
var parser = require('markdown-parse')
var spinner = ora('Compiling markdown files to json…\n')
var maxArticlesPerPage = 10

var folders = {
	'src': path.join('src', 'content'),
	'dist': path.join('static', 'posts')
}

function getDirectories(srcpath) {
	return fs.readdirSync(srcpath).filter(function(file) {
		return fs.statSync(path.join(srcpath, file)).isDirectory();
	});
}

function createJsonFiles(fileName, fileContent) {
	jsonfile.writeFile(fileName, fileContent)
	console.log('	+ Files: ' + fileName)
}

spinner.start()

mkdirp(folders.dist)

getDirectories(folders.src).forEach(function(directory) {
	var srcFolder = path.join(folders.src, directory)
	var distFolder = path.join(folders.dist, directory)

	// Create folder if don't exist
	mkdirp(distFolder)
	console.log(('+ Creating folder ' + distFolder).green)

	var files = fs.readdirSync(srcFolder, 'utf8')
	var fileContent = {'posts': []}
	var pageCount = 1
	var items = 0

	files.forEach(function(element) {
		var post = fs.readFileSync(path.join(srcFolder, element), 'utf8')
		var fileName = distFolder + '/page-' + pageCount + '.json'

		items++

		parser(post, function(err, result) {
			fileContent.posts.push({
				'title': result.attributes.title,
				'author': result.attributes.author || '<NAME>.',
				'date': result.attributes.date || new Date().format('Y-m-d h:i'),
				'tags': result.attributes.tags ||,
				'template': result.attributes.template || 'post.vue',
				'basename': result.attributes.basename || slug(result.attributes.title, { lower: true }),
				'content': result.html
			})
		})

		if (index === fileArray.length - 1) {
			createJsonFiles(fileName, fileContent)
		}

		else if (items === maxArticlesPerPage) {
			createJsonFiles(fileName, fileContent)
			pageCount++
			items = 0
			fileContent = {'posts': []}
		}
	})

})

spinner.stop()
<commit_msg>Better build pages<commit_after>require('shelljs/global')

var fs = require('fs-extra')
var ora = require('ora')
var slug = require('slug')
var path = require('path')
var colors = require('colors')
var mkdirp = require('mkdirp')
var jsonfile = require('jsonfile')
var parser = require('markdown-parse')
var spinner = ora('Compiling markdown files to json…\n')
var maxArticlesPerPage = 10

var folders = {
	'src': path.join('src', 'content'),
	'dist': path.join('static', 'posts')
}

function getDirectories(srcpath) {
	return fs.readdirSync(srcpath).filter(function(file) {
		return fs.statSync(path.join(srcpath, file)).isDirectory();
	});
}

function createJsonFiles(fileName, fileContent) {
	jsonfile.writeFile(fileName, fileContent)
	console.log('	+ Files: ' + fileName)
}

spinner.start()

mkdirp(folders.dist)

getDirectories(folders.src).forEach(function(directory) {
	var srcFolder = path.join(folders.src, directory)
	var distFolder = path.join(folders.dist, directory)

	// Create folder if don't exist
	mkdirp(distFolder)
	console.log(('+ Creating folder ' + distFolder).green)

	var files = fs.readdirSync(srcFolder, 'utf8')
	var fileContent = {'posts': []}
	var pageCount = 1
	var items = 0

	files.forEach(function(element, index, fileArray) {
		var post = fs.readFileSync(path.join(srcFolder, element), 'utf8')
		var fileName = distFolder + '/page-' + pageCount + '.json'

		items++

		parser(post, function(err, result) {
			fileContent.posts.push({
				'title': result.attributes.title,
				'author': result.attributes.author || '<NAME>.',
				'date': result.attributes.date || new Date().format('Y-m-d h:i'),
				'tags': result.attributes.tags || '',
				'template': result.attributes.template || 'post.vue',
				'basename': result.attributes.basename || slug(result.attributes.title, { lower: true }),
				'content': result.html
			})
		})

		if (index === fileArray.length - 1) {
			createJsonFiles(fileName, fileContent)
		}

		else if (items === maxArticlesPerPage) {
			createJsonFiles(fileName, fileContent)
			pageCount++
			items = 0
			fileContent = {'posts': []}
		}
	})

})

spinner.stop()

{'fraction_non_alphanumeric': 0.16429249762583095, 'fraction_numerical': 0.003798670465337132, 'mean_word_length': 4.662634408602151, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7015016', 'n_tokens_mistral': 1541, 'n_tokens_neox': 1421, 'n_words': 302}