<commit_before>import fs from 'fs';
import path from 'path';

import _ from 'lodash';
import frontMatter from 'front-matter';
import markdownIt from 'markdown-it';

const md = markdownIt({
  html: true,
  linkify: true,
  typographer: false,
});

export default function loadPosts(providedOptions) {
  const options = providedOptions || {};

  const limit = options.limit || Infinity;
  const markdown = typeof options.markdown === 'undefined' ? true : options.markdown;

  const postsPath = path.join(__dirname, '../../pages/posts');
  const postFiles = fs.readdirSync(postsPath);

  return _.chain(postFiles)
    .sortBy()
    .reverse() // eslint-disable-line
    .take(limit)
    .map(file => {
      const filePath = path.join(postsPath, file);
      const contents = fs.readFileSync(filePath).toString();
      const metadata = frontMatter(contents);

      return {
        path: filePath,
        contents,
        body: markdown ? md.render(metadata.body) : null,
        data: metadata.attributes,
      };
    })
    .value();<commit_msg>fix(loadPosts): exclude license.txt from loadPosts<commit_after>import fs from 'fs';
import path from 'path';

import _ from 'lodash';
import frontMatter from 'front-matter';
import markdownIt from 'markdown-it';

const md = markdownIt({
  html: true,
  linkify: true,
  typographer: false,
});
const fileFilter = /.md$/;

export default function loadPosts(providedOptions) {
  const options = providedOptions || {};

  const limit = options.limit || Infinity;
  const markdown = typeof options.markdown === 'undefined' ? true : options.markdown;

  const postsPath = path.join(__dirname, '../../pages/posts');
  const postFiles = fs.readdirSync(postsPath);

  return _.chain(postFiles)
    .filter(file => fileFilter.test(file))
    .sortBy()
    .reverse() // eslint-disable-line
    .take(limit)
    .map(file => {
      const filePath = path.join(postsPath, file);
      const contents = fs.readFileSync(filePath).toString();
      const metadata = frontMatter(contents);

      return {
        path: filePath,
        contents,
        body: markdown ? md.render(metadata.body) : null,
        data: metadata.attributes,
      };
    })
    .value();
{'fraction_non_alphanumeric': 0.13424036281179139, 'fraction_numerical': 0.0, 'mean_word_length': 3.2751937984496124, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3679622', 'n_tokens_mistral': 671, 'n_tokens_neox': 647, 'n_words': 170}