<commit_before>const frontmatter = require('front-matter');
const markdown = require('../utils/markdown');
const highlight = require('../utils/highlight');

module.exports = function (source) {
  const result = frontmatter(source);

  result.attributes = result.attributes || {};
  result.body = markdown().process(result.body, highlight);

  return `module.exports = ${JSON.stringify(result)};`;
};<commit_msg>chore: Sketch out more data generators for a page loader<commit_after>const _ = require('lodash');
const frontmatter = require('front-matter');
const markdown = require('../utils/markdown');
const highlight = require('../utils/highlight');

module.exports = function (source) {
  const result = frontmatter(source);

  result.attributes = result.attributes || {};
  result.body = markdown().process(result.body, highlight);
  result.preview = generatePreview(result);
  result.description = generateDescription(result);
  result.keywords = generateKeywords(result);

  return `module.exports = ${JSON.stringify(result)};`;
};

function generatePreview(file) {
  if (file.attributes && file.attributes.preview) {
    return file.attributes.preview;
  }

  return file.body && file.body.slice(0, 100) + '…';
}

function generateDescription(file) {
  if (file.attributes) {
    return file.attributes.description || file.attributes.preview;
  }

  return file.preview;
}

function generateKeywords(file) {
  let keywords;

  if (file.attributes && file.attributes.keywords) {
    keywords = file.attributes.keywords;
  }

  if (_.isString(keywords)) {
    return keywords.split(',');
  }

  return keywords;
}
{'fraction_non_alphanumeric': 0.15089672232529375, 'fraction_numerical': 0.0024737167594310453, 'mean_word_length': 4.758007117437723, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4010522', 'n_tokens_mistral': 481, 'n_tokens_neox': 452, 'n_words': 99}