<commit_before>'use strict';
var _ = require('lodash');
var removeMd = require('remove-markdown');

var themeFunctions = require('theme').functions || {};

var MdHelper = require('./elements/MdHelper');
var itemHooks = require('./itemHooks');
var config = require('config');
var siteFunctions = config.functions || {} ;

function allItems() {
  var items = [].concat.apply([], _.keys(config.paths).map(function(sectionName) {
    var section = config.paths[sectionName];

    var paths = parseModules(sectionName, section, section.path());

    var draftPaths = [];
    if(__DEV__ && section.draft) {
      draftPaths = parseModules(sectionName, section, section.draft()).map(function(module) {
        module.isDraft = true;

        return module;
      });
    }

    return (section.sort || defaultSort)(paths.concat(draftPaths));
  }));

  items = itemHooks.preProcessItems(items);
  items = _.map(items, function(o) {
    var item = processItem(o.file, o.url, o.name, o.sectionName, o.section);

    item.isDraft = o.isDraft;

    return item;
  });
  items = itemHooks.postProcessItems(items);

  // TODO: can this conversion be avoided?
  var ret = {};

  _.each(items, function(o) {
    ret[o.url] = o;
  });

  return ret;
}
exports.allItems = allItems;

function defaultSort(files) {
    return _.sortBy(files, 'date').reverse();
}

function parseModules(sectionName, section, modules) {
  return _.map(modules.keys(), function(name) {
    return {
      name: name.slice(2),
      file: modules(name),
      section: section,
      sectionName: sectionName === '/' ? '' : sectionName,
    };
  });
}

function itemForPath(path) {
  var items = allItems();

  // this check is needed for root paths to work (they have extra slash!)
  return items[path] || items['/' + path];
}
exports.itemForPath = itemForPath;

function renderContent(content) {
  return MdHelper.render(content);
}
exports.renderContent = renderContent;

function processItem(o, url, fileName, sectionName, section) {

  var layout = section.layout;

  var sectionFunctions = section.processItem || {};

  var functions = _.assign({
    date: function(file, fileName, sectionName) {
      return file.date || null;
    },
    content: function(file, fileName, sectionName) {
      return MdHelper.render(file.__content);
    },
    preview: function(file, fileName, sectionName) {
      if (file.preview) {
        return file.preview;
      }
      else {
        var previewLimit = 100;

        if(!file.content) {
          return;
        }

        var stripped = removeMd(file.__content);

        if (stripped.length > previewLimit) {
          return stripped.substr(0, previewLimit) + '…';
        }

        return stripped;
      }
      return file.preview || MdHelper.getContentPreview(file.__content);
    },
    title: function(file, fileName, sectionName) {
      return file.title;
    },
    url: function(file, fileName, sectionName) {
      return sectionName + '/' + fileName.split('.')[0];
    },
    layout: function(file, fileName, sectionName) {
      return layout;
    },
  }, themeFunctions, siteFunctions, sectionFunctions);

  _.forEach(functions, function(fn, name) {
    o[name] = fn(o, fileName, sectionName);
  });

  o.section = sectionName;

  return o;
}
<commit_msg>Simplify section hook parameter convention<commit_after>'use strict';
var _ = require('lodash');
var removeMd = require('remove-markdown');

var themeFunctions = require('theme').functions || {};

var MdHelper = require('./elements/MdHelper');
var itemHooks = require('./itemHooks');
var config = require('config');
var siteFunctions = config.functions || {} ;

function allItems() {
  var items = [].concat.apply([], _.keys(config.paths).map(function(sectionName) {
    var section = config.paths[sectionName];

    var paths = parseModules(sectionName, section, section.path());

    var draftPaths = [];
    if(__DEV__ && section.draft) {
      draftPaths = parseModules(sectionName, section, section.draft()).map(function(module) {
        module.isDraft = true;

        return module;
      });
    }

    return (section.sort || defaultSort)(paths.concat(draftPaths));
  }));

  items = itemHooks.preProcessItems(items);
  items = _.map(items, function(o) {
    var item = processItem(o.file, o.url, o.name, o.sectionName, o.section);

    item.isDraft = o.isDraft;

    return item;
  });
  items = itemHooks.postProcessItems(items);

  // TODO: can this conversion be avoided?
  var ret = {};

  _.each(items, function(o) {
    ret[o.url] = o;
  });

  return ret;
}
exports.allItems = allItems;

function defaultSort(files) {
    return _.sortBy(files, 'date').reverse();
}

function parseModules(sectionName, section, modules) {
  return _.map(modules.keys(), function(name) {
    return {
      name: name.slice(2),
      file: modules(name),
      section: section,
      sectionName: sectionName === '/' ? '' : sectionName,
    };
  });
}

function itemForPath(path) {
  var items = allItems();

  // this check is needed for root paths to work (they have extra slash!)
  return items[path] || items['/' + path];
}
exports.itemForPath = itemForPath;

function renderContent(content) {
  return MdHelper.render(content);
}
exports.renderContent = renderContent;

function processItem(o, url, fileName, sectionName, section) {

  var layout = section.layout;

  var sectionFunctions = section.processItem || {};

  var functions = _.assign({
    date: function(o) {
      return o.file.date || null;
    },
    content: function(o) {
      return MdHelper.render(o.file.__content);
    },
    preview: function(o) {
      var file = o.file;

      if (file.preview) {
        return file.preview;
      }
      else {
        var previewLimit = 100;

        if(!file.content) {
          return;
        }

        var stripped = removeMd(file.__content);

        if (stripped.length > previewLimit) {
          return stripped.substr(0, previewLimit) + '…';
        }

        return stripped;
      }
      return file.preview || MdHelper.getContentPreview(file.__content);
    },
    title: function(o) {
      return o.file.title;
    },
    url: function(o) {
      return o.sectionName + '/' + o.fileName.split('.')[0];
    },
    layout: function(o) {
      return layout;
    },
  }, themeFunctions, siteFunctions, sectionFunctions);

  _.forEach(functions, function(fn, name) {
    o[name] = fn({
      file: o,
      fileName: fileName,
      sectionName: sectionName
    });
  });

  o.section = sectionName;

  return o;
}

{'fraction_non_alphanumeric': 0.14493419038873584, 'fraction_numerical': 0.0018365472910927456, 'mean_word_length': 3.076731129132876, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '544081', 'n_tokens_mistral': 2010, 'n_tokens_neox': 1965, 'n_words': 453}