<commit_before>// DecentCMS (c) 2014 <NAME>, under MIT. See LICENSE.txt for licensing details.
'use strict';
var path = require('path');
var fs = require('fs');
var async = require('async');

/**
 * @description
 * A content store that uses JSON files.
 * @param shell
 * @constructor
 */
function FileContentStore(shell) {
  this.shell = shell;
}
FileContentStore.isShellSingleton = true;

FileContentStore.on = {
  'decent.core.load-items': function(shell, payload) {
    var shapeHelper = shell.require('shape');

    var items = payload.items;
    var itemsToFetch = payload.itemsToFetch;
    var callback = payload.callback;

    var siteDataRoot = path.join(shell.rootPath, 'data/content');

    var handle = function handleITemData(id, filePath, data, callback) {
      // Parse the content item file
      var item = JSON.parse(data);
      // Look for any part that needs to load an additional file.
      // Typically, that could be a markdown file for the body.
      async.each(Object.keys(item), function lookForExtraFile(partName, next) {
          var part;
          if (partName !== 'meta'
            && partName !== 'temp'
            && (part = item[partName])
            && part.hasOwnProperty('path')) {
            var extraFilePath = path.join(path.dirname(filePath), part.path);
            fs.readFile(extraFilePath, function extraFileRead(err, data) {
              if (err) { callback(err); return; }
              part._data = data.toString();
              next();
            });
          }
          else {
            next();
          }
        },
        function allExtraFilesRead(err) {
          if (err) { callback(err); return; }
          // Add the file path to temp meta data
          var temp = shapeHelper.temp(item);
          temp.filePath = filePath;
          // Add to the items list
          items[id] = item;
          // Call all the item-specific callbacks
          if (Array.isArray(itemsToFetch[id])) {
            for (var j = 0; j < itemsToFetch[id].length; j++) {
              itemsToFetch[id](null, item);
            }
          }
          // Remove from the list of remaining items to fetch
          delete itemsToFetch[id];
          callback();
        });
    };

    async.each(Object.keys(itemsToFetch), function findItemFile(id, next) {
      // TODO: support for redirected roots where a folder points at another. This will enable module documentation sub sites.
      var itemFilePath = path.join(siteDataRoot, id, 'index.json');
      fs.readFile(itemFilePath, function readIndexFile(err, data) {
        if (!err) {
          handle(id, itemFilePath, data.toString(), next);
        }
        else {
          // File not found is normal, so let it flow, but re-throw others
          if (err.code !== 'ENOENT') { callback(err); return; }
          // This was not a folder, maybe it was already a file.
          itemFilePath = path.join(siteDataRoot, id);
          fs.readFile(itemFilePath, function readItemFile(err, data) {
            if (err) { callback(err); return; }
            handle(id, itemFilePath, data.toString(), next);
          });
        }
      });
    },
    callback);
  }
};

module.exports = FileContentStore;<commit_msg>Fixed typo in function name.<commit_after>// DecentCMS (c) 2014 <NAME>, under MIT. See LICENSE.txt for licensing details.
'use strict';
var path = require('path');
var fs = require('fs');
var async = require('async');

/**
 * @description
 * A content store that uses JSON files.
 * @param shell
 * @constructor
 */
function FileContentStore(shell) {
  this.shell = shell;
}
FileContentStore.isShellSingleton = true;

FileContentStore.on = {
  'decent.core.load-items': function(shell, payload) {
    var shapeHelper = shell.require('shape');

    var items = payload.items;
    var itemsToFetch = payload.itemsToFetch;
    var callback = payload.callback;

    var siteDataRoot = path.join(shell.rootPath, 'data/content');

    var handle = function handleItemData(id, filePath, data, callback) {
      // Parse the content item file
      var item = JSON.parse(data);
      // Look for any part that needs to load an additional file.
      // Typically, that could be a markdown file for the body.
      async.each(Object.keys(item), function lookForExtraFile(partName, next) {
          var part;
          if (partName !== 'meta'
            && partName !== 'temp'
            && (part = item[partName])
            && part.hasOwnProperty('path')) {
            var extraFilePath = path.join(path.dirname(filePath), part.path);
            fs.readFile(extraFilePath, function extraFileRead(err, data) {
              if (err) { callback(err); return; }
              part._data = data.toString();
              next();
            });
          }
          else {
            next();
          }
        },
        function allExtraFilesRead(err) {
          if (err) { callback(err); return; }
          // Add the file path to temp meta data
          var temp = shapeHelper.temp(item);
          temp.filePath = filePath;
          // Add to the items list
          items[id] = item;
          // Call all the item-specific callbacks
          if (Array.isArray(itemsToFetch[id])) {
            for (var j = 0; j < itemsToFetch[id].length; j++) {
              itemsToFetch[id](null, item);
            }
          }
          // Remove from the list of remaining items to fetch
          delete itemsToFetch[id];
          callback();
        });
    };

    async.each(Object.keys(itemsToFetch), function findItemFile(id, next) {
      // TODO: support for redirected roots where a folder points at another. This will enable module documentation sub sites.
      var itemFilePath = path.join(siteDataRoot, id, 'index.json');
      fs.readFile(itemFilePath, function readIndexFile(err, data) {
        if (!err) {
          handle(id, itemFilePath, data.toString(), next);
        }
        else {
          // File not found is normal, so let it flow, but re-throw others
          if (err.code !== 'ENOENT') { callback(err); return; }
          // This was not a folder, maybe it was already a file.
          itemFilePath = path.join(siteDataRoot, id);
          fs.readFile(itemFilePath, function readItemFile(err, data) {
            if (err) { callback(err); return; }
            handle(id, itemFilePath, data.toString(), next);
          });
        }
      });
    },
    callback);
  }
};

module.exports = FileContentStore;
{'fraction_non_alphanumeric': 0.12069231957966312, 'fraction_numerical': 0.0015453562046051615, 'mean_word_length': 2.1663405088062624, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3224379', 'n_tokens_mistral': 1791, 'n_tokens_neox': 1769, 'n_words': 533}