<commit_before>'use strict';
var es = require('event-stream');
var gutil = require('gulp-util');
var walk = require('walk');
var jf = require('jsonfile');
var chalk = require('chalk');
var fs = require('fs');
var path = require('path');
var _s = require ('underscore.string');

var dirTree = function (filename) {
  var name = path.basename(filename);

  if (name.charAt(0) === '.' || name === '') {
    return {
      skip: true,
      path: path.basename(filename)
    };
  }
  name = name.replace('.json', '');

  console.log(filename);
  var base = filename.replace(path.basename(filename), '').split('/')[2];

  var menuName = name.replace(/--/g, '|&&|');
      menuName = menuName.replace(/-/g, ' ');
      menuName = menuName.replace(/\|\&\&\|/g, ' - ');
      menuName = _s.titleize(menuName);

  var stats = fs.lstatSync(filename),
      info = {
        title: menuName
      };

  var group = filename.replace(path.basename(filename), '');
      group = group.split('/');
      group = group.splice(3, group.length - 4);
      // console.log(group);
      // console.log('-------------');
      group = group.join('-');

  if (stats.isDirectory()) {
    group = filename.split('/').splice(3).join('-');
    // console.log(group);
    // console.log('-------------');

    // info.type = "folder";
    info.submenu = fs.readdirSync(filename).map(function(child) {
      return dirTree(filename + '/' + child);
    });
  } else {
      if (jf.readFileSync(filename) === null) {
        return {
          skip: true,
          path: path.basename(filename)
        };
    }

    // console.log(group);
    // console.log(name);
    // console.log('-------------');

    var id = group + '--' + name;
        id = id.indexOf('--') === 0 ? id.substr(2) : id;
    info.href = '#/pages?id=' + id;
      // Assuming it's a file. In real life it could be a symlink or
      // something else!
  }



  return info;

}

var pagewalk = function (options) {
  // Make sure Data folder is available
  if (!fs.existsSync('.www')) {
    fs.mkdirSync('.www');
  }
  if (!fs.existsSync('.tmp')) {
    fs.mkdirSync('.tmp');
  }
  if (!fs.existsSync('.tmp/data')) {
    fs.mkdirSync('.tmp/data');
  }

  return es.map(function (gulpFile, cb) {
    var fileName = '.tmp/data/pages.json';
    var partials = '.tmp/pages';
    var files = [];
    var folders = [];
    var walker = walk.walk(partials);

    var fileConfig = jf.readFileSync('.www/config/files.json');
    // console.log(partials);

    walker.on('file', function(root, stat, next) {

      var fileName = stat.name;
          fileName = fileName.replace('.json', '');
      var fileTitle = fileName.replace(/--/g, '|&&|');
          fileTitle = fileTitle.replace(/-/g, ' ');
          fileTitle = fileTitle.replace(/\|\&\&\|/g, ' - ');
          fileTitle = _s.titleize(fileTitle);
      var filePath = root + '/' + stat.name;
          filePath = filePath.replace('.www/', '');
      var fileRoot = root.charAt(0) === '/' ? root.substr(1) : root;
      var fileId = fileRoot.split('/').splice(3).join('-') + '--' + fileName;
          fileId = fileId.indexOf('--') === 0 ? fileId.substr(2) : fileId;
      var settings = jf.readFileSync(root + '/' + stat.name);

      var includes = [];
      if (settings && settings.includes) {
        for (var i in settings.includes) {
          var inc = settings.includes[i];
          if (fileConfig[inc.source]) {
            for (var j in fileConfig[inc.source]) {
              if (fileConfig[inc.source][j]) {
                if (fileConfig[inc.source][j].id === inc.name) {
                  var group = {
                    'name': i,
                    'path': fileConfig[inc.source][j].path
                  }
                  includes.push(group);
                }
              }<commit_msg>Includes equal to name, not id<commit_after>'use strict';
var es = require('event-stream');
var gutil = require('gulp-util');
var walk = require('walk');
var jf = require('jsonfile');
var chalk = require('chalk');
var fs = require('fs');
var path = require('path');
var _s = require ('underscore.string');

var dirTree = function (filename) {
  var name = path.basename(filename);

  if (name.charAt(0) === '.' || name === '') {
    return {
      skip: true,
      path: path.basename(filename)
    };
  }
  name = name.replace('.json', '');

  // console.log(filename);
  var base = filename.replace(path.basename(filename), '').split('/')[2];

  var menuName = name.replace(/--/g, '|&&|');
      menuName = menuName.replace(/-/g, ' ');
      menuName = menuName.replace(/\|\&\&\|/g, ' - ');
      menuName = _s.titleize(menuName);

  var stats = fs.lstatSync(filename),
      info = {
        title: menuName
      };

  var group = filename.replace(path.basename(filename), '');
      group = group.split('/');
      group = group.splice(3, group.length - 4);
      // console.log(group);
      // console.log('-------------');
      group = group.join('-');

  if (stats.isDirectory()) {
    group = filename.split('/').splice(3).join('-');
    // console.log(group);
    // console.log('-------------');

    // info.type = "folder";
    info.submenu = fs.readdirSync(filename).map(function(child) {
      return dirTree(filename + '/' + child);
    });
  } else {
      if (jf.readFileSync(filename) === null) {
        return {
          skip: true,
          path: path.basename(filename)
        };
    }

    // console.log(group);
    // console.log(name);
    // console.log('-------------');

    var id = group + '--' + name;
        id = id.indexOf('--') === 0 ? id.substr(2) : id;
    info.href = '#/pages?id=' + id;
      // Assuming it's a file. In real life it could be a symlink or
      // something else!
  }



  return info;

}

var pagewalk = function (options) {
  // Make sure Data folder is available
  if (!fs.existsSync('.www')) {
    fs.mkdirSync('.www');
  }
  if (!fs.existsSync('.tmp')) {
    fs.mkdirSync('.tmp');
  }
  if (!fs.existsSync('.tmp/data')) {
    fs.mkdirSync('.tmp/data');
  }

  return es.map(function (gulpFile, cb) {
    var fileName = '.tmp/data/pages.json';
    var partials = '.tmp/pages';
    var files = [];
    var folders = [];
    var walker = walk.walk(partials);

    var fileConfig = jf.readFileSync('.www/config/files.json');
    // console.log(partials);

    walker.on('file', function(root, stat, next) {

      var fileName = stat.name;
          fileName = fileName.replace('.json', '');
      var fileTitle = fileName.replace(/--/g, '|&&|');
          fileTitle = fileTitle.replace(/-/g, ' ');
          fileTitle = fileTitle.replace(/\|\&\&\|/g, ' - ');
          fileTitle = _s.titleize(fileTitle);
      var filePath = root + '/' + stat.name;
          filePath = filePath.replace('.www/', '');
      var fileRoot = root.charAt(0) === '/' ? root.substr(1) : root;
      var fileId = fileRoot.split('/').splice(3).join('-') + '--' + fileName;
          fileId = fileId.indexOf('--') === 0 ? fileId.substr(2) : fileId;
      var settings = jf.readFileSync(root + '/' + stat.name);

      var includes = [];
      if (settings && settings.includes) {
        for (var i in settings.includes) {
          var inc = settings.includes[i];
          if (fileConfig[inc.source]) {
            for (var j in fileConfig[inc.source]) {
              if (fileConfig[inc.source][j]) {
                if (fileConfig[inc.source][j].name === inc.name) {
                  var group = {
                    'name': i,
                    'path': fileConfig[inc.source][j].path
                  }
                  includes.push(group);
                }
              }
{'fraction_non_alphanumeric': 0.20400943396226415, 'fraction_numerical': 0.0031446540880503146, 'mean_word_length': 2.099066179455948, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 8, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2079714', 'n_tokens_mistral': 2498, 'n_tokens_neox': 2433, 'n_words': 495}