<commit_before>/*
 * Assemble
 * https://github.com/assemble/assemble
 *
 * Copyright (c) 2012 <NAME>
 * Licensed under the MIT license.
 */

module.exports = function(grunt) {

  // Grunt utilities
  var file     = grunt.file;
  var log      = grunt.log;
  var kindOf   = grunt.util.kindOf;
  var _        = grunt.util._;
  var lodash   = require('lodash'); // required since grunt's lodash might be older

  var assemble = require('../lib/assemble');

  // external dependencies
  var path     = require('path');
  var fs       = require('fs');
  var util     = require('util');

  var extensions = assemble.Utils.ExtensionMap;

  grunt.registerMultiTask('assemble', 'Compile template files with specified engines', function(){


    var options = this.options({
      layout        : '',
      partials      : [],
      data          : [],
      assets        : 'dist/assets',
      ext           : '.html'
    });

    logBlock("options: ", util.inspect(options));
    logBlock("this.files: ", util.inspect(this.files));

    options.data = mergeOptionsArrays(this.target, 'data');
    options.partials = mergeOptionsArrays(this.target, 'partials');

    // try to get a src to use for configuration
    var src = false;
    this.files.forEach(function(fp) {
      if(!src) {
        src = fp.src;
      }
    });

    if(!src || src.length === 0) {
      grunt.warn('No source files found.');
      return false;
    }

    // find an engine to use
    options.engine = options.engine || getEngineOf(src);
    if(!options.engine) {
      grunt.warn('No compatible engine available');
      return false;
    }

    var EngineLoader = options.EngineLoader = assemble.EngineLoader(options);
    var engine = null;
    EngineLoader.getEngine(function(err, results) {
      if(err) {
        console.log(err);
        return;
      }
      engine = options.engine = results;
    });

    var yamlPreprocessor = EngineLoader.getPreprocessor('YamlPreprocessor');

    var partials = file.expand(options.partials);
    var dataFiles = file.expand(options.data);
    var fileExt = extension(src);
    var filenameRegex = /[^\\\/:*?"<>|\r\n]+$/i;
    var fileExtRegex = new RegExp("\\." + fileExt + "$");

    options.filenameRegex = filenameRegex;
    options.fileExtRegex = fileExtRegex;

    var done = this.async();

    // clear out the partials and data objects on options
    options.partials = {};
    options.data = {};

    // load default layout
    var defaultLayoutName,
        defaultLayout,
        defaultLayoutData = {};

    loadLayout(
      options.layout,
      {
        filenameRegex: filenameRegex,
        fileExtRegex: fileExtRegex,
        EngineLoader: EngineLoader,
        engine: engine
      },
      function(err, results) {
        if(!err) {
          defaultLayoutName = results.layoutName;
          defaultLayout = results.layout;
          defaultLayoutData = results.data;
        } else {
          grunt.warn(err.message);
        }
      });

    // merge any layoutData with options
    options.data = _.extend(defaultLayoutData.context, options.data || {});

    var complete = 0;
    var increment = 10;

    // load partials if specified
    if(partials && partials.length > 0) {
      complete = 0;
      increment = Math.round(partials.length / 10);
      log.write(('\n' + 'Processing partials...').grey);

      partials.forEach(function(filepath) {
        var filename = _.first(filepath.match(filenameRegex)).replace(fileExtRegex, '');
        grunt.verbose.writeln(('Processing ' + filename + ' partial').cyan);
        if(complete%increment === 0) {
          log.write('.'.cyan);
        }

        var partial = fs.readFileSync(filepath, 'utf8');

        partial = engine.compile(partial, {
          preprocessers: [
            yamlPreprocessor(filename, function(output) {
              options.data[output.name] = _.extend(output.output.context, options.data[output.name] || {});
            })
          ]
        });

        // register the partial with the engine
        engine.engine.registerPartial(filename, partial);
        complete++;
      });
      log.notverbose.writeln('\n');
    }

    // load data if specified
    if(dataFiles && dataFiles.length > 0) {
      complete = 0;
      increment = Math.round(dataFiles.length / 10);
      log.writeln(('\n' + 'Begin processing data...').grey);

      dataFiles.forEach(function(filepath) {
        var ext = path.extname(filepath);
        var filename = path.basename(filepath, ext);

        var fileReader = dataFileReaderFactory(ext);

        if(complete%increment === 0) {
          log.notverbose.write('.'.cyan);
        }

        if(filename === 'data') {
          // if this is the base data file, load it into the options.data object directly
          options.data = _.extend(options.data || {}, fileReader(filepath));
        } else {
          // otherwise it's an element in options.data
          var d = fileReader(filepath);
          if(d[filename]) {
            // json object contains root object name so extend it in options.json
            options.data[filename] = _.extend(options.data[filename] || {}, d[filename]);
          } else {
            // add the entire object
            options.data[filename] = _.extend(options.data[filename] || {}, d);
          }
        }
        complete++;
      });
      log.writeln('\n');
    }

    options.defaultLayout     = defaultLayout;
    options.defaultLayoutName = defaultLayoutName;

    // build each page
    log.writeln(('\n' + 'Building pages...').grey);

    var info = buildInfo(this, options);
    grunt.log.writeln(require('util').inspect(info, 10, null));
    options.pages = info.pages;
    options.tags = info.tags;
    options.categories = info.categories;

    options.pages.forEach(function(page) {

      grunt.verbose.writeln(require('util').inspect(page));

      build(page.src, page.basename, options, function(err, result) {
        if(err) {
          grunt.warn(err);
          done(false);
          return;
        }

        file.write(page.dest, result);
        grunt.log.ok('File ' + (page.basename + page.ext).magenta + ' created.' + ' ok '.green);
      }); // build

    });


    if(done) {
      done();
    }

  });
<commit_msg>removing debugging info<commit_after>/*
 * Assemble
 * https://github.com/assemble/assemble
 *
 * Copyright (c) 2012 <NAME>
 * Licensed under the MIT license.
 */

module.exports = function(grunt) {

  // Grunt utilities
  var file     = grunt.file;
  var log      = grunt.log;
  var kindOf   = grunt.util.kindOf;
  var _        = grunt.util._;
  var lodash   = require('lodash'); // required since grunt's lodash might be older

  var assemble = require('../lib/assemble');

  // external dependencies
  var path     = require('path');
  var fs       = require('fs');
  var util     = require('util');

  var extensions = assemble.Utils.ExtensionMap;

  grunt.registerMultiTask('assemble', 'Compile template files with specified engines', function(){


    var options = this.options({
      layout        : '',
      partials      : [],
      data          : [],
      assets        : 'dist/assets',
      ext           : '.html'
    });

    logBlock("options: ", util.inspect(options));
    logBlock("this.files: ", util.inspect(this.files));

    options.data = mergeOptionsArrays(this.target, 'data');
    options.partials = mergeOptionsArrays(this.target, 'partials');

    // try to get a src to use for configuration
    var src = false;
    this.files.forEach(function(fp) {
      if(!src) {
        src = fp.src;
      }
    });

    if(!src || src.length === 0) {
      grunt.warn('No source files found.');
      return false;
    }

    // find an engine to use
    options.engine = options.engine || getEngineOf(src);
    if(!options.engine) {
      grunt.warn('No compatible engine available');
      return false;
    }

    var EngineLoader = options.EngineLoader = assemble.EngineLoader(options);
    var engine = null;
    EngineLoader.getEngine(function(err, results) {
      if(err) {
        console.log(err);
        return;
      }
      engine = options.engine = results;
    });

    var yamlPreprocessor = EngineLoader.getPreprocessor('YamlPreprocessor');

    var partials = file.expand(options.partials);
    var dataFiles = file.expand(options.data);
    var fileExt = extension(src);
    var filenameRegex = /[^\\\/:*?"<>|\r\n]+$/i;
    var fileExtRegex = new RegExp("\\." + fileExt + "$");

    options.filenameRegex = filenameRegex;
    options.fileExtRegex = fileExtRegex;

    var done = this.async();

    // clear out the partials and data objects on options
    options.partials = {};
    options.data = {};

    // load default layout
    var defaultLayoutName,
        defaultLayout,
        defaultLayoutData = {};

    loadLayout(
      options.layout,
      {
        filenameRegex: filenameRegex,
        fileExtRegex: fileExtRegex,
        EngineLoader: EngineLoader,
        engine: engine
      },
      function(err, results) {
        if(!err) {
          defaultLayoutName = results.layoutName;
          defaultLayout = results.layout;
          defaultLayoutData = results.data;
        } else {
          grunt.warn(err.message);
        }
      });

    // merge any layoutData with options
    options.data = _.extend(defaultLayoutData.context, options.data || {});

    var complete = 0;
    var increment = 10;

    // load partials if specified
    if(partials && partials.length > 0) {
      complete = 0;
      increment = Math.round(partials.length / 10);
      log.write(('\n' + 'Processing partials...').grey);

      partials.forEach(function(filepath) {
        var filename = _.first(filepath.match(filenameRegex)).replace(fileExtRegex, '');
        grunt.verbose.writeln(('Processing ' + filename + ' partial').cyan);
        if(complete%increment === 0) {
          log.write('.'.cyan);
        }

        var partial = fs.readFileSync(filepath, 'utf8');

        partial = engine.compile(partial, {
          preprocessers: [
            yamlPreprocessor(filename, function(output) {
              options.data[output.name] = _.extend(output.output.context, options.data[output.name] || {});
            })
          ]
        });

        // register the partial with the engine
        engine.engine.registerPartial(filename, partial);
        complete++;
      });
      log.notverbose.writeln('\n');
    }

    // load data if specified
    if(dataFiles && dataFiles.length > 0) {
      complete = 0;
      increment = Math.round(dataFiles.length / 10);
      log.writeln(('\n' + 'Begin processing data...').grey);

      dataFiles.forEach(function(filepath) {
        var ext = path.extname(filepath);
        var filename = path.basename(filepath, ext);

        var fileReader = dataFileReaderFactory(ext);

        if(complete%increment === 0) {
          log.notverbose.write('.'.cyan);
        }

        if(filename === 'data') {
          // if this is the base data file, load it into the options.data object directly
          options.data = _.extend(options.data || {}, fileReader(filepath));
        } else {
          // otherwise it's an element in options.data
          var d = fileReader(filepath);
          if(d[filename]) {
            // json object contains root object name so extend it in options.json
            options.data[filename] = _.extend(options.data[filename] || {}, d[filename]);
          } else {
            // add the entire object
            options.data[filename] = _.extend(options.data[filename] || {}, d);
          }
        }
        complete++;
      });
      log.writeln('\n');
    }

    options.defaultLayout     = defaultLayout;
    options.defaultLayoutName = defaultLayoutName;

    // build each page
    log.writeln(('\n' + 'Building pages...').grey);

    var info = buildInfo(this, options);
    options.pages = info.pages;
    options.tags = info.tags;
    options.categories = info.categories;

    options.pages.forEach(function(page) {

      grunt.verbose.writeln(require('util').inspect(page));

      build(page.src, page.basename, options, function(err, result) {
        if(err) {
          grunt.warn(err);
          done(false);
          return;
        }

        file.write(page.dest, result);
        grunt.log.ok('File ' + (page.basename + page.ext).magenta + ' created.' + ' ok '.green);
      }); // build

    });


    if(done) {
      done();
    }

  });

{'fraction_non_alphanumeric': 0.1386662391792241, 'fraction_numerical': 0.0032061558191728116, 'mean_word_length': 2.2182099561516635, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3259658', 'n_tokens_mistral': 3684, 'n_tokens_neox': 3550, 'n_words': 785}