<commit_before>'use strict';

var debug = require('debug')('bigpipe:compiler')
  , preprocess = require('smithy')
  , mkdirp = require('mkdirp')
  , crypto = require('crypto')
  , async = require('async')
  , File = require('./file')
  , path = require('path')
  , fs = require('fs');

//
// Find the pipe.js client library.
//
var pipejs = path.join(require.resolve('pipe.js'), '../dist/pipe.js');

/**
 * Asset compiler and management.
 *
 * @constructor
 * @param {String} directory The directory where we save our static files.
 * @param {Pipe} pipe The configured Pipe instance.
 * @param {Object} options Configuration.
 * @api private
 */
function Compiler(directory, pipe, options) {
  options = options || {};
  this.pipe = pipe;

  // The pipe.js client code.
  this.client = fs.readFileSync(pipejs, 'utf-8');

  // The namespace where we can download files.
  this.pathname = options.pathname || '/';

  // Directory to save the compiled files.
  this.dir = directory;

  // List of pre-compiled or previous compiled files.
  this.list = [];

  // Contains template engines that are used to render.
  this.core = [];

  this.buffer = Object.create(null); // Precompiled asset cache
  this.origin = Object.create(null); // Original location.
  this.alias = Object.create(null);  // Path aliases.

  //
  // Create the provided directory, will short circuit if present.
  //
  mkdirp.sync(directory);
}

Compiler.prototype.__proto__ = require('eventemitter3').prototype;

/**
 * Create the BigPipe base front-end framework that's required for the handling
 * of the real-time connections and the initialisation of the arriving pagelets.
 *
 * @param {String} client Core libraries that need to be loaded on every page.
 * @api private
 */
Compiler.prototype.bigPipe = function bigPipe(client) {
  var library = [
    '//',
    '// Powered by BigPipe v'+ this.pipe.version,
    '// https://github.com/bigpipe/bigpipe',
    '//',

    this.pipe.primus.library(),
    client,
    this.core.join('\n')
  ].join('\n');

  debug('creating the pipe.js front-end library which contains the Primus, client information and core');

  return this.register({
    pathname: this.pathname + this.hash(library) + '.js',
    origin: path.resolve(__dirname, '../dist/pipe.js'),
    alias: '/dist/pipe.js',
    code: library
  });
};

/**
 * Merge in objects.
 *
 * @param {Object} target The object that receives the props
 * @param {Object} additional Extra object that needs to be merged in the target
 * @api private
 */
Compiler.prototype.merge = function merge(target, additional) {
  var result = target
    , compiler = this
    , undefined;

  if (Array.isArray(target)) {
    compiler.forEach(additional, function arrayForEach(index) {
      if (JSON.stringify(target).indexOf(JSON.stringify(additional[index])) === -1) {
        result.push(additional[index]);
      }
    });
  } else if ('object' === typeof target) {
    compiler.forEach(additional, function objectForEach(key, value) {
      if (target[key] === undefined) {
        result[key] = value;
      } else {
        result[key] = compiler.merge(target[key], additional[key]);
      }
    });
  } else {
    result = additional;
  }

  return result;
};

/**
 * Iterate over a collection. When you return false, it will stop the iteration.
 *
 * @param {Mixed} collection Either an Array or Object.
 * @param {Function} iterator Function to be called for each item
 * @api private
 */
Compiler.prototype.forEach = function forEach(collection, iterator, context) {
  if (arguments.length === 1) {
    iterator = collection;
    collection = this;
  }

  var isArray = Array.isArray(collection || this)
    , length = collection.length
    , i = 0
    , value;

  if (context) {
    if (isArray) {
      for (; i < length; i++) {
        value = iterator.apply(collection[i], context);
        if (value === false) break;
      }
    } else {
      for (i in collection) {
        value = iterator.apply(collection[i], context);
        if (value === false) break;
      }
    }
  } else {
    if (isArray) {
      for (; i < length; i++) {
        value = iterator.call(collection[i], i, collection[i]);
        if (value === false) break;
      }
    } else {
      for (i in collection) {
        value = iterator.call(collection[i], i, collection[i]);
        if (value === false) break;
      }
    }
  }

  return this;
};

/**
 * Generate destination path which is used to store the generated file on disk.
 *
 * @param {String} filepath full path to file
 * @param {String} hash sha1 representation of the parsed content
 * @api public
 */
Compiler.prototype.destination = function destination(filepath, hash) {
  var processor = this.processor(filepath);

  return [
    this.pathname,
    hash,
    processor ? '.' + processor.export : path.extname(filepath)
  ].join('');
};

/**
 * Get preprocessor.
 *
 * @param {String} filepath
 * @returns {Function}
 * @api public
 */
Compiler.prototype.processor = function processor(filepath) {
  return preprocess[path.extname(filepath).substr(1)];
};

/**
 * Upsert new file in compiler cache.
 *
 * @param {String} filepath full path to file
 * @api private
 */
Compiler.prototype.put = function put(filepath) {
  var compiler = this
    , hash;

  compiler.process(filepath, function processed(error, code) {
    if (error) return compiler.emit('error', error);

    hash = compiler.hash(code);
    compiler.emit('preprocessed', filepath);

    compiler.register({
      pathname: compiler.destination(filepath, hash),
      alias: filepath,
      code: code,
      hash: hash
    });
  });
};

/**
 * Read the file from disk and preprocess it depending on extension.
 *
 * @param {String} filepath full path to file
 * @param {Function} fn callback
 * @api private
 */
Compiler.prototype.process = function process(filepath, fn) {
  var processor = this.processor(filepath)
    , paths = [ path.dirname(filepath) ];

  fs.readFile(filepath, 'utf-8', function read(error, code) {
    if (error || !processor) return fn(error, code);

    //
    // Only preprocess the file if required.
    //
    processor(code, { location: filepath, paths: paths }, fn);
  });
};

/**
 * Create a hash of the code which can be used a filename. This allows us to
 * aggressively cache the data.
 *
 * @param {String} code The code that is send to the client.
 * @returns {String} The compiled hash.
 * @api private
 */
Compiler.prototype.hash = function hash(code) {
  return crypto.createHash('sha1').update(code).digest('hex').toString('hex');
};

/**
 * Prefix selectors of CSS with [data-pagelet='name'] to contain CSS to
 * specific pagelets. File can have the following properties.
 *
 *   - alias {String} How we know this dependency.
 *   - pathname {String} The exact matching pathname to serve the given code.
 *   - code {Mixed} The library that needs to be transfered.
 *   - origin {String} Optional the original pathname before compiling
 *   - hash {String} sha1 hash of the orginal content
 *
 * @param {Object} representation of parsed content.
 * @param {Function} fn completion callback.
 * @api public
 */
Compiler.prototype.namespace = function prefix(file, fn) {
  //
  // Only prefix if the code is CSS content.
  //
  if ('.css' !== path.extname(file.pathname)) return fn();

  var processor = preprocess.css
    , options = {}
    , pagelets;

  //
  // Transform the pagelets names to data selectors.
  //
  pagelets = file.pagelets.map(function prepare(pagelet) {
    return '[data-pagelet="'+ pagelet +'"]';
  });

  options.plugins = [ processor.plugins.namespace(pagelets) ];
  processor(file.code, options, function done(error, code) {
    if (error) return fn(error);
    fn(null, file.code = code);
  });
};

/**
 * Register a new library with the compiler. The following traits can be
 * provided to register a specific file.
 *
 *   - alias {String} How we know this dependency.
 *   - pathname {String} The exact matching pathname to serve the given code.
 *   - code {Mixed} The library that needs to be transfered.
 *   - origin {String} Optional the original pathname before compiling
 *   - hash {String} sha1 hash of the orginal content
 *
 * @param {Object} traits of compiled content to register
 * @api private
 */
Compiler.prototype.register = function register(traits) {
  var pathname = traits.pathname
    , filename = path.basename(pathname)
    , extname = path.extname(pathname)
    , alias = traits.alias
    , code = traits.code
    , origin = traits.origin || alias
    , file;

  //
  // Update the CSS with a selector that contains the filename which is
  // required for async loading of CSS.
  //
  if (extname === '.css') {
    code = [
      code,
      '#pagelet_',
      traits.hash,
      ' { height: 42px }'
    ].join('');
  }

  //
  // Make sure that the given code is a buffer.
  //
  if (!Buffer.isBuffer(code)) code = new Buffer(code);
  file = this.buffer[pathname] = new File(code, extname);

  //
  // Get file and dirname from alias, unique in 99.9% of the cases.
  //
  alias = path.sep + alias.split(path.sep).splice(-2).join(path.sep);

  //
  // Add file to alias and origin.
  //
  this.alias[alias] = pathname;
  this.origin[origin] = alias;
  this.emit('register', file, alias, pathname, origin);

  return this.save(filename, file);
};

/**
 * Catalog the pages. As we're not caching the file look ups, this method can be
 * called when a file changes so we will generate new.
 *
 * @param {Array} pages The array of pages.
 * @param {Function} done callback
 * @api private
 */
Compiler.prototype.catalog = function catalog(pages, done) {
  var temper = this.pipe.temper
    , core = this.core
    , compiler = this
    , assemble = {}
    , list = {};

  /**
   * Filter out external dependencies, like jQuery and similar. Also remove them
   * from the assemble so it can be reused later on.
   *
   * @param {String} filepath absolute path to file.
   * @return {Boolean} keep or remove dependency.
   * @api private
   */
  function external(filepath) {
    if (!/^(http:|https:)?\/\//.test(filepath)) return true;

    delete list[filepath];
    return false;
  }

  /**
   * Process the dependencies.
   *
   * @param {String} filepath The location of a file.
   * @api private
   */
  function prefab(filepath, fn) {
    compiler.process(filepath, function store(error, code) {
      if (error) return fn(error);
      var hash = compiler.hash(code);

      assemble[hash] = compiler.merge(assemble[hash] || {}, {
        pathname: compiler.destination(filepath, hash),
        pagelets: list[filepath],
        alias: filepath,
        code: code,
        hash: hash
      });

      list[filepath] = hash;
      fn(null, code);
    });
  }

  function namespace(hash, fn) {
    compiler.namespace(assemble[hash], fn);
  }

  //
  // Check all pages for dependencies and files to add to the list.
  //
  pages.forEach(function each(Page) {
    var dependencies = ['/dist/pipe.js']
      , page = Page.prototype;

    page.pagelets.forEach(function each(Pagelet) {
      var pagelet = Pagelet.prototype
        , view;

      function add(file) {
        file = list[file] = list[file] || [];
        if (!~file.indexOf(pagelet.name)) file.push(pagelet.name);
      }

      if (pagelet.js) add(pagelet.js);
      if (pagelet.css) add(pagelet.css);

      pagelet.dependencies.forEach(function each(dependency) {
        add(dependency);

        if (!~dependencies.indexOf(dependency)) {
          dependencies.push(dependency);
        }
      });

      if (!pagelet.view) return;

      //
      // The views can be rendered on the client, but some of them require
      // a library, this library should be cached in the core library.
      //
      view = temper.fetch(pagelet.view);
      if (view.library && !~core.indexOf(view.library)) {
        core.push(view.library);
      }

      compiler.register({
        pathname: compiler.pathname + compiler.hash(view.client) +'.js',
        alias: pagelet.view,
        code: view.client
      });
    });

    page.dependencies = dependencies.reduce(function reduce(memo, dependency) {
      var extname = path.extname(dependency);

      memo[extname] = memo[extname] || [];
      memo[extname].push(dependency);

      return memo;
    }, Object.create(null));
  });

  //
  // list the CSS/JS of the pagelet with prefab.
  //
  async.each(Object.keys(list).filter(external), prefab, function register(error) {
    if (error) return done(error);

    async.each(Object.keys(assemble), namespace, function namespaced(error) {
      if (error) return done(error);

      Object.keys(list).forEach(function add(file) {
        file = list[file];
        compiler.register(assemble[file]);
      });

      done(null);
    });
  });

  //
  // Last, but not least, update the pipe.js library.
  //
  this.bigPipe(this.client);
};

/**
 * Find all required dependencies for given page constructor.
 *<commit_msg>[fix] refactor some code for improved readability<commit_after>'use strict';

var debug = require('debug')('bigpipe:compiler')
  , preprocess = require('smithy')
  , mkdirp = require('mkdirp')
  , crypto = require('crypto')
  , async = require('async')
  , File = require('./file')
  , path = require('path')
  , fs = require('fs');

//
// Find the pipe.js client library.
//
var pipejs = path.join(require.resolve('pipe.js'), '../dist/pipe.js');

/**
 * Asset compiler and management.
 *
 * @constructor
 * @param {String} directory The directory where we save our static files.
 * @param {Pipe} pipe The configured Pipe instance.
 * @param {Object} options Configuration.
 * @api private
 */
function Compiler(directory, pipe, options) {
  options = options || {};
  this.pipe = pipe;

  // The pipe.js client code.
  this.client = fs.readFileSync(pipejs, 'utf-8');

  // The namespace where we can download files.
  this.pathname = options.pathname || '/';

  // Directory to save the compiled files.
  this.dir = directory;

  // List of pre-compiled or previous compiled files.
  this.list = [];

  // Contains template engines that are used to render.
  this.core = [];

  this.buffer = Object.create(null); // Precompiled asset cache
  this.origin = Object.create(null); // Original location.
  this.alias = Object.create(null);  // Path aliases.

  //
  // Create the provided directory, will short circuit if present.
  //
  mkdirp.sync(directory);
}

Compiler.prototype.__proto__ = require('eventemitter3').prototype;

/**
 * Create the BigPipe base front-end framework that's required for the handling
 * of the real-time connections and the initialisation of the arriving pagelets.
 *
 * @param {String} client Core libraries that need to be loaded on every page.
 * @api private
 */
Compiler.prototype.bigPipe = function bigPipe(client) {
  var library = [
    '//',
    '// Powered by BigPipe v'+ this.pipe.version,
    '// https://github.com/bigpipe/bigpipe',
    '//',

    this.pipe.primus.library(),
    client,
    this.core.join('\n')
  ].join('\n');

  debug('creating the pipe.js front-end library which contains the Primus, client information and core');

  return this.register({
    pathname: this.pathname + this.hash(library) + '.js',
    origin: path.resolve(__dirname, '../dist/pipe.js'),
    alias: '/dist/pipe.js',
    code: library
  });
};

/**
 * Merge in objects.
 *
 * @param {Object} target The object that receives the props
 * @param {Object} additional Extra object that needs to be merged in the target
 * @api private
 */
Compiler.prototype.merge = function merge(target, additional) {
  var result = target
    , compiler = this
    , undefined;

  if (Array.isArray(target)) {
    compiler.forEach(additional, function arrayForEach(index) {
      if (JSON.stringify(target).indexOf(JSON.stringify(additional[index])) === -1) {
        result.push(additional[index]);
      }
    });
  } else if ('object' === typeof target) {
    compiler.forEach(additional, function objectForEach(key, value) {
      if (target[key] === undefined) {
        result[key] = value;
      } else {
        result[key] = compiler.merge(target[key], additional[key]);
      }
    });
  } else {
    result = additional;
  }

  return result;
};

/**
 * Iterate over a collection. When you return false, it will stop the iteration.
 *
 * @param {Mixed} collection Either an Array or Object.
 * @param {Function} iterator Function to be called for each item
 * @api private
 */
Compiler.prototype.forEach = function forEach(collection, iterator, context) {
  if (arguments.length === 1) {
    iterator = collection;
    collection = this;
  }

  var isArray = Array.isArray(collection || this)
    , length = collection.length
    , i = 0
    , value;

  if (context) {
    if (isArray) {
      for (; i < length; i++) {
        value = iterator.apply(collection[i], context);
        if (value === false) break;
      }
    } else {
      for (i in collection) {
        value = iterator.apply(collection[i], context);
        if (value === false) break;
      }
    }
  } else {
    if (isArray) {
      for (; i < length; i++) {
        value = iterator.call(collection[i], i, collection[i]);
        if (value === false) break;
      }
    } else {
      for (i in collection) {
        value = iterator.call(collection[i], i, collection[i]);
        if (value === false) break;
      }
    }
  }

  return this;
};

/**
 * Generate destination path which is used to store the generated file on disk.
 *
 * @param {String} filepath full path to file
 * @param {String} hash sha1 representation of the parsed content
 * @api public
 */
Compiler.prototype.destination = function destination(filepath, hash) {
  var processor = this.processor(filepath);

  return [
    this.pathname,
    hash,
    processor ? '.' + processor.export : path.extname(filepath)
  ].join('');
};

/**
 * Get preprocessor.
 *
 * @param {String} filepath
 * @returns {Function}
 * @api public
 */
Compiler.prototype.processor = function processor(filepath) {
  return preprocess[path.extname(filepath).substr(1)];
};

/**
 * Upsert new file in compiler cache.
 *
 * @param {String} filepath full path to file
 * @api private
 */
Compiler.prototype.put = function put(filepath) {
  var compiler = this
    , hash;

  compiler.process(filepath, function processed(error, code) {
    if (error) return compiler.emit('error', error);

    hash = compiler.hash(code);
    compiler.emit('preprocessed', filepath);

    compiler.register({
      pathname: compiler.destination(filepath, hash),
      alias: filepath,
      code: code,
      hash: hash
    });
  });
};

/**
 * Read the file from disk and preprocess it depending on extension.
 *
 * @param {String} filepath full path to file
 * @param {Function} fn callback
 * @api private
 */
Compiler.prototype.process = function process(filepath, fn) {
  var processor = this.processor(filepath)
    , paths = [ path.dirname(filepath) ];

  fs.readFile(filepath, 'utf-8', function read(error, code) {
    if (error || !processor) return fn(error, code);

    //
    // Only preprocess the file if required.
    //
    processor(code, { location: filepath, paths: paths }, fn);
  });
};

/**
 * Create a hash of the code which can be used a filename. This allows us to
 * aggressively cache the data.
 *
 * @param {String} code The code that is send to the client.
 * @returns {String} The compiled hash.
 * @api private
 */
Compiler.prototype.hash = function hash(code) {
  return crypto.createHash('sha1').update(code).digest('hex').toString('hex');
};

/**
 * Prefix selectors of CSS with [data-pagelet='name'] to contain CSS to
 * specific pagelets. File can have the following properties.
 *
 *   - alias {String} How we know this dependency.
 *   - pathname {String} The exact matching pathname to serve the given code.
 *   - code {Mixed} The library that needs to be transfered.
 *   - origin {String} Optional the original pathname before compiling
 *   - hash {String} sha1 hash of the orginal content
 *
 * @param {Object} representation of parsed content.
 * @param {Function} fn completion callback.
 * @api public
 */
Compiler.prototype.namespace = function prefix(file, fn) {
  //
  // Only prefix if the code is CSS content.
  //
  if ('.css' !== path.extname(file.pathname)) return fn(null, file);

  var processor = preprocess.css
    , options = {}
    , pagelets;

  //
  // Transform the pagelets names to data selectors.
  //
  pagelets = file.pagelets.map(function prepare(pagelet) {
    return '[data-pagelet="'+ pagelet +'"]';
  });

  options.plugins = [ processor.plugins.namespace(pagelets) ];
  processor(file.code, options, function done(error, code) {
    if (error) return fn(error);

    file.code = code;
    fn(null, file);
  });
};

/**
 * Register a new library with the compiler. The following traits can be
 * provided to register a specific file.
 *
 *   - alias {String} How we know this dependency.
 *   - pathname {String} The exact matching pathname to serve the given code.
 *   - code {Mixed} The library that needs to be transfered.
 *   - origin {String} Optional the original pathname before compiling
 *   - hash {String} sha1 hash of the orginal content
 *
 * @param {Object} traits of compiled content to register
 * @api private
 */
Compiler.prototype.register = function register(traits) {
  var alias = Array.isArray(traits.alias) ? traits.alias : [ traits.alias ]
    , pathname = traits.pathname
    , filename = path.basename(pathname)
    , extname = path.extname(pathname)
    , code = traits.code
    , compiler = this
    , file;

  //
  // Update the CSS with a selector that contains the filename which is
  // required for async loading of CSS.
  //
  if (extname === '.css') {
    code = [
      code,
      '#pagelet_',
      traits.hash,
      ' { height: 42px }'
    ].join('');
  }

  //
  // Make sure that the given code is a buffer.
  //
  if (!Buffer.isBuffer(code)) code = new Buffer(code);
  traits.code = this.buffer[pathname] = new File(code, extname);

  //
  // Add file references to alias and origin by file and dirname from alias,
  // unique in 99.9% of the cases.
  //
  alias.forEach(function add(alias) {
    var simple = path.sep + alias.split(path.sep).splice(-2).join(path.sep);

    compiler.origin[traits.origin || alias] = simple;
    compiler.alias[simple] = pathname;
  });

  this.emit('register', traits);
  return this.save(filename, traits.code);
};

/**
 * Catalog the pages. As we're not caching the file look ups, this method can be
 * called when a file changes so we will generate new.
 *
 * @param {Array} pages The array of pages.
 * @param {Function} done callback
 * @api private
 */
Compiler.prototype.catalog = function catalog(pages, done) {
  var temper = this.pipe.temper
    , core = this.core
    , compiler = this
    , list = {};

  /**
   * Process the dependencies.
   *
   * @param {Object} assemble generated collection of file properties.
   * @param {String} filepath The location of a file.
   * @param {Function} next completion callback.
   * @api private
   */
  function prefab(assemble, filepath, next) {
    compiler.process(filepath, function store(error, code) {
      if (error) return next(error);
      var hash = compiler.hash(code);

      assemble[hash] = compiler.merge(assemble[hash] || {}, {
        pathname: compiler.destination(filepath, hash),
        pagelets: list[filepath],
        alias: [ filepath ],
        code: code,
        hash: hash
      });

      next(null, assemble);
    });
  }

  /**
   * Register the files in the assembly, prefix CSS first.
   *
   * @param {Object} assemble generated collection of file properties.
   * @param {Function} next completion callback.
   * @api private
   */
  function register(assemble, next) {
    async.each(Object.keys(assemble), function prefix(hash, fn) {
      compiler.namespace(assemble[hash], function namespaced(error, file) {
        if (error) return fn(error);

        compiler.register(file);
        fn();
      });
    }, next);
  }

  //
  // Check all pages for dependencies and files to add to the list.
  //
  pages.forEach(function each(Page) {
    var dependencies = ['/dist/pipe.js']
      , page = Page.prototype;

    page.pagelets.forEach(function each(Pagelet) {
      var pagelet = Pagelet.prototype
        , view;

      /**
       * Add file to the process list.
       *
       * @param {file} file path to file.
       * @api private
       */
      function add(file) {
        if (/^(http:|https:)?\/\//.test(file)) return;

        file = list[file] = list[file] || [];
        if (!~file.indexOf(pagelet.name)) file.push(pagelet.name);
      }

      if (pagelet.js) add(pagelet.js);
      if (pagelet.css) add(pagelet.css);

      pagelet.dependencies.forEach(function each(dependency) {
        add(dependency);

        if (!~dependencies.indexOf(dependency)) {
          dependencies.push(dependency);
        }
      });

      if (!pagelet.view) return;

      //
      // The views can be rendered on the client, but some of them require
      // a library, this library should be cached in the core library.
      //
      view = temper.fetch(pagelet.view);
      if (view.library && !~core.indexOf(view.library)) {
        core.push(view.library);
      }

      compiler.register({
        pathname: compiler.pathname + compiler.hash(view.client) +'.js',
        alias: pagelet.view,
        code: view.client
      });
    });

    page.dependencies = dependencies.reduce(function reduce(memo, dependency) {
      var extname = path.extname(dependency);

      memo[extname] = memo[extname] || [];
      memo[extname].push(dependency);

      return memo;
    }, Object.create(null));
  });

  //
  // Process and register the CSS/JS of all the pagelets.
  //
  async.waterfall([
    async.apply(async.reduce, Object.keys(list), {}, prefab),
    register
  ], done);

  //
  // Last, but not least, update the pipe.js library.
  //
  this.bigPipe(this.client);
};

/**
 * Find all required dependencies for given page constructor.
 *
{'fraction_non_alphanumeric': 0.1359245630174793, 'fraction_numerical': 0.00130328120208525, 'mean_word_length': 2.8519120035434815, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3254879', 'n_tokens_mistral': 7862, 'n_tokens_neox': 7687, 'n_words': 2369}