<commit_before>/**
 * index.js
 *
 * Async stuff for html-snapshots
 *   Notifier class
 *
 * Copyright (c) 2013 - 2017, <NAME>, LocalNerve, contributors
 * Licensed under the MIT license.
 */
/* global Promise */
"use strict";

var fs = require("fs");
var crypto = require("crypto");
var AsyncLock = require("async-lock");
var combineErrors = require("combine-errors");
var nodeCall = require("../common/node");

var F_OK = (fs.constants && fs.constants.F_OK) || fs.F_OK;
var L_WAIT = 10000;
var TIMEOUT_PAD_FLOOR = 800;
function NOOP () {}

/**
 * Create the lock factory.
 */
function createLockFactory () {
  // Create the per instance async lock.
  var lock = new AsyncLock();

  // Make a random id
  var rid = crypto.randomBytes(16).toString("hex");

  /**
   * Force a serial execution context.
   *
   * @param {Function} fn - The function to guard.
   * @param {Number} timeout - The max time to wait for the lock.
   */
  return function lockFactory (fn, timeout) {
    return function protectedContext () {
      lock.acquire("cs-guard-" + rid, function (done) {
        fn(function () {
          done(null, 0);
        });
      }, NOOP, {
        timeout: timeout
      });
    };
  };
}

/**
 * Notifier Constructor
 * Polls the outputDir and when all the files exist,
 * calls the listener indicating the snapshots are done.
 */
function Notifier () {

  // Create the serial execution context mechanism.
  this.csFactory = createLockFactory();

  // The private files collection
  // Contains a file and timer: "filename": {timer: timerid}
  // Used for tracking work left to do. When empty, work is done.
  this.files = {};

  // Contains files successfully processed
  this.filesDone = [];

  // true if a timeout occurred, or set by abort
  this.errors = [];

  // the holder of the current failure timeout padding
  this.padTimeout = TIMEOUT_PAD_FLOOR;

  // our reference to the listener
  this.callback = NOOP;

  // our reference to the watcher (an interval id)
  // initial value undefined is important
  // this.watcher;

  // the working pollInterval for this run
  // this.interval

  // flag set by qEmpty callback
  // when the last item from the queue is given to a worker
  this.qempty = false;
}

Notifier.prototype = {
  /**
   * Set the qempty flag to true.
   */
  qEmpty: function qEmpty () {
    this.qempty = true;
  },

  /**
   * Start a watch interval, when a file exists, remove it from our files array.
   * If the files array is empty, call the listener and stop the watch interval.
   *
   * @param {Number} pollInterval - The millisecond poll interval to check for
   * file completion.
   * @param {Function} listener - The callback to notify when complete.
   * @param {Object} input - The inputGenerator.
   * @param {Function} input.EOI - The inputGenerator function that signals end
   * of input.
   * @returns {Boolean} true if started successfully, false otherwise.
   */
  start: function start (pollInterval, listener, input) {
    var result = (
      pollInterval > 0 &&
      typeof listener === "function" &&
      (!!input)
      );

    if (result) {
      if (this.isStarted()) {
        throw new Error("Notifier already started");
      }

      this.callback = listener;
      this.interval = parseInt(pollInterval, 10);

      // Poll the filesystem for the files to exist
      // Checks the child process expected output to determine success or failure
      // if the file exists, then it succeeded.
      this.watcher = setInterval(this.csFactory(function (done) {
        var self = this;
        var eoi = typeof input.EOI === "function" && input.EOI();

        if (eoi) {
          Promise.all(Object.keys(self.files).map(function (file) {
            return nodeCall(fs.access, file, F_OK)
              .then(function () {
                return file;
              })
              .catch(function () {
                return null;
              });
          }))
            .then(function (files) {
              try {
                files.forEach(function (file) {
                  file && self._remove(file, true);
                });

                // if done, exit
                if (self._isDone() && self._closeWatcher()) {
                  setImmediate(function () {
                    self.callback(self.getError(), self.filesDone);
                  });
                }
              } catch (e) {
                console.error(e);
              }
              done();
            });
        } else {
          done();
        }
      }.bind(this), this.interval), this.interval);

    } else {
      console.error("Bad poll interval, async listener, or input generator supplied");
    }

    return result;
  },

  /**
   * Indicates if start has been called.
   *
   * @returns {Boolean} true if start has been called, false otherwise.<commit_msg>switch to L_WAIT for interval access check<commit_after>/**
 * index.js
 *
 * Async stuff for html-snapshots
 *   Notifier class
 *
 * Copyright (c) 2013 - 2017, <NAME>, LocalNerve, contributors
 * Licensed under the MIT license.
 */
/* global Promise */
"use strict";

var fs = require("fs");
var crypto = require("crypto");
var AsyncLock = require("async-lock");
var combineErrors = require("combine-errors");
var nodeCall = require("../common/node");

var F_OK = (fs.constants && fs.constants.F_OK) || fs.F_OK;
var L_WAIT = 10000;
var TIMEOUT_PAD_FLOOR = 800;
function NOOP () {}

/**
 * Create the lock factory.
 */
function createLockFactory () {
  // Create the per instance async lock.
  var lock = new AsyncLock();

  // Make a random id
  var rid = crypto.randomBytes(16).toString("hex");

  /**
   * Force a serial execution context.
   *
   * @param {Function} fn - The function to guard.
   * @param {Number} timeout - The max time to wait for the lock.
   */
  return function lockFactory (fn, timeout) {
    return function protectedContext () {
      lock.acquire("cs-guard-" + rid, function (done) {
        fn(function () {
          done(null, 0);
        });
      }, NOOP, {
        timeout: timeout
      });
    };
  };
}

/**
 * Notifier Constructor
 * Polls the outputDir and when all the files exist,
 * calls the listener indicating the snapshots are done.
 */
function Notifier () {

  // Create the serial execution context mechanism.
  this.csFactory = createLockFactory();

  // The private files collection
  // Contains a file and timer: "filename": {timer: timerid}
  // Used for tracking work left to do. When empty, work is done.
  this.files = {};

  // Contains files successfully processed
  this.filesDone = [];

  // true if a timeout occurred, or set by abort
  this.errors = [];

  // the holder of the current failure timeout padding
  this.padTimeout = TIMEOUT_PAD_FLOOR;

  // our reference to the listener
  this.callback = NOOP;

  // our reference to the watcher (an interval id)
  // initial value undefined is important
  // this.watcher;

  // the working pollInterval for this run
  // this.interval

  // flag set by qEmpty callback
  // when the last item from the queue is given to a worker
  this.qempty = false;
}

Notifier.prototype = {
  /**
   * Set the qempty flag to true.
   */
  qEmpty: function qEmpty () {
    this.qempty = true;
  },

  /**
   * Start a watch interval, when a file exists, remove it from our files array.
   * If the files array is empty, call the listener and stop the watch interval.
   *
   * @param {Number} pollInterval - The millisecond poll interval to check for
   * file completion.
   * @param {Function} listener - The callback to notify when complete.
   * @param {Object} input - The inputGenerator.
   * @param {Function} input.EOI - The inputGenerator function that signals end
   * of input.
   * @returns {Boolean} true if started successfully, false otherwise.
   */
  start: function start (pollInterval, listener, input) {
    var result = (
      pollInterval > 0 &&
      typeof listener === "function" &&
      (!!input)
      );

    if (result) {
      if (this.isStarted()) {
        throw new Error("Notifier already started");
      }

      this.callback = listener;
      this.interval = parseInt(pollInterval, 10);

      // Poll the filesystem for the files to exist
      // Checks the child process expected output to determine success or failure
      // if the file exists, then it succeeded.
      this.watcher = setInterval(this.csFactory(function (done) {
        var self = this;
        var eoi = typeof input.EOI === "function" && input.EOI();

        if (eoi) {
          Promise.all(Object.keys(self.files).map(function (file) {
            return nodeCall(fs.access, file, F_OK)
              .then(function () {
                return file;
              })
              .catch(function () {
                return null;
              });
          }))
            .then(function (files) {
              try {
                files.forEach(function (file) {
                  file && self._remove(file, true);
                });

                // if done, exit
                if (self._isDone() && self._closeWatcher()) {
                  setImmediate(function () {
                    self.callback(self.getError(), self.filesDone);
                  });
                }
              } catch (e) {
                console.error(e);
              }
              done();
            });
        } else {
          done();
        }
      }.bind(this), L_WAIT), this.interval);

    } else {
      console.error("Bad poll interval, async listener, or input generator supplied");
    }

    return result;
  },

  /**
   * Indicates if start has been called.
   *
   * @returns {Boolean} true if start has been called, false otherwise.
{'fraction_non_alphanumeric': 0.11569313039901276, 'fraction_numerical': 0.004524886877828055, 'mean_word_length': 2.1452134540750323, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '721828', 'n_tokens_mistral': 2764, 'n_tokens_neox': 2657, 'n_words': 927}