<commit_before>var Http = require('http')
,   Promise = require('promise')
,   Fs = require('fs')
,   List = require('immutable').List
,   Url = require('url')
,   Specberus = require("../specberus/lib/validator").Specberus
,   Request = require('request');
require('./const.js');

// Zip a list with another list
// Example: (a, b) zip (c, d) == ((a, c), (b, d))
// This is temporary until progress on
// https://github.com/facebook/immutable-js/issues/51
List.prototype.zip = function(list) {
  return this.map(function (item, index) {
    return List.of(item, list.get(index));
  });
};

var DocumentDownloader = function() {};

DocumentDownloader.fetch = function (url) {
  return new Promise(function (resolve) {
    Http.get(url, function(res) {
      var content = '';

      res.on('data', function(data) { content = content + data; });
      res.on('end', function() { resolve(content); });
    });
  });
};

DocumentDownloader.fetchAll = function (urls) {
  return Promise.all(urls.toArray().map(this.fetch)).then(List);
};

DocumentDownloader.install = function (dest, content) {
  return Promise.denodeify(Fs.writeFile)(dest, content);
};

DocumentDownloader.installAll = function(destsContents) {
  return Promise.all(destsContents.toArray().map(function (e) { // e == [dest, content]
    return DocumentDownloader.install(e.get(0), e.get(1));
  }));
};

DocumentDownloader.fetchAndInstall = function (url, dest, isManifest) {
  var mkdir = Promise.denodeify(Fs.mkdir);

  return new Promise(function (resolve) {
    Fs.exists(dest, function (exists) {
      resolve(exists);
    });
  }).then(function (pathExists) {
    if (!pathExists) {
      return mkdir(dest);
    }
  }).then(function () {
    return DocumentDownloader.fetch(url).then(function (content) {
      if (isManifest) {
        var filenames = DocumentDownloader.getFilenames(content);
        var dests = filenames.set(0, 'Overview.html').map(function (filename) {
          return dest + '/' + filename;
        });
        var urls = filenames.map(function (filename) {
          return Url.resolve(url, filename);
        });
        return DocumentDownloader.fetchAll(urls).then(function (contents) {
          return DocumentDownloader.installAll(dests.zip(contents));
        });
      }
      else {
        return DocumentDownloader.install(dest + '/Overview.html', content);
      }
    });
  });
};

DocumentDownloader.getFilenames = function getFilenames(manifest) {
  return manifest.split('\n').reduce(function (acc, line) {
    var filename = line.split('#')[0].trim();

    if (filename !== '') {
      return acc.push(filename);
    }
    else {
      return acc;
    }
  }, List());
};

exports.DocumentDownloader = DocumentDownloader;

var SpecberusWrapper = function () {};

SpecberusWrapper.validate = function (url) {
  return new Promise(function (resolve, reject) {
    function Sink () {}
    require("util").inherits(Sink, require("events").EventEmitter);

    var sink = new Sink();
    var errors = List();
    var metadata = List();

    sink.on("end-all", function (profilename) {
      resolve({
        errors: errors,
        metadata: metadata
      });
    });

    sink.on("metadata", function (item) {
      metadata = metadata.push(item);
    });

    sink.on("err", function (type, data) {
      data.type = type;
      errors = errors.push(data);
    });

    sink.on("exception", function (exception) {
      reject(new Error(exception.message));
    });

    new Specberus().validate({
      url: url,
      profile: require("../specberus/lib/profiles/WD"),
      events: sink,
      // validation: "recursive",
      validation: "simple-validation",
      noRecTrack: false,
      informativeOnly: false,
      processDocument: "2014"
    });
  });
}

exports.SpecberusWrapper = SpecberusWrapper;<commit_msg>Use a Map to store metadata received from Specberus extraction rather than a List<commit_after>var Http = require('http')
,   Promise = require('promise')
,   Fs = require('fs')
,   List = require('immutable').List
,   Map = require('immutable').Map
,   Url = require('url')
,   Specberus = require("../specberus/lib/validator").Specberus
,   Request = require('request');
require('./const.js');

// Zip a list with another list
// Example: (a, b) zip (c, d) == ((a, c), (b, d))
// This is temporary until progress on
// https://github.com/facebook/immutable-js/issues/51
List.prototype.zip = function(list) {
  return this.map(function (item, index) {
    return List.of(item, list.get(index));
  });
};

var DocumentDownloader = function() {};

DocumentDownloader.fetch = function (url) {
  return new Promise(function (resolve) {
    Http.get(url, function(res) {
      var content = '';

      res.on('data', function(data) { content = content + data; });
      res.on('end', function() { resolve(content); });
    });
  });
};

DocumentDownloader.fetchAll = function (urls) {
  return Promise.all(urls.toArray().map(this.fetch)).then(List);
};

DocumentDownloader.install = function (dest, content) {
  return Promise.denodeify(Fs.writeFile)(dest, content);
};

DocumentDownloader.installAll = function(destsContents) {
  return Promise.all(destsContents.toArray().map(function (e) { // e == [dest, content]
    return DocumentDownloader.install(e.get(0), e.get(1));
  }));
};

DocumentDownloader.fetchAndInstall = function (url, dest, isManifest) {
  var mkdir = Promise.denodeify(Fs.mkdir);

  return new Promise(function (resolve) {
    Fs.exists(dest, function (exists) {
      resolve(exists);
    });
  }).then(function (pathExists) {
    if (!pathExists) {
      return mkdir(dest);
    }
  }).then(function () {
    return DocumentDownloader.fetch(url).then(function (content) {
      if (isManifest) {
        var filenames = DocumentDownloader.getFilenames(content);
        var dests = filenames.set(0, 'Overview.html').map(function (filename) {
          return dest + '/' + filename;
        });
        var urls = filenames.map(function (filename) {
          return Url.resolve(url, filename);
        });
        return DocumentDownloader.fetchAll(urls).then(function (contents) {
          return DocumentDownloader.installAll(dests.zip(contents));
        });
      }
      else {
        return DocumentDownloader.install(dest + '/Overview.html', content);
      }
    });
  });
};

DocumentDownloader.getFilenames = function getFilenames(manifest) {
  return manifest.split('\n').reduce(function (acc, line) {
    var filename = line.split('#')[0].trim();

    if (filename !== '') {
      return acc.push(filename);
    }
    else {
      return acc;
    }
  }, List());
};

exports.DocumentDownloader = DocumentDownloader;

var SpecberusWrapper = function () {};

SpecberusWrapper.validate = function (url) {
  return new Promise(function (resolve, reject) {
    function Sink () {}
    require("util").inherits(Sink, require("events").EventEmitter);

    var sink = new Sink();
    var errors = List();
    var metadata = Map();

    sink.on("end-all", function (profilename) {
      resolve({
        errors: errors,
        metadata: metadata
      });
    });

    sink.on("metadata", function (key, value) {
      metadata = metadata.set(key, value);
    });

    sink.on("err", function (type, data) {
      data.type = type;
      errors = errors.push(data);
    });

    sink.on("exception", function (exception) {
      reject(new Error(exception.message));
    });

    new Specberus().validate({
      url: url,
      profile: require("../specberus/lib/profiles/WD"),
      events: sink,
      // validation: "recursive",
      validation: "simple-validation",
      noRecTrack: false,
      informativeOnly: false,
      processDocument: "2014"
    });
  });
}

exports.SpecberusWrapper = SpecberusWrapper;
{'fraction_non_alphanumeric': 0.16163044875916163, 'fraction_numerical': 0.0025716857400025716, 'mean_word_length': 3.1839698762775686, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 30, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7076016', 'n_tokens_mistral': 2390, 'n_tokens_neox': 2372, 'n_words': 496}