<commit_before>var amazon = AWSSum.load('amazon/amazon');
var S3 = AWSSum.load('amazon/s3', 'S3');
var Fiber = Npm.require("fibers");
var Future = Npm.require("fibers/future");
var child_process = Npm.require("child_process");

var WAREHOUSE_BUCKET = "meteor-warehouse";

var die = function (msg) {
  console.error(msg);
  process.exit(1);
};

var doOrDie = function (errorMessage, f) {
  try {
    return f();
  } catch (e) {
    die(errorMessage);
  }
};

// accumulated while running, printed at the end
var publishedArtifacts = [];

var getFromSettings = function (key) {
  var val = Meteor.settings[key];
  if (!val) {
    die("Run via publish-release.sh");
  }
  return val;
};

// gets git sha passed in via --settings
var getGitSha = function () {
  return getFromSettings('git-sha');
};

// gets release name passed in via --settings
var getReleaseName = function () {
  return getFromSettings('release-name');
};

// runs a command, returns stdout.
// XXX should we have a smart package for these? 'process'?
var execFileSync = function (binary, args) {
  return Future.wrap(function(cb) {
    var cb2 = function(err, stdout, stderr) { cb(err, stdout); };
    child_process.execFile(binary, args, cb2);
  })().wait();
};

// returns usable S3 object
var configureS3 = function () {
  // calls 's3cmd --dump-config', returns {accessKey: ..., secretKey: ...}
  var getS3Credentials = function () {
    var unparsedConfig = execFileSync("s3cmd", ["--dump-config"]);
    var accessKey = /access_key = (.*)/.exec(unparsedConfig)[1];
    var secretKey = /secret_key = (.*)/.exec(unparsedConfig)[1];
    return {accessKey: accessKey, secretKey: secretKey};
  };

  var s3Credentials = getS3Credentials();
  var s3 = new S3({
    accessKeyId: s3Credentials.accessKey,
    secretAccessKey: s3Credentials.secretKey,
    region: amazon.US_EAST_1
  });

  return s3;
};

// fetch release manifest
var getManifest = function(s3, gitSha) {
  var manifestMetas = list3FilesWithPrefix(
    s3, ["unpublished", gitSha, "release.json-"].join("/"));
  var manifests = _.map(manifestMetas, function (meta) {
    return s3.GetObject({
      BucketName: WAREHOUSE_BUCKET,
      ObjectName: meta.Key
    }).Body;
  });

  manifests = _.uniq(manifests);
  if (manifests.length !== 1) {
    die("Manifests are not identical!" + manifests);
  }

  return manifests[0];
};

// are there any files with this prefix?
var anyWithPrefix = function(s3, prefix) {
  var files = s3.ListObjects({
    BucketName: WAREHOUSE_BUCKET,
    Prefix: prefix
  });
  return !_.isEmpty(files.Body.ListBucketResult.Contents);
};

var list3FilesWithPrefix = function (s3, prefix) {
  var artifacts = s3.ListObjects({
    BucketName: WAREHOUSE_BUCKET,
    Prefix: prefix
  }).Body.ListBucketResult.Contents;

  // We support 3 platforms.
  if (artifacts.length !== 3)
    throw new Error("Expected three artifacts with prefix " + prefix +
                    ", found " + artifacts.length);
  return artifacts;
};

var copy3FilesWithPrefix = function (s3, prefix, destDir) {<commit_msg>Better error when releasing an unbuilt git sha<commit_after>var amazon = AWSSum.load('amazon/amazon');
var S3 = AWSSum.load('amazon/s3', 'S3');
var Fiber = Npm.require("fibers");
var Future = Npm.require("fibers/future");
var child_process = Npm.require("child_process");

var WAREHOUSE_BUCKET = "meteor-warehouse";

var die = function (msg) {
  console.error(msg);
  process.exit(1);
};

var doOrDie = function (errorMessage, f) {
  try {
    return f();
  } catch (e) {
    die(errorMessage);
  }
};

// accumulated while running, printed at the end
var publishedArtifacts = [];

var getFromSettings = function (key) {
  var val = Meteor.settings[key];
  if (!val) {
    die("Run via publish-release.sh");
  }
  return val;
};

// gets git sha passed in via --settings
var getGitSha = function () {
  return getFromSettings('git-sha');
};

// gets release name passed in via --settings
var getReleaseName = function () {
  return getFromSettings('release-name');
};

// runs a command, returns stdout.
// XXX should we have a smart package for these? 'process'?
var execFileSync = function (binary, args) {
  return Future.wrap(function(cb) {
    var cb2 = function(err, stdout, stderr) { cb(err, stdout); };
    child_process.execFile(binary, args, cb2);
  })().wait();
};

// returns usable S3 object
var configureS3 = function () {
  // calls 's3cmd --dump-config', returns {accessKey: ..., secretKey: ...}
  var getS3Credentials = function () {
    var unparsedConfig = execFileSync("s3cmd", ["--dump-config"]);
    var accessKey = /access_key = (.*)/.exec(unparsedConfig)[1];
    var secretKey = /secret_key = (.*)/.exec(unparsedConfig)[1];
    return {accessKey: accessKey, secretKey: secretKey};
  };

  var s3Credentials = getS3Credentials();
  var s3 = new S3({
    accessKeyId: s3Credentials.accessKey,
    secretAccessKey: s3Credentials.secretKey,
    region: amazon.US_EAST_1
  });

  return s3;
};

// fetch release manifest
var getManifest = function(s3, gitSha) {
  var manifestMetas = list3FilesWithPrefix(
    s3, ["unpublished", gitSha, "release.json-"].join("/"));
  if (! manifestMetas)
    die("A build for " + gitSha + " can't be found on S3. Maybe you forgot to build it on Jenkins?");

  var manifests = _.map(manifestMetas, function (meta) {
    return s3.GetObject({
      BucketName: WAREHOUSE_BUCKET,
      ObjectName: meta.Key
    }).Body;
  });

  manifests = _.uniq(manifests);
  if (manifests.length !== 1) {
    die("Manifests are not identical!" + manifests);
  }

  return manifests[0];
};

// are there any files with this prefix?
var anyWithPrefix = function(s3, prefix) {
  var files = s3.ListObjects({
    BucketName: WAREHOUSE_BUCKET,
    Prefix: prefix
  });
  return !_.isEmpty(files.Body.ListBucketResult.Contents);
};

var list3FilesWithPrefix = function (s3, prefix) {
  var artifacts = s3.ListObjects({
    BucketName: WAREHOUSE_BUCKET,
    Prefix: prefix
  }).Body.ListBucketResult.Contents;

  if (! artifacts)
    return null;

  // We support 3 platforms.
  if (artifacts.length !== 3)
    throw new Error("Expected three artifacts with prefix " + prefix +
                    ", found " + artifacts.length);
  return artifacts;
};

var copy3FilesWithPrefix = function (s3, prefix, destDir) {
{'fraction_non_alphanumeric': 0.14383343928798475, 'fraction_numerical': 0.011602034329307056, 'mean_word_length': 3.6614814814814816, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7146912', 'n_tokens_mistral': 2112, 'n_tokens_neox': 2059, 'n_words': 550}