<commit_before>'use strict';

var fs = require('fs');
var path = require('path');
var hostname = require('os').hostname();
var Auth = require('sivart-GCE/Auth');
var gcloud = require('gcloud');
var WriteData = require('sivart-data/WriteBuildData');
var storage = gcloud.storage(Auth);

// Get info
var logDir = process.env.SIVART_BASE_LOG_DIR;
var repoName = process.env.TRAVIS_REPO_SLUG;
var branch = process.env.TRAVIS_BRANCH;
var buildId = process.env.SIVART_BUILD_ID;
var buildNumber = process.env.SIVART_BUILD_NUMBER;

var writeData = new WriteData(repoName);
var bucketname = writeData.getBucketName();

fs.writeFileSync(path.join(logDir, 'environment.json'), JSON.stringify(process.env));

// Save files
var basepath = path.join(branch, buildId, buildNumber);

// Store files
storage.createBucket(bucketname, function(err, bucket) {
  if (err) {
    bucket = storage.bucket(hostname);
  }
  var files = fs.readdirSync(logDir);
  files.forEach(function(file) {
    fs.createReadStream(path.join(logDir, file)).pipe(bucket.file(path.join(basepath, file)).createWriteStream());
  });
  fs.createReadStream('/tmp/user-script.sh').pipe(bucket.file(path.join(basepath, 'user-script.sh')).createWriteStream());
  fs.createReadStream('/tmp/user-script.log').pipe(bucket.file(path.join(basepath, 'user-script.log')).createWriteStream());
  fs.createReadStream('/var/log/startupscript.log').pipe(bucket.file(path.join(basepath, 'startupscript.log')).createWriteStream());
});<commit_msg>fixup<commit_after>'use strict';

var fs = require('fs');
var path = require('path');
var hostname = require('os').hostname();
var Auth = require('sivart-GCE/Auth');
var gcloud = require('gcloud');
var WriteData = require('sivart-data/WriteBuildData');
var storage = gcloud.storage(Auth);

// Get info
var logDir = process.env.SIVART_BASE_LOG_DIR;
var repoName = process.env.TRAVIS_REPO_SLUG;
var branch = process.env.TRAVIS_BRANCH;
var buildId = process.env.SIVART_BUILD_ID;
var buildNumber = process.env.SIVART_BUILD_NUMBER;

var writeData = new WriteData(repoName);
var bucketname = writeData.getBucketName();

fs.writeFileSync(path.join(logDir, 'environment.json'), JSON.stringify(process.env));

// Save files
var basepath = path.join(branch, String(buildId), String(buildNumber));

// Store files
storage.createBucket(bucketname, function(err, bucket) {
  if (err) {
    bucket = storage.bucket(hostname);
  }
  var files = fs.readdirSync(logDir);
  files.forEach(function(file) {
    fs.createReadStream(path.join(logDir, file)).pipe(bucket.file(path.join(basepath, file)).createWriteStream());
  });
  fs.createReadStream('/tmp/user-script.sh').pipe(bucket.file(path.join(basepath, 'user-script.sh')).createWriteStream());
  fs.createReadStream('/tmp/user-script.log').pipe(bucket.file(path.join(basepath, 'user-script.log')).createWriteStream());
  fs.createReadStream('/var/log/startupscript.log').pipe(bucket.file(path.join(basepath, 'startupscript.log')).createWriteStream());
});
{'fraction_non_alphanumeric': 0.16599462365591397, 'fraction_numerical': 0.0, 'mean_word_length': 6.24330900243309, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1112751', 'n_tokens_mistral': 984, 'n_tokens_neox': 993, 'n_words': 158}