<commit_before>'use strict';

// var rarity = require('rarity');
var log = require('anyfetch-provider').log;
var request = require('supertest');
var async = require('async');
var crypto = require('crypto');
var http = require('https');
var fs = require('fs');


module.exports = function upload(file, anyfetchClient, accessToken, cb) {
  var apiUrl = "https://api.box.com/2.0/";
  var url = "/files/" + file.id;
  var filename = '/tmp/' + crypto.randomBytes(10).toString('hex');

  var document = {
    actions: {},
    data: {},
    document_type: "file",
    user_access: [anyfetchClient.accessToken]
  };

  // file to send
  var fileConfig = {};

  async.waterfall([
    function getFileMetadata(cb) {
      request(apiUrl)
        .get(url)
        .set('Content-Type', 'application/x-www-form-urlencoded')
        .set('Authorization', 'Bearer ' + accessToken)
        .expect(200)
        .end(cb);
    },
    function handleMetada(metadata, cb) {
      metadata = metadata.body;
      // Let's build the file address for the user
      document.identifier = 'https://app.box.com/files/0/f/' + metadata.parent.id + '/1/f_' + metadata.id;
      document.actions.show = document.identifier;
      // do not download the file when it is scheduled for deletion or if it is bigger than 50Mb
      console.log(new Date(metadata.created_at.toString()).getTime(), file.cursor);
      if(!metadata.trashed_at && metadata.size < 52428800 && new Date(metadata.created_at).getTime() > file.cursor) {<commit_msg>removing console<commit_after>'use strict';

// var rarity = require('rarity');
var log = require('anyfetch-provider').log;
var request = require('supertest');
var async = require('async');
var crypto = require('crypto');
var http = require('https');
var fs = require('fs');


module.exports = function upload(file, anyfetchClient, accessToken, cb) {
  var apiUrl = "https://api.box.com/2.0/";
  var url = "/files/" + file.id;
  var filename = '/tmp/' + crypto.randomBytes(10).toString('hex');

  var document = {
    actions: {},
    data: {},
    document_type: "file",
    user_access: [anyfetchClient.accessToken]
  };

  // file to send
  var fileConfig = {};

  async.waterfall([
    function getFileMetadata(cb) {
      request(apiUrl)
        .get(url)
        .set('Content-Type', 'application/x-www-form-urlencoded')
        .set('Authorization', 'Bearer ' + accessToken)
        .expect(200)
        .end(cb);
    },
    function handleMetada(metadata, cb) {
      metadata = metadata.body;
      // Let's build the file address for the user
      document.identifier = 'https://app.box.com/files/0/f/' + metadata.parent.id + '/1/f_' + metadata.id;
      document.actions.show = document.identifier;
      // do not download the file when it is scheduled for deletion or if it is bigger than 50Mb
      if(!metadata.trashed_at && metadata.size < 52428800 && new Date(metadata.created_at).getTime() > file.cursor) {
{'fraction_non_alphanumeric': 0.15728420334356874, 'fraction_numerical': 0.012964858410098942, 'mean_word_length': 3.200573065902579, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 4, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4700226', 'n_tokens_mistral': 929, 'n_tokens_neox': 892, 'n_words': 222}