<commit_before>var userConfig = require('../user-config.json'),
    Twitter = require('ntwitter'),
    fs = require('fs'),
    fse = require('fs.extra'),
    prettyBytes = require('pretty-bytes');


// Return a string containing file size and unit e.g 1 kB
function getFileSize(file,cb) {
  fs.stat(file, function (err, stats) {
    return cb(prettyBytes(stats.size));
  });
}

// Return true if file size is larger than maxSize, otherwise false
function compareFileSize(file,maxSize,cb) {

  console.log("Comparing size " + file + " max size " + maxSize );
  var units = ["B", "kB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"  ];
  getFileSize(file,function(size){

    var fileUnit = size.split(" ");
    var maxUnit = maxSize.split(" ");

    if(units.indexOf(fileUnit[1]) < units.indexOf(maxUnit[1])) {
      return cb(0);
    }
    else if(units.indexOf(fileUnit[1]) === units.indexOf(maxUnit[1]) && Number(fileUnit[0]) <= Number(maxUnit[0])) {
      return cb(0);
    }
    else{
      return cb(1);
    }
  });

}

module.exports = {

  connection: {},
  storage: "",
  maxSize: "",

  initConfig: function(){

    this.connection = {
      "consumer_key": process.env.CONSUMER_KEY || userConfig.twitter.consumer_key,
      "consumer_secret": process.env.CONSUMER_SECRET || userConfig.twitter.consumer_secret,
      "access_token_key": process.env.ACCESS_TOKEN_KEY || userConfig.twitter.access_token_key,
      "access_token_secret": process.env.ACCESS_TOKEN_SECRET || userConfig.twitter.access_token_secret,
      "stream" : userConfig.twitter.streamType.basic,
      "optionalStream": userConfig.twitter.streamType.userSpecific
    }

    this.storage = userConfig.local.storage;
    this.maxSize = userConfig.local.maxSize;
  },
  createClient: function() {
    var client =  new Twitter({
      consumer_key: this.connection.consumer_key,
      consumer_secret: this.connection.consumer_secret,
      access_token_key: this.connection.access_token_key,
      access_token_secret: this.connection.access_token_secret
    });

    return client;
  },

  saveStream: function(client,writeStream,test) {
    var storage = this.storage,
    maxSize = this.maxSize,
    logNumber = 0;

    writeStream.once('open', function(fd) {
      client.stream('statuses/sample', function(stream) {
        stream.on('data', function (data) {
          writeStream.write(JSON.stringify(data)+'\n');
        });
        if(test){
          setTimeout(stream.destroy, 2000);
        }
      });
    });

    fs.watchFile(storage, function (cb) {
      compareFileSize(storage,maxSize,function(res){
        if(res){
          console.log("File size exceeded");
          fse.copy(storage, './'+storage+'.'+logNumber, function (err) {
            if (err) {
              throw err;
            }
            logNumber++;
            fs.truncate(storage, 0, function(){console.log('log copied')})
          });
        }
        else {
          console.log("File size under desired size");
        }
      });
    });

  },

  getConnection: function(){
    return this.connection;
  },

  getStorage: function() {
    return this.storage;
  },

  getCompareFileSize: function(file,maxSize,cb) {
    compareFileSize(file,maxSize,function(ret) {
      return cb(ret);
    });
  },
  testSaveStream: function(writeStream) {
    this.saveStream("test",writeStream);
  }
};<commit_msg>removed test function<commit_after>var userConfig = require('../user-config.json'),
    Twitter = require('ntwitter'),
    fs = require('fs'),
    fse = require('fs.extra'),
    prettyBytes = require('pretty-bytes');


// Return a string containing file size and unit e.g 1 kB
function getFileSize(file,cb) {
  fs.stat(file, function (err, stats) {
    return cb(prettyBytes(stats.size));
  });
}

// Return true if file size is larger than maxSize, otherwise false
function compareFileSize(file,maxSize,cb) {

  console.log("Comparing size " + file + " max size " + maxSize );
  var units = ["B", "kB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"  ];
  getFileSize(file,function(size){

    var fileUnit = size.split(" ");
    var maxUnit = maxSize.split(" ");

    if(units.indexOf(fileUnit[1]) < units.indexOf(maxUnit[1])) {
      return cb(0);
    }
    else if(units.indexOf(fileUnit[1]) === units.indexOf(maxUnit[1]) && Number(fileUnit[0]) <= Number(maxUnit[0])) {
      return cb(0);
    }
    else{
      return cb(1);
    }
  });

}

module.exports = {

  connection: {},
  storage: "",
  maxSize: "",

  initConfig: function(){

    this.connection = {
      "consumer_key": process.env.CONSUMER_KEY || userConfig.twitter.consumer_key,
      "consumer_secret": process.env.CONSUMER_SECRET || userConfig.twitter.consumer_secret,
      "access_token_key": process.env.ACCESS_TOKEN_KEY || userConfig.twitter.access_token_key,
      "access_token_secret": process.env.ACCESS_TOKEN_SECRET || userConfig.twitter.access_token_secret,
      "stream" : userConfig.twitter.streamType.basic,
      "optionalStream": userConfig.twitter.streamType.userSpecific
    }

    this.storage = userConfig.local.storage;
    this.maxSize = userConfig.local.maxSize;
  },
  createClient: function() {
    var client =  new Twitter({
      consumer_key: this.connection.consumer_key,
      consumer_secret: this.connection.consumer_secret,
      access_token_key: this.connection.access_token_key,
      access_token_secret: this.connection.access_token_secret
    });

    return client;
  },

  saveStream: function(client,writeStream,test) {
    var storage = this.storage,
    maxSize = this.maxSize,
    logNumber = 0;

    writeStream.once('open', function(fd) {
      client.stream('statuses/sample', function(stream) {
        stream.on('data', function (data) {
          writeStream.write(JSON.stringify(data)+'\n');
        });
        if(test){
          setTimeout(stream.destroy, 2000);
        }
      });
    });

    fs.watchFile(storage, function (cb) {
      compareFileSize(storage,maxSize,function(res){
        if(res){
          console.log("File size exceeded");
          fse.copy(storage, './'+storage+'.'+logNumber, function (err) {
            if (err) {
              throw err;
            }
            logNumber++;
            fs.truncate(storage, 0, function(){console.log('log copied')})
          });
        }
        else {
          console.log("File size under desired size");
        }
      });
    });

  },

  getConnection: function(){
    return this.connection;
  },

  getStorage: function() {
    return this.storage;
  },

  getCompareFileSize: function(file,maxSize,cb) {
    compareFileSize(file,maxSize,function(ret) {
      return cb(ret);
    });
  },
};
{'fraction_non_alphanumeric': 0.14978238030917004, 'fraction_numerical': 0.0048026414527990394, 'mean_word_length': 2.969029184038118, 'pattern_counts': {'":': 10, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4401120', 'n_tokens_mistral': 2130, 'n_tokens_neox': 2062, 'n_words': 373}