<commit_before>var config = require('./config.js');
var knex = require('knex')({
  dialect: 'mysql',
  connection: config.mysql,
  debug: false
});

Data = {}

FIELDS_KEEP = {
  user: ['id', 'name', 'screen_name', 'location', 'url', 'description', 'protected', 'verified', 'followers_count', 
  'listed_count', 'favourites_count', 'statuses_count', 'utc_offset', 'time_zone', 'geo_enabled', 'lang',
  'contributors_enabled', 'is_translator', 'default_profile', 'friends_count'],
  tweet: ['id', 'text', 'source', 'truncated', 'coordinates', 'contributors', 'retweet_count',
  'favorite_count', 'entities', 'created_at', 'favorited', 'retweeted', 'filter_level', 'possibly_sensitive', 'lang', 
  'userId', 'flag']
}

var flags = ['angular','bootstrap','famous','famous_angular','ionic','meteor','backbone','bmarrionette',
'd3','ember','enyo','react','hammer','jquerymobile','jquery','kendo','knockout','polymer','sencha','threejs','titanium'];

Data.saveUser = function (tweet){
  return knex('users').insert(_pickFields(tweet.user, FIELDS_KEEP.user))
  .then(function(id){
    //object safely stored
  }, function(error) {
    //error occured, duplicate entry?
    console.log('SAVE USER ERROR', tweet);
  });
}

Data.saveTweet = function (tweet){
  tweet.userId = tweet.user.id;
  //convert string to timestamps
  tweet.created_at = Date.parse(tweet.created_at) * 0.001;
  tweet.entities = tweet.entities.toString();

  var new_tweet = _pickFields(tweet, FIELDS_KEEP.tweet);
  return knex('tweets').insert(new_tweet).then(function(data){
  }, function(error){
    console.log('SAVE TWEET ERROR', tweet);
  });
}

Data.getAllTweets = function () {
  //retrieve tweets for showing.  
  return knex('tweets').select().then(function(data){
    return _formatStream(data, flags);
  });
}

_pickFields = function (object, whitelist) {
  var newObj = {};
  for(var key in whitelist) {
    var fieldName = whitelist[key];<commit_msg>using timestamp_ms instead, created_at is nonuniform<commit_after>var config = require('./config.js');
var knex = require('knex')({
  dialect: 'mysql',
  connection: config.mysql,
  debug: false
});

Data = {}

FIELDS_KEEP = {
  user: ['id', 'name', 'screen_name', 'location', 'url', 'description', 'protected', 'verified', 'followers_count', 
  'listed_count', 'favourites_count', 'statuses_count', 'utc_offset', 'time_zone', 'geo_enabled', 'lang',
  'contributors_enabled', 'is_translator', 'default_profile', 'friends_count'],
  tweet: ['id', 'text', 'source', 'truncated', 'coordinates', 'contributors', 'retweet_count',
  'favorite_count', 'entities', 'created_at', 'favorited', 'retweeted', 'filter_level', 'possibly_sensitive', 'lang', 
  'userId', 'flag']
}

var flags = ['angular','bootstrap','famous','famous_angular','ionic','meteor','backbone','bmarrionette',
'd3','ember','enyo','react','hammer','jquerymobile','jquery','kendo','knockout','polymer','sencha','threejs','titanium'];

Data.saveUser = function (tweet){
  debugger;
  var picked = _pickFields(tweet.user, FIELDS_KEEP.user);
  return knex('users').insert(picked)
  .then(function(id){
    //object safely stored
    debugger;
  }, function(error) {
    debugger;
    //error occured, duplicate entry?
    console.log('SAVE USER ERROR', tweet);
  });
}

Data.saveTweet = function (tweet){
  debugger;
  tweet.userId = tweet.user.id;
  //convert string to timestamps
  tweet.created_at = Math.floor(tweet.timestamp_ms * 0.001);  
  tweet.entities = tweet.entities.toString();

  var new_tweet = _pickFields(tweet, FIELDS_KEEP.tweet);
  debugger;
  return knex('tweets').insert(new_tweet).then(function(data){
  }, function(error){
    debugger;
    console.log('SAVE TWEET ERROR', tweet);
  });
}

Data.getAllTweets = function () {
  //retrieve tweets for showing.  
  return knex('tweets').select().then(function(data){
    return _formatStream(data, flags);
  });
}

_pickFields = function (object, whitelist) {
  var newObj = {};
  for(var key in whitelist) {
    var fieldName = whitelist[key];
{'fraction_non_alphanumeric': 0.18643643643643643, 'fraction_numerical': 0.0025025025025025025, 'mean_word_length': 5.617549668874172, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5018872', 'n_tokens_mistral': 1413, 'n_tokens_neox': 1372, 'n_words': 263}