<commit_before>//var jsonKey = '/Users/dhc4z/Downloads/uvalib-api-firebase-adminsdk-urtjy-b407df0805.json'
var jsonKey = '/home/bamboo/uvalib-api-firebase-adminsdk-urtjy-b407df0805.json'

var fbadmin = require('firebase-admin'),
    fbserviceAccount = require(jsonKey),
    cheerio = require('cheerio'),
    fs = require('fs'),
    dataIn = "",
    moment = require('moment'),
    imagemin = require('imagemin'),
    imageminJpegtran = require('imagemin-jpegtran'),
    imageminPngquant = require('imagemin-pngquant'),    
    imageminWebp = require('imagemin-webp'),
    im = require('imagemagick'),
    gcloud = require('gcloud'),
    request = require('request'),
    fbFiles = {};

var storage = gcloud.storage({
  projectId: 'uvalib-api',
  keyFilename: jsonKey 
});

var bucket = storage.bucket('uvalib-api.appspot.com');

/**
 * Connect & Setup of Firebase DB
 */
fbadmin.initializeApp({
  credential: fbadmin.credential.cert(fbserviceAccount),
  databaseURL: "https://uvalib-api.firebaseio.com"
});

var fbdb = fbadmin.database();
var filesRef = fbdb.ref('files');

console.log('attempt to get known files'); 

// Get all the known files from our api
filesRef.once("value", function(knownFiles){

  fbFiles = knownFiles.val();
  console.log('file count in db currently '+knownFiles.numChildren());

  process.stdin.resume();
  process.stdin.setEncoding('utf8')

  process.stdin.on('data', function(chunk) {
    dataIn += chunk;
  });

  // Read in from stdin (from drupal), then parse and start iterating recursivly
  process.stdin.on('end', function(){
    $ = cheerio.load(dataIn);
    var fileRows = $('.views-row').toArray();
    munchFileRows(fileRows)
  });

});

var getFilesizeInBytes = function(filename) {
    const stats = fs.statSync(filename)
    const fileSizeInBytes = stats.size
    return fileSizeInBytes
}

// Todo, skip known files that haven't been changed
var munchFileRows = function(fileRows){
  row = $( fileRows.shift() ); 
  var uuid = getValue(row, 'views-field-uuid');
  var file = {};
  file.uuid = uuid;
  file.name = getValue(row, 'views-field-filename');
  file.ext = '.'+file.name.replace(/.*\./,'');
  var href = $(row).find('.views-field-filename .field-content a').attr('href');
  file.type = getValue(row, 'views-field-filemime');
  file.created = getTime(row, 'views-field-created');
  file.changed = getTime(row, 'views-field-changed');

  if(fbFiles[uuid] && fbFiles[uuid].changed == file.changed) {
    continueProcessing(fileRows);
  } else {
<commit_msg>quick fix<commit_after>var jsonKey = '/Users/dhc4z/Downloads/uvalib-api-firebase-adminsdk-urtjy-b407df0805.json'
//var jsonKey = '/home/bamboo/uvalib-api-firebase-adminsdk-urtjy-b407df0805.json'

var fbadmin = require('firebase-admin'),
    fbserviceAccount = require(jsonKey),
    cheerio = require('cheerio'),
    fs = require('fs'),
    dataIn = "",
    moment = require('moment'),
    imagemin = require('imagemin'),
    imageminJpegtran = require('imagemin-jpegtran'),
    imageminPngquant = require('imagemin-pngquant'),    
    imageminWebp = require('imagemin-webp'),
    im = require('imagemagick'),
    gcloud = require('gcloud'),
    request = require('request'),
    fbFiles = {};

var storage = gcloud.storage({
  projectId: 'uvalib-api',
  keyFilename: jsonKey 
});

var bucket = storage.bucket('uvalib-api.appspot.com');

/**
 * Connect & Setup of Firebase DB
 */
fbadmin.initializeApp({
  credential: fbadmin.credential.cert(fbserviceAccount),
  databaseURL: "https://uvalib-api.firebaseio.com"
});

var fbdb = fbadmin.database();
var filesRef = fbdb.ref('files');

console.log('attempt to get known files'); 

// Get all the known files from our api
filesRef.once("value", function(knownFiles){

  fbFiles = knownFiles.val();
  console.log('file count in db currently '+knownFiles.numChildren());

  process.stdin.resume();
  process.stdin.setEncoding('utf8')

  process.stdin.on('data', function(chunk) {
    dataIn += chunk;
  });

  // Read in from stdin (from drupal), then parse and start iterating recursivly
  process.stdin.on('end', function(){
    $ = cheerio.load(dataIn);
    var fileRows = $('.views-row').toArray();
    munchFileRows(fileRows)
  });

});

var getFilesizeInBytes = function(filename) {
    const stats = fs.statSync(filename)
    const fileSizeInBytes = stats.size
    return fileSizeInBytes
}

// Todo, skip known files that haven't been changed
var munchFileRows = function(fileRows){
  row = $( fileRows.shift() ); 
  var uuid = getValue(row, 'views-field-uuid');
  var file = {};
  file.uuid = uuid;
  file.name = getValue(row, 'views-field-filename');
  file.ext = '.'+file.name.replace(/.*\./,'');
  var href = $(row).find('.views-field-filename .field-content a').attr('href');
  file.type = getValue(row, 'views-field-filemime');
  file.created = getTime(row, 'views-field-created');
  file.changed = getTime(row, 'views-field-changed');

  if(fbFiles && fbFiles[uuid] && fbFiles[uuid].changed == file.changed) {
    continueProcessing(fileRows);
  } else {

{'fraction_non_alphanumeric': 0.16503886784931235, 'fraction_numerical': 0.006378313733306757, 'mean_word_length': 4.130879345603272, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6669050', 'n_tokens_mistral': 1746, 'n_tokens_neox': 1678, 'n_words': 328}