<commit_before>'use strict';

/////// GLOBALS
var image_files_directory = "/var/www/x-gram.gsapp.org/public_html/public/img/instagram/";
var image_files_extension = ".jpg";

var days = ["Sunday", "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday"];
var months = ["January", "February", "March", "April", "May", "June", "July", "August", "September", "October", "November", "December"];

var media_buffer = []; //container array for all fetched Instagram images

var FETCH_TIME = 3000; //fetch every 3 seconds
var FETCHING_FROM_INSTAGRAM_BUSY_FLAG = false; //flag if in the middle of fetching
var DOWNLOADING_FROM_INSTAGRAM_BUSY_FLAG = false; //flag if in the middle of downloading
/////// END GLOBALS

/////// HELPER FUNCTIONS
function pushArray(arr, arr2) {
    arr.push.apply(arr, arr2);
}
/////// END HELPER FUNCTIONS

var mongoose = require('mongoose'),
	fs = require('fs');
  //request = require('request'),
  //_ = require('lodash');

var programs = require('../programs');

// load image model schema
require('../models/image');
var Image = mongoose.model('Image');


/////// INSTAGRAM
var ig = require('instagram-node').instagram();

//import Instagram secrets from json file
var instagram_secrets = require('../secrets.json').instagram;

//set up secrets for Instagram module
ig.use({ client_id: instagram_secrets.client_id,
    client_secret: instagram_secrets.client_secret });
/////// END INSTAGRAM








// test function
exports.test = function(req, res){
  console.log('controllers/images::test()');
  res.send(200);
};







// fetch from Instagram API
exports.fetchByHashtag = function(req, res, tag){
  console.log('searching for tag: ' + tag);

  ig.tag_media_recent(tag, instagram_fetch_handler);

  res.redirect('/edit');
};





var instagram_fetch_handler = function(err, medias, pagination, limit) {
    if(err){
        console.log(err);
    }else{
        console.log('\nFetched ' + medias.length + ' images'.cyan);

        pushArray(media_buffer, medias);

        if(pagination.next && (media_buffer.length < 100)){
          console.log('\nPAGINATION.NEXT()\n');

          console.log('media_buffer.length: '+ media_buffer.length);

          pagination.next(instagram_fetch_handler);
        }else{
          // add all other information
          var clean_buffer = sanitizeArray(media_buffer);

          // store all media in media_buffer[] in db
          saveImagesInArray(clean_buffer);
        }
    }
};

// takes raw Instagram fetch data in an array and adds other data
var sanitizeArray = function(medias){
  var sanitized_array = [];
  for(var m = 0; m < medias.length; m++){
    var obj = {};
    obj.content = medias[m];
    obj.downloaded = false;
    obj.programs = extractPrograms(medias[m].tags);

    sanitized_array.push(obj);
  }

  return sanitized_array;
};

// hunts through tags for programs, return programs that are in tags as arrray in all lowercase
var extractPrograms = function(tags){
  var pgms = [];

  for(var t = 0; t < tags.length; t++){
    for(var p = 0; p < programs.length; p++){
      if(tags[t].toLowerCase() == programs[p].toLowerCase()){
        pgms.push( programs[p].toLowerCase() );
      }
    }
  }

  return pgms;
};


// save array of sanitized media to db
var saveImagesInArray = exports.addArray = function(medias){
  // passes off the recursive saving to a handler function
  saveImagesInArray_handler(medias, 0);
};


// saves all Instagram data in a sanitized array to the db
// only saves new images, skips those already in the db
var saveImagesInArray_handler = exports.saveImagesInArray = function(medias, index){

  // flag to avoid saving the same images again
  var already_in_db = false;

  // hunt for images already in the db
  Image.findOne({ 'content.id': medias[index].content.id}, function(err, image) {
    if (err){
      console.log('saveImagesInArray_handler()::error attempting to get image by instagram id with msg: ' + err);
    }else{
      // don't save it again if already in db
      if(image != null){
        already_in_db = true;

        // recursion
        index++;
        if(index < medias.length){
          saveImagesInArray_handler(medias, index);
        }else{
          console.log('finished saving*******');
        }
      }else{
        // not in db, so save the image
        var image = new Image();

        image.content = medias[index].content;
        image.programs = medias[index].programs;
        image.downloaded = medias[index].downloaded;

        image.save(function(err) {
          if (err) {
            console.log('saveImagesInArray_handler()::error attempting to save new image');
          } else {
            // recursion
            index++;<commit_msg>default visible to true<commit_after>'use strict';

/////// GLOBALS
var image_files_directory = "/var/www/x-gram.gsapp.org/public_html/public/img/instagram/";
var image_files_extension = ".jpg";

var days = ["Sunday", "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday"];
var months = ["January", "February", "March", "April", "May", "June", "July", "August", "September", "October", "November", "December"];

var media_buffer = []; //container array for all fetched Instagram images

var FETCH_TIME = 3000; //fetch every 3 seconds
var FETCHING_FROM_INSTAGRAM_BUSY_FLAG = false; //flag if in the middle of fetching
var DOWNLOADING_FROM_INSTAGRAM_BUSY_FLAG = false; //flag if in the middle of downloading
/////// END GLOBALS

/////// HELPER FUNCTIONS
function pushArray(arr, arr2) {
    arr.push.apply(arr, arr2);
}
/////// END HELPER FUNCTIONS

var mongoose = require('mongoose'),
	fs = require('fs');
  //request = require('request'),
  //_ = require('lodash');

var programs = require('../programs');

// load image model schema
require('../models/image');
var Image = mongoose.model('Image');


/////// INSTAGRAM
var ig = require('instagram-node').instagram();

//import Instagram secrets from json file
var instagram_secrets = require('../secrets.json').instagram;

//set up secrets for Instagram module
ig.use({ client_id: instagram_secrets.client_id,
    client_secret: instagram_secrets.client_secret });
/////// END INSTAGRAM








// test function
exports.test = function(req, res){
  console.log('controllers/images::test()');
  res.send(200);
};







// fetch from Instagram API
exports.fetchByHashtag = function(req, res, tag){
  console.log('searching for tag: ' + tag);

  ig.tag_media_recent(tag, instagram_fetch_handler);

  res.redirect('/edit');
};





var instagram_fetch_handler = function(err, medias, pagination, limit) {
    if(err){
        console.log(err);
    }else{
        console.log('\nFetched ' + medias.length + ' images'.cyan);

        pushArray(media_buffer, medias);

        if(pagination.next && (media_buffer.length < 100)){
          console.log('\nPAGINATION.NEXT()\n');

          console.log('media_buffer.length: '+ media_buffer.length);

          pagination.next(instagram_fetch_handler);
        }else{
          // add all other information
          var clean_buffer = sanitizeArray(media_buffer);

          // store all media in media_buffer[] in db
          saveImagesInArray(clean_buffer);
        }
    }
};

// takes raw Instagram fetch data in an array and adds other data
var sanitizeArray = function(medias){
  var sanitized_array = [];
  for(var m = 0; m < medias.length; m++){
    var obj = {};
    obj.content = medias[m];
    obj.downloaded = false;
    obj.programs = extractPrograms(medias[m].tags);

    sanitized_array.push(obj);
  }

  return sanitized_array;
};

// hunts through tags for programs, return programs that are in tags as arrray in all lowercase
var extractPrograms = function(tags){
  var pgms = [];

  for(var t = 0; t < tags.length; t++){
    for(var p = 0; p < programs.length; p++){
      if(tags[t].toLowerCase() == programs[p].toLowerCase()){
        pgms.push( programs[p].toLowerCase() );
      }
    }
  }

  return pgms;
};


// save array of sanitized media to db
var saveImagesInArray = exports.addArray = function(medias){
  // passes off the recursive saving to a handler function
  saveImagesInArray_handler(medias, 0);
};


// saves all Instagram data in a sanitized array to the db
// only saves new images, skips those already in the db
var saveImagesInArray_handler = exports.saveImagesInArray = function(medias, index){

  // flag to avoid saving the same images again
  var already_in_db = false;

  // hunt for images already in the db
  Image.findOne({ 'content.id': medias[index].content.id}, function(err, image) {
    if (err){
      console.log('saveImagesInArray_handler()::error attempting to get image by instagram id with msg: ' + err);
    }else{
      // don't save it again if already in db
      if(image != null){
        already_in_db = true;

        // recursion
        index++;
        if(index < medias.length){
          saveImagesInArray_handler(medias, index);
        }else{
          console.log('finished saving*******');
        }
      }else{
        // not in db, so save the image
        var image = new Image();

        image.content = medias[index].content;
        image.programs = medias[index].programs;
        image.downloaded = medias[index].downloaded;
        image.visible = true;

        image.save(function(err) {
          if (err) {
            console.log('saveImagesInArray_handler()::error attempting to save new image');
          } else {
            // recursion
            index++;
{'fraction_non_alphanumeric': 0.140101201771031, 'fraction_numerical': 0.0035842293906810036, 'mean_word_length': 3.0439045183290707, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4978366', 'n_tokens_mistral': 3028, 'n_tokens_neox': 2859, 'n_words': 790}