<commit_before>#!/usr/bin/env node

var request = require('request');
var pays = require('../config/pays');
var util = require('util');
var async = require('async');
var database = require('./node-mongodb');

// database.dropCollection();
database.ensureIndex();

REQ_TIMEOUT = 300000;
// Max events per page allowed by SK API = 100
MAX_PER_PAGE = 100;
var apiKey = '<KEY>';
var per_page = '&per_page='+MAX_PER_PAGE;
var api_loc = 'http://api.songkick.com/api/3.0/search/locations.json?'+apiKey+per_page;
var api_eve = 'http://api.songkick.com/api/3.0/events.json?'+apiKey+'&page=%s'+per_page;
var img_cdn = 'http://images.sk-static.com/images/media/profile_images/artists/%s/card_avatar';
var api_geo = 'http://open.mapquestapi.com/nominatim/v1/reverse.php?format=json&lat=%s&lon=%s';
// var api_geo = 'http://nominatim.openstreetmap.org/reverse?format=json&lat=%s&lon=%s';

console.log('\n*** Total number of locations to process : ' + pays.country.length + ' ***\n');
console.log('*** Please be patient while sending requests this may take a while... ****\n');

/*
 * iterating cities
 */
function iterateCityFile(url, pays){
    for(i=0; i<pays.country.length; i++){
        var _url = url + '&query=' + pays.country[i];
        getCityId(_url, pays.country[i]);
    };
}

function getCityId (url, country) {
    options = { uri: url, timeout: REQ_TIMEOUT};
    console.log('Fetching attributes for '+country+' with : '+options+'\n');
    request(options, function(err, res, results){
        if(!err) {
            var r = JSON.parse(results).resultsPage;
            if(r.status=='ok'){
                (r.results.location).forEach(function(item){
                    var city = item.city.displayName;
                    _url = api_eve + '&location=sk:'+ item.metroArea.id;
                    getEventAttr(_url, city);
                });
            }
        }
    });
}

function getEventAttr (url, city_name) {
    // Replace page number placeholder in url
    options = { uri: util.format(url, '1'), timeout: REQ_TIMEOUT};
    request(options, function(err, res, results){
        if(!err){
            var r = (JSON.parse(results)).resultsPage;
            console.log('Fetching attributes for '+city_name+' with : '+options + ' -- totalEntries= '+r.totalEntries);
            // (r.totalEntries/MAX_PER_PAGE)+1 = total pages
            var totalPages = parseInt(r.totalEntries/MAX_PER_PAGE)+1;
            for(p=1; p<totalPages; p++){
                _url = util.format(url, p);
                getEventPerCityId(_url, city_name);
            }
        }
    });
}

function getEventPerCityId (url, city_name) {
    options = { uri: url, timeout: REQ_TIMEOUT};
    // console.log('Fetching events for : ' + city_name + ' with : ' + options+'\n');
    request(options, function(err, res, results){
        if(!err){
            var r = JSON.parse(results).resultsPage;
            if(r.status=='ok'){
                async.map(r.results.event, function eachItem (item, callback){
                    var artist = [];
                    (item.performance).forEach(function(a){
                        artist.push(a.displayName);
                    });
                    reverseGeocode(item.location.lat, item.location.lng, function(err, result){<commit_msg>skip locations where totalEntries = 0<commit_after>#!/usr/bin/env node

var request = require('request');
var pays = require('../config/pays');
var util = require('util');
var async = require('async');
var database = require('./node-mongodb');

// database.dropCollection();
database.ensureIndex();

REQ_TIMEOUT = 300000;
// Max events per page allowed by SK API = 100
MAX_PER_PAGE = 100;
var apiKey = '<KEY>';
var per_page = '&per_page='+MAX_PER_PAGE;
var api_loc = 'http://api.songkick.com/api/3.0/search/locations.json?'+apiKey+per_page;
var api_eve = 'http://api.songkick.com/api/3.0/events.json?'+apiKey+'&page=%s'+per_page;
var img_cdn = 'http://images.sk-static.com/images/media/profile_images/artists/%s/card_avatar';
var api_geo = 'http://open.mapquestapi.com/nominatim/v1/reverse.php?format=json&lat=%s&lon=%s';
// var api_geo = 'http://nominatim.openstreetmap.org/reverse?format=json&lat=%s&lon=%s';

console.log('\n*** Total number of locations to process : ' + pays.country.length + ' ***\n');
console.log('*** Please be patient while sending requests this may take a while... ****\n');

/*
 * iterating cities
 */
function iterateCityFile(url, pays){
    for(i=0; i<pays.country.length; i++){
        var _url = url + '&query=' + pays.country[i];
        getCityId(_url, pays.country[i]);
    };
}

function getCityId (url, country) {
    options = { uri: url, timeout: REQ_TIMEOUT};
    console.log('Fetching attributes for '+country+' with : '+options+'\n');
    request(options, function(err, res, results){
        if(!err) {
            var r = JSON.parse(results).resultsPage;
            if(r.status=='ok'){
                (r.results.location).forEach(function(item){
                    var city = item.city.displayName;
                    _url = api_eve + '&location=sk:'+ item.metroArea.id;
                    getEventAttr(_url, city);
                });
            }
        }
    });
}

function getEventAttr (url, city_name) {
    // Replace page number placeholder in url
    options = { uri: util.format(url, '1'), timeout: REQ_TIMEOUT};
    request(options, function(err, res, results){
        if(!err){
            var r = (JSON.parse(results)).resultsPage;
            console.log('Fetching attributes for '+city_name+' with : '+options + ' -- totalEntries= '+r.totalEntries);
            if(r.totalEntries > 0){
                // (r.totalEntries/MAX_PER_PAGE)+1 = total pages
                var totalPages = parseInt(r.totalEntries/MAX_PER_PAGE)+1;
                for(p=1; p<totalPages; p++){
                    _url = util.format(url, p);
                    getEventPerCityId(_url, city_name);
                }
            }
        }
    });
}

function getEventPerCityId (url, city_name) {
    options = { uri: url, timeout: REQ_TIMEOUT};
    // console.log('Fetching events for : ' + city_name + ' with : ' + options+'\n');
    request(options, function(err, res, results){
        if(!err){
            var r = JSON.parse(results).resultsPage;
            if(r.status=='ok'){
                async.map(r.results.event, function eachItem (item, callback){
                    var artist = [];
                    (item.performance).forEach(function(a){
                        artist.push(a.displayName);
                    });
                    reverseGeocode(item.location.lat, item.location.lng, function(err, result){
{'fraction_non_alphanumeric': 0.16200361010830325, 'fraction_numerical': 0.0069193742478941035, 'mean_word_length': 2.572810317033853, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '463919', 'n_tokens_mistral': 2129, 'n_tokens_neox': 2060, 'n_words': 419}