<commit_before>#!/usr/bin/env node
'use strict';

var data = require('./OA_PR_JSON.json');
var app = require('../../../server/server');
var _ = require('lodash');
var GeoTweet = app.models.GeoTweet;

var docs = data.docs;

var createGeoTweets = docs.map(doc => {
  var latlng = doc.latLon[0].split(',');
  var lat = +latlng[0],
    lng = +latlng[1];

  return GeoTweet.create({
    username: doc.feedName,
    tweet_id: doc.guid,
    full_tweet: doc.originalTitle,
    post_date: doc.datePublished,
    hashtags: _([doc.feedSource, doc.storySource, doc.all_KeyTag, doc.goWord, doc.locationName])
      .flatten().compact().uniq().value(),
    lat: lat,
    lng: lng
  });
});

Promise.all(createGeoTweets)
.then(() => console.log('done'))
.catch(err => {
  GeoTweet.destroyAll();
  console.error(err);
});
<commit_msg>geocode puerto rico data before loading<commit_after>#!/usr/bin/env node
'use strict';

var data = require('./OA_PR_JSON.json');
var app = require('../../../server/server');
var request = require('request-json');
var client = request.createClient('http://localhost:3001/api/geocoder/');
var _ = require('lodash');
var GeoTweet = app.models.GeoTweet;

var docs = data.docs;

function requestPromise(loc, defaults /*i.e. {latitude: 0, longitude: 0}*/ ) {
  return new Promise((resolve, reject) => {
    client.post('forward-geo/', { address: loc }, (err, _, body) => {
      // console.log(body)
      if (err) {
        console.log(err);
        // return reject(err);
      }
      if (!body.length) {
        return resolve([defaults]); // echo back defaults if no results
      }
      resolve(body);
    });
  });
}

// geocode all locations in doc
function geocode(doc) {
  var locationNames = doc.locationName;
  var latLngs = doc.latLon; // array of strings matching order of location names

  var geocodePromises = locationNames.map((loc,i) => {
    var latlng = latLngs[i].split(','),
      lat = latlng[0],
      lng = latlng[1];

    // use the first result. TODO: use confidence score if available?
    return requestPromise(loc, {latitude: lat, longitude: lng})
    .then(results => {
      doc.geo = results[0]; // add geo result to original doc
      return doc;
    });
  });

  return Promise.all(geocodePromises);
}

function createGeoTweet(doc) {
  var geo = doc.geo;
  GeoTweet.create({
    username: doc.feedName,
    tweet_id: doc.guid,
    full_tweet: doc.originalTitle,
    post_date: doc.datePublished,
    hashtags: _([doc.feedSource, doc.storySource, doc.all_KeyTag, doc.goWord, doc.locationName])
      .flatten().compact().uniq().value(),
    lat: geo.latitude,
    lng: geo.longitude
  });
}

function tweetify(docs) {
  // create geotweet record for each location (dupes tweet text)
  docs.forEach(createGeoTweet);
}

// collect all geocode-to-geotweet promises
var processDocs = docs.map(doc => {
  return geocode(doc)
  .then(tweetify);
});

Promise.all(processDocs)
.then(() => console.log('done'))
.catch(err => {
  console.error(err);
  GeoTweet.destroyAll();
  console.error('Removing geotweets records');
});

{'fraction_non_alphanumeric': 0.16677545691906004, 'fraction_numerical': 0.0039164490861618795, 'mean_word_length': 3.4100719424460433, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2249686', 'n_tokens_mistral': 1100, 'n_tokens_neox': 1065, 'n_words': 235}