<commit_before>var bunyan = require('bunyan');             // for log
var Twitter = require('twitter');           // for twitter client api
var solr = require('solr-client');          // for solr-client
var cfg = require('./conf/cfg.json');       // general configuration file

var to_solr = require('./field_tweet2solr.json');

var log = bunyan.createLogger({name: 'tweets2solr'});

function process(o_key,o_value,p_value,params) {


  if (o_value !== undefined) {
    if (p_value.search(/_dt$/) != -1) {
      var date = new Date(o_value);
      o_value = date.toISOString();
    }
    else if (p_value.search(/_p$/) != -1) {
      o_value = o_value[0]+','+o_value[1];
    }
    else if (p_value.search(/^(coordinates_s|geo_s)$/) != -1) {

      o_value = o_value[0]+','+o_value[1];
      //console.log(p_value+" =!! "+o_value);
      //o_value = JSON.stringify(o_value);

    }
    else if (p_value.search(/_(coordinates_s)$/) != -1) {

      //o_value = o_value[0]+','+o_value[1];
      o_value = JSON.stringify(o_value);
      //console.log("add: p:'"+p_value+"' o:'"+o_value);

    }
    params[p_value] = o_value;

  }

}

function traverse(o,p,params,func) {
  for (var i in o) {
    //console.log("traverse:  o_i:"+o[i] + " i:"+i+ " ->p_i:"+p[i]);
    if (i == 0) {
      //console.log('coordinates break');
      break;
    };

    if (p[i] === undefined ||p[i] == null) {
      //console.log('undefined continue');
      continue;
    };   // complete the field_tweet2solr.json if needed


    if ((typeof(p[i])!="object")&& (typeof(o[i])!="object"||(p[i].search(/(coordinates|geo)_(s|p)$/) != -1)) && o[i]!=null) {<commit_msg>add a true geo data for search in solr<commit_after>var bunyan = require('bunyan');             // for log
var Twitter = require('twitter');           // for twitter client api
var solr = require('solr-client');          // for solr-client
var cfg = require('./conf/cfg.json');       // general configuration file

var to_solr = require('./field_tweet2solr.json');

var log = bunyan.createLogger({name: 'tweets2solr'});

function process(o_key,o_value,p_value,params) {


  if (o_value !== undefined) {
    if (p_value.search(/_dt$/) != -1) {
      var date = new Date(o_value);
      o_value = date.toISOString();
    }
    else if (p_value.search(/_p$/) != -1) {
      o_value = o_value[0]+','+o_value[1];
    }
    else if (p_value.search(/^(coordinates_s|geo_s)$/) != -1) {

      o_value = o_value[0]+','+o_value[1];
      //console.log(p_value+" =!! "+o_value);
      //o_value = JSON.stringify(o_value);

    }
    else if (p_value.search(/_(coordinates_s)$/) != -1) {

      //o_value = o_value[0]+','+o_value[1];
      o_value = JSON.stringify(o_value);
      //console.log("add: p:'"+p_value+"' o:'"+o_value);

    }
    params[p_value] = o_value;
    if (p_value == 'geo_s') {
      params['geo_p'] = o_value;
    }
    if (p_value == 'coordinates_s') {
      params['coordinates_p'] = o_value;
    }

  }

}

function traverse(o,p,params,func) {
  for (var i in o) {
    //console.log("traverse:  o_i:"+o[i] + " i:"+i+ " ->p_i:"+p[i]);
    if (i == 0) {
      //console.log('coordinates break');
      break;
    };

    if (p[i] === undefined ||p[i] == null) {
      //console.log('undefined continue');
      continue;
    };   // complete the field_tweet2solr.json if needed


    if ((typeof(p[i])!="object")&& (typeof(o[i])!="object"||(p[i].search(/(coordinates|geo)_(s|p)$/) != -1)) && o[i]!=null) {
{'fraction_non_alphanumeric': 0.2141823444283647, 'fraction_numerical': 0.008683068017366137, 'mean_word_length': 2.8442714126807562, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5725479', 'n_tokens_mistral': 1345, 'n_tokens_neox': 1315, 'n_words': 234}