<commit_before>
var Transform = require('stream').Transform,
    geoJsonCenter = require('../../lib/geoJsonCenter'),
    mapper = new Transform({ objectMode: true });

mapper._write = function( data, enc, next ){

  try {

    this.push({
      _index: 'pelias', _type: 'locality', _id: data.properties.qs_loc_lc,
      data: {
        gn_id: data.properties.gs_gn_id || '',
        woe_id: data.properties.woe_id || '',
        boundaries: data.geometry,
        center_point: geoJsonCenter( data.geometry ),
        suggest: data.properties.qs_loc
      }
    });

  } catch( e ) {
<commit_msg>Add skip for records where geometry is too large for ES to handle<commit_after>
var Transform = require('stream').Transform,
    geoJsonCenter = require('../../lib/geoJsonCenter'),
    mapper = new Transform({ objectMode: true });

mapper._write = function( data, enc, next ){

  try {

    // Skip records where geometry is too large for ES to process
    if( data.geometry.coordinates.length > 1000 ){
      throw new Error( 'SKIPPING RECORD - COORDS LENGTH" ' + data.geometry.coordinates.length );
    }

    this.push({
      _index: 'pelias', _type: 'locality', _id: data.properties.qs_loc_lc,
      data: {
        gn_id: data.properties.gs_gn_id || '',
        woe_id: data.properties.woe_id || '',
        boundaries: data.geometry,
        center_point: geoJsonCenter( data.geometry ),
        suggest: data.properties.qs_loc
      }
    });

  } catch( e ) {

{'fraction_non_alphanumeric': 0.1372013651877133, 'fraction_numerical': 0.0027303754266211604, 'mean_word_length': 2.7881136950904395, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5081620', 'n_tokens_mistral': 473, 'n_tokens_neox': 451, 'n_words': 110}