<commit_before>var Path = require('path');
var Fs = require('fs');
var config = require('../config');
// TODO: when to use relative paths, absolute paths in node ?
var Namer = require('../lib/namer');
var _ = require('underscore')._;

var Q = require('q');

var Twitter = require('../lib/twitterlib');
var JSCR = require('jscr-api/jscr-api');
require('jscr-api/jscr-memory');
require('jscr-git/jscr-git');

/* ========================================================================== */
/* Data transformation utilities */
/* -------------------------------------------------------------------------- */

/** Updates the given resource with values from the GeoJSON object */
function updateResourceFields(resource, geoJson) {
    _.each([ 'properties', 'geometry' ], function(name) {
        var from = geoJson[name] || {};
        var to = resource[name] = resource[name] || {};
        _.each(from, function(value, key) {
            to[key] = value;
        })
    })
    return resource;
}

/** Copies all required fields from the specified resource */
function getGeoJsonFromResource(resource, showSystemProperties) {
    var id = resource.getPath();
    var coordinates = resource.geometry ? _
            .clone(resource.geometry.coordinates) : [];
    // console.log(' * ' + id);
    // console.log(JSON.stringify(resource));
    var properties = _.clone(resource.properties);
    properties.updated = resource.sys.updated || resource.sys.created;
    var result = {
        id : id,
        type : "Feature",
        geometry : {
            "type" : "Point",
            "coordinates" : coordinates
        },
        properties : properties
    };
    if (showSystemProperties !== false) {
        result.sys = _.clone(resource.sys);
    } else {
        delete result.properties.email;
    }
    return result;
}

/**
 * Detects and returns path of a resource corresponding to the given GeoJSON<commit_msg>the updated timestamp added to public export<commit_after>var Path = require('path');
var Fs = require('fs');
var config = require('../config');
// TODO: when to use relative paths, absolute paths in node ?
var Namer = require('../lib/namer');
var _ = require('underscore')._;

var Q = require('q');

var Twitter = require('../lib/twitterlib');
var JSCR = require('jscr-api/jscr-api');
require('jscr-api/jscr-memory');
require('jscr-git/jscr-git');

/* ========================================================================== */
/* Data transformation utilities */
/* -------------------------------------------------------------------------- */

/** Updates the given resource with values from the GeoJSON object */
function updateResourceFields(resource, geoJson) {
    _.each([ 'properties', 'geometry' ], function(name) {
        var from = geoJson[name] || {};
        var to = resource[name] = resource[name] || {};
        _.each(from, function(value, key) {
            to[key] = value;
        })
    })
    return resource;
}

/** Copies all required fields from the specified resource */
function getGeoJsonFromResource(resource, showSystemProperties) {
    var id = resource.getPath();
    var coordinates = resource.geometry ? _
            .clone(resource.geometry.coordinates) : [];
    // console.log(' * ' + id);
    // console.log(JSON.stringify(resource));
    var properties = _.clone(resource.properties);
    var updated = resource.sys.updated || resource.sys.created;
    properties.updated = updated.timestamp;
    
    var result = {
        id : id,
        type : "Feature",
        geometry : {
            "type" : "Point",
            "coordinates" : coordinates
        },
        properties : properties
    };
    if (showSystemProperties !== false) {
        result.sys = _.clone(resource.sys);
    } else {
        delete result.properties.email;
    }
    return result;
}

/**
 * Detects and returns path of a resource corresponding to the given GeoJSON
{'fraction_non_alphanumeric': 0.21247762720531832, 'fraction_numerical': 0.0, 'mean_word_length': 2.553133514986376, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2724092', 'n_tokens_mistral': 1038, 'n_tokens_neox': 1013, 'n_words': 270}