<commit_before>"use strict";
/**
 * nacarta map application logic
 *
 * @author <NAME> <<EMAIL>>
 */
let map, bb, persons = [], locations = [];

/**
 * Callback for Google Maps to initialize map.
 */
function initMap() {
    let mapOptions = {
        zoom: 8,
        center: new google.maps.LatLng(51.051, 13.735),
        mapTypeId: google.maps.MapTypeId.ROADMAP
    };

    let canvas = jQuery("<div>").attr({id: "map-canvas"}).appendTo("body");
    map = new google.maps.Map(canvas.get(0), mapOptions);
    bb = new google.maps.LatLngBounds();

    loadPersons();
}

/**
 * Collect locations of persons database.
 */
function loadPersons() {
    let promises = [];
    for (let file of files)
        promises.push(jQuery.getJSON(basePath + file + ".json", data => persons.push(...data)));

    Promise.all(promises).then(() => { // wait for all JSON files processed
        for (let person of persons) {
            if (person.birth && person.birth.location) locations.push(person.birth.location);
            if (person.death && person.death.location) locations.push(person.death.location);
            // TODO make birth and death locations selectable individually
        }

        let locationsUnique = Array.from(new Set(locations));
        console.info(locationsUnique);

        // try to get geo-coded address from cache
        let locationsUncached = [];
        for (let location of locationsUnique) {
            if (geocodeCache.hasOwnProperty(location)) {
                let latlng = new google.maps.LatLng(geocodeCache[location].lat, geocodeCache[location].lng);
                handleLocation(latlng, location);
            }
            else locationsUncached.push(location);
        }

        // get remaining locations live and log them to add to cache
        let interval = window.setInterval(function () {
            if (locationsUncached.length < 1) {
                window.clearInterval(interval);
                return;
            }

            let location = locationsUncached.pop();
            new google.maps.Geocoder().geocode({"address": location}, (results, status) => {
                if (status == google.maps.GeocoderStatus.OK) {
                    let coordinates = results[0].geometry.location;
                    console.info('ADD THIS LINE TO geocode-cache.js: "' + location + '": ' + JSON.stringify(coordinates) + ",");
                    handleLocation(coordinates, location);
                } else {
                    console.warn("geocoding status = " + status + " for location: " + location);
                }
            });
        }, 2000);
    });<commit_msg>Add preprocessing to locations<commit_after>"use strict";
/**
 * nacarta map application logic
 *
 * @author <NAME> <<EMAIL>>
 */
let map, bb, persons = [], locations = [];

/**
 * Callback for Google Maps to initialize map.
 */
function initMap() {
    let mapOptions = {
        zoom: 8,
        center: new google.maps.LatLng(51.051, 13.735),
        mapTypeId: google.maps.MapTypeId.ROADMAP
    };

    let canvas = jQuery("<div>").attr({id: "map-canvas"}).appendTo("body");
    map = new google.maps.Map(canvas.get(0), mapOptions);
    bb = new google.maps.LatLngBounds();

    loadPersons();
}

/**
 * Collect locations of persons database.
 */
function loadPersons() {
    let promises = [];
    for (let file of files)
        promises.push(jQuery.getJSON(basePath + file + ".json", data => persons.push(...data)));

    Promise.all(promises).then(() => { // wait for all JSON files processed
        for (let person of persons) {
            if (person.birth && person.birth.location) locations.push(person.birth.location);
            if (person.death && person.death.location) locations.push(person.death.location);
            // TODO make birth and death locations selectable individually
        }

        let locationsUnique = Array.from(new Set(locations));
        console.info(locationsUnique);

        // try to get geo-coded address from cache
        let locationsUncached = [];
        for (const locationRaw of locationsUnique) {
            const location = locationRaw.trim().replace(/\?/, "");
            if (geocodeCache.hasOwnProperty(location)) {
                let latlng = new google.maps.LatLng(geocodeCache[location].lat, geocodeCache[location].lng);
                handleLocation(latlng, location);
            }
            else locationsUncached.push(location);
        }

        // get remaining locations live and log them to add to cache
        let interval = window.setInterval(function () {
            if (locationsUncached.length < 1) {
                window.clearInterval(interval);
                return;
            }

            let location = locationsUncached.pop();
            new google.maps.Geocoder().geocode({"address": location}, (results, status) => {
                if (status == google.maps.GeocoderStatus.OK) {
                    let coordinates = results[0].geometry.location;
                    console.info('ADD THIS LINE TO geocode-cache.js: "' + location + '": ' + JSON.stringify(coordinates) + ",");
                    handleLocation(coordinates, location);
                } else {
                    console.warn("geocoding status = " + status + " for location: " + location);
                }
            });
        }, 2000);
    });
{'fraction_non_alphanumeric': 0.13105253318377266, 'fraction_numerical': 0.006730229949523275, 'mean_word_length': 2.173190984578885, 'pattern_counts': {'":': 4, '<': 13, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1630323', 'n_tokens_mistral': 1500, 'n_tokens_neox': 1410, 'n_words': 350}