<commit_before>function VectorRenderer (leaflet, layers, styles)
{
    // this.tile_base_url = 'http://tile.openstreetmap.us/vectiles-all/';
    // this.tile_base_url = 'http://api-vector-test.mapzen.com/vector/all/';
    this.tile_base_url = 'http://api-vector-dev.mapzen.com/vector/all/';
    // this.tile_base_url = 'http://localhost:8080/all/';

    this.leaflet = leaflet;
    this.layers = layers;
    this.styles = styles;
    this.tiles = {};
}

VectorRenderer.prototype.loadTile = function (coords, div)
{
    var tile_url = this.tile_base_url + coords.z + '/' + coords.x + '/' + coords.y + '.json';
    var key = [coords.x, coords.y, coords.z].join('/');
    var req = new XMLHttpRequest();
    var renderer = this;

    this.removeTile(key);

    var tile = this.tiles[key] = this.tiles[key] || {};
    tile.key = key;
    tile.coords = coords;
    tile.xhr = req;
    tile.timers = {};
    tile.timers.network_start = +new Date();
    tile.loading = true;
    tile.loaded = false;

    req.onload = function () {
        var tile = renderer.tiles[key]; // = {};
        if (tile == null) {
            return;
        }

        tile.layers = JSON.parse(req.response);
        tile.timers.network = +new Date() - tile.timers.network_start; // network/JSON parsing

        div.setAttribute('data-tile-key', tile.key); // tile info for debugging
        div.style.width = '256px';
        div.style.height = '256px';

        // var debug_overlay = document.createElement('div');
        // debug_overlay.textContent = tile.key;
        // debug_overlay.style.position = 'absolute';
        // debug_overlay.style.left = 0;
        // debug_overlay.style.top = 0;
        // debug_overlay.style.color = 'white';
        // div.appendChild(debug_overlay);

        // Extract desired layers from full GeoJSON response
        var layers = {};
        for (var t=0; t < renderer.layers.length; t++) {
            renderer.layers[t].number = t;
            layers[renderer.layers[t].name] = renderer.layers[t].data(tile.layers) || { type: 'FeatureCollection', features: [] };
        }
        tile.layers = layers;

        // Mercator projection for geometry and bounds
        timer = +new Date();
        for (var t in tile.layers) {
            var num_features = tile.layers[t].features.length;
            for (var f=0; f < num_features; f++) {
                var feature = tile.layers[t].features[f];
                feature.geometry.coordinates = Geo.transformGeometry(feature.geometry, function (coordinates) {
                    var m = Geo.latLngToMeters(Point(coordinates[0], coordinates[1]));
                    return [m.x, m.y];
                });
            };
        }
        tile.min = Geo.metersForTile(tile.coords);
        tile.max = Geo.metersForTile({ x: tile.coords.x + 1, y: tile.coords.y + 1, z: tile.coords.z });
        tile.timers.projection = +new Date() - timer; // mercator projection

        tile.loading = false;
        tile.loaded = true;

        // Render
        timer = +new Date();
        renderer.addTile(tile, div);

        tile.timers.rendering = +new Date() - timer; // rendering/geometry prep
        console.log("timers for tile " + tile.key +
            ": network: " + (tile.timers.network) +
            ", projection: " + (tile.timers.projection) +
            ", rendering: " + (tile.timers.rendering)
        );

        renderer.leaflet.layer.tileDrawn(div);
    };
    req.open('GET', tile_url, true /* async flag */);
    req.send();
};

VectorRenderer.prototype.removeTile = function (key)
{
    console.log("tile unload for " + key);
    var tile = renderer.tiles[key];
    if (tile != null && tile.loading == true) {
        console.log("cancel tile load for " + key);
        tile.loaded = false;
        tile.xhr.abort();
        tile.xhr = null;
        tile.loading = false;
    }

    delete this.tiles[key];
};<commit_msg>extract some tile processing to separate functions and generalize timers<commit_after>function VectorRenderer (leaflet, layers, styles)
{
    // this.tile_base_url = 'http://tile.openstreetmap.us/vectiles-all/';
    // this.tile_base_url = 'http://api-vector-test.mapzen.com/vector/all/';
    this.tile_base_url = 'http://api-vector-dev.mapzen.com/vector/all/';
    // this.tile_base_url = 'http://localhost:8080/all/';

    this.leaflet = leaflet;
    this.layers = layers;
    this.styles = styles;
    this.tiles = {};
}

VectorRenderer.prototype.loadTile = function (coords, div)
{
    var tile_url = this.tile_base_url + coords.z + '/' + coords.x + '/' + coords.y + '.json';
    var key = [coords.x, coords.y, coords.z].join('/');
    var req = new XMLHttpRequest();
    var renderer = this;

    this.removeTile(key);

    var tile = this.tiles[key] = this.tiles[key] || {};
    tile.key = key;
    tile.coords = coords;
    tile.xhr = req;
    tile.timers = {};
    tile.timers.network = +new Date();
    tile.loading = true;
    tile.loaded = false;

    req.onload = function () {
        var tile = renderer.tiles[key]; // = {};
        if (tile == null) {
            return;
        }

        tile.layers = JSON.parse(req.response);
        tile.timers.network = +new Date() - tile.timers.network; // network/JSON parsing

        div.setAttribute('data-tile-key', tile.key); // tile info for debugging
        div.style.width = '256px';
        div.style.height = '256px';

        // var debug_overlay = document.createElement('div');
        // debug_overlay.textContent = tile.key;
        // debug_overlay.style.position = 'absolute';
        // debug_overlay.style.left = 0;
        // debug_overlay.style.top = 0;
        // debug_overlay.style.color = 'white';
        // div.appendChild(debug_overlay);

        // Extract desired layers from full GeoJSON response
        renderer.processLayersForTile(tile);

        // Mercator projection for geometry and bounds
        tile.min = Geo.metersForTile(tile.coords);
        tile.max = Geo.metersForTile({ x: tile.coords.x + 1, y: tile.coords.y + 1, z: tile.coords.z });
        renderer.projectTile(tile);

        tile.loading = false;
        tile.loaded = true;

        // Render
        var timer = +new Date();
        renderer.addTile(tile, div);
        tile.timers.rendering = +new Date() - timer; // rendering/geometry prep

        // Processing time
        console.log(
            "timers for " + tile.key + ': [ ' + 
            Object.keys(tile.timers).map(function (t) { return t + ': ' + tile.timers[t]; }).join(', ') + ' ]'
        );

        renderer.leaflet.layer.tileDrawn(div);
    };
    req.open('GET', tile_url, true /* async flag */);
    req.send();
};

VectorRenderer.prototype.removeTile = function (key)
{
    console.log("tile unload for " + key);
    var tile = renderer.tiles[key];
    if (tile != null && tile.loading == true) {
        console.log("cancel tile load for " + key);
        tile.loaded = false;
        tile.xhr.abort();
        tile.xhr = null;
        tile.loading = false;
    }

    delete this.tiles[key];
};

// Processes the tile response to create layers as defined by this renderer
// Can include post-processing to partially filter or re-arrange data, e.g. only including POIs that have names
VectorRenderer.prototype.processLayersForTile = function (tile)
{
    var layers = {};
    for (var t=0; t < renderer.layers.length; t++) {
        renderer.layers[t].number = t;
        layers[renderer.layers[t].name] = renderer.layers[t].data(tile.layers) || { type: 'FeatureCollection', features: [] };
    }
    tile.layers = layers;
    return tile;
};

VectorRenderer.prototype.projectTile = function (tile)
{
    var timer = +new Date();
    for (var t in tile.layers) {
        var num_features = tile.layers[t].features.length;
        for (var f=0; f < num_features; f++) {
            var feature = tile.layers[t].features[f];
            feature.geometry.coordinates = Geo.transformGeometry(feature.geometry, function (coordinates) {
                var m = Geo.latLngToMeters(Point(coordinates[0], coordinates[1]));
                return [m.x, m.y];
            });
        };
    }
    tile.timers.projection = +new Date() - timer;
    return tile;
};
{'fraction_non_alphanumeric': 0.15158536585365853, 'fraction_numerical': 0.004390243902439025, 'mean_word_length': 2.2492076069730587, 'pattern_counts': {'":': 1, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6230384', 'n_tokens_mistral': 2633, 'n_tokens_neox': 2560, 'n_words': 539}