<commit_before>// an object that represents a set of streams, and metadata about
// those streams.

// requires d3.js and lodash
StreamSet = (function() {

function StreamSet(activities, xhrContext, resolution) {
    this.xhr_ = xhrContext;
    this.activities_ = activities;
    this.resolution = resolution;
}

// Get a stream and attach it to the activity. Returns a promise that
// resolves to a copy of the activity, and a 'stream' property with
// stream data.
StreamSet.prototype.load_stream = function(activity) {
    var activity_id = activity.activity_id;
    if (activity.stream)
        return Promise.resolve(activity);
    return this.xhr_('/api/streams', {activity_id: activity_id,
                                      resolution: this.resolution })
        .then(function(streams) {
            var stream = streams.result.streams[activity_id];
            var geojson = {
                    type: 'Feature',
                    geometry: {
                        type: 'LineString',
                        coordinates: stream.latlng.data.map(function(latlng, i) {
                            // geojson uses lnglat
                            return [latlng[1], latlng[0],
                                    stream.altitude.data[i]];
                        })
                    }
                };
            stream.geojson = geojson;
            //console.log("Making geojson: ", stream.geojson);

            activity.stream = stream;
            return activity;
        });
};


// Returns a promise that resolves the an array of all activities
StreamSet.prototype.load_streams = function(activities) {
    var results = [];
    for (var i = 0; i < activities.length; ++i) {
        results.push(this.load_stream(activities[i]));
    }
    return Promise.all(results)
        .then(function(e) {
            console.log("Streams loaded (", results.length, ')');
            return e;
        });
};


StreamSet.prototype.ready = function() {
    if (!this.ready_) {
        this.ready_ =
            this.load_streams(this.activities_)
            .then(index_streams)
            .then(this.consume_index.bind(this))
            .catch(function(ex) {
                console.error("Error loading and indexing: ", ex);
            });
    }
    return this.ready_;
};

function join_geojsons(geojsons) {
    return {
        type: 'FeatureCollection',
        features: geojsons,
    };
}

// xxx need a better name
StreamSet.prototype.consume_index = function(stream_indexes) {

    this.features = join_geojsons(stream_indexes.map(function(metadata) {
        return metadata.geojson;
    }));

    var extents = this.extents_ = {
        min_lat: d3.min(stream_indexes,
                        function(d) { return d.domain.lat[0]; }),
        max_lat: d3.max(stream_indexes,
                        function(d) { return d.domain.lat[1]; }),
        min_lng: d3.min(stream_indexes,
                        function(d) { return d.domain.lng[0]; }),
        max_lng: d3.max(stream_indexes,
                        function(d) { return d.domain.lng[1]; }),
        min_alt: d3.min(stream_indexes,
                        function(d) { return d.domain.alt[0]; }),
        max_alt: d3.max(stream_indexes,
                        function(d) { return d.domain.alt[1]; }),
    };

    this.scale_z = d3.scale.linear().domain([extents.min_alt,
                                             extents.max_alt]);
    this.generate_proximity_streams(20);
};

StreamSet.prototype.activities = function() {
    return this.activities_;
};

StreamSet.prototype.withGeoData = function(callback) {
    return this.activities_.map(function(activity, i) {
        var coordinates = activity.stream.geojson.geometry.coordinates;
        return coordinates.map(function(coordinates) {
            return callback.call(this,
                                 coordinates[1], // lat
                                 coordinates[0], // lng
                                 coordinates[2], // alt
                                 i);
         });
    });
};

// adds a 'proximity' stream to each activity
// A proximity stream is
StreamSet.prototype.generate_proximity_streams = function(n) {
    // first index the 3d space
    var bucket_lng = d3.scale.linear().domain([this.extents_.min_lng,
                                               this.extents_.max_lng])
            .rangeRound([0,n]).clamp(true);
    var bucket_lat = d3.scale.linear().domain([this.extents_.min_lng,
                                               this.extents_.max_lng])
            .rangeRound([0,n]).clamp(true);
    var bucket_z = this.scale_z.copy().rangeRound([0, n]).clamp(true);

    var bucketCount = {};
    function inc(lat, lng, alt, id) {
        lng = bucket_lng(lng);
        lat = bucket_lat(lat);
        var z = bucket_z(alt);
        if (!(lng in bucketCount))
            bucketCount[lng] = {};
        if (!(lat in bucketCount[lng]))
            bucketCount[lng][lat] = {};
        if (!(z in bucketCount[lng][lat]))
            bucketCount[lng][lat][z] = d3.set();
        bucketCount[lng][lat][z].add(id);
    }
    function val(lat, lng, alt) {
        lng = bucket_lng(lng);
        lat = bucket_lat(lat);
        var z = bucket_z(alt);
        return bucketCount[lng][lat][z].size();
    }
    // this is kind of a map-reduce style index: count up all
    // lat/lng/altitude combos, then redistribute the counts out to
    // each stream.

    // Accumulate counts (map)
    this.withGeoData(inc);

    // Now summarize (reduce)
    var proximities = this.withGeoData(val);
    console.log("Made proximities: ", proximities);<commit_msg>generalize to use geojson for proximity matrix<commit_after>// an object that represents a set of streams, and metadata about
// those streams.

// requires d3.js and lodash
StreamSet = (function() {

function StreamSet(activities, xhrContext, resolution) {
    this.xhr_ = xhrContext;
    this.activities_ = activities;
    this.resolution = resolution;
}

// Get a stream and attach it to the activity. Returns a promise that
// resolves to a copy of the activity, and a 'stream' property with
// stream data.
StreamSet.prototype.load_stream = function(activity) {
    var activity_id = activity.activity_id;
    if (activity.stream)
        return Promise.resolve(activity);
    return this.xhr_('/api/streams', {activity_id: activity_id,
                                      resolution: this.resolution })
        .then(function(streams) {
            var stream = streams.result.streams[activity_id];
            var geojson = {
                    type: 'Feature',
                    geometry: {
                        type: 'LineString',
                        coordinates: stream.latlng.data.map(function(latlng, i) {
                            // geojson uses lnglat
                            return [latlng[1], latlng[0],
                                    stream.altitude.data[i]];
                        })
                    }
                };
            stream.geojson = geojson;
            //console.log("Making geojson: ", stream.geojson);

            activity.stream = stream;
            return activity;
        });
};


// Returns a promise that resolves the an array of all activities
StreamSet.prototype.load_streams = function(activities) {
    var results = [];
    for (var i = 0; i < activities.length; ++i) {
        results.push(this.load_stream(activities[i]));
    }
    return Promise.all(results)
        .then(function(e) {
            console.log("Streams loaded (", results.length, ')');
            return e;
        });
};


StreamSet.prototype.ready = function() {
    if (!this.ready_) {
        this.ready_ =
            this.load_streams(this.activities_)
            .then(index_streams)
            .then(this.consume_index.bind(this))
            .catch(function(ex) {
                console.error("Error loading and indexing: ", ex);
            });
    }
    return this.ready_;
};

function join_geojsons(geojsons) {
    return {
        type: 'FeatureCollection',
        features: geojsons,
    };
}

// xxx need a better name
StreamSet.prototype.consume_index = function(stream_indexes) {

    this.features = join_geojsons(stream_indexes.map(function(metadata) {
        return metadata.geojson;
    }));

    var extents = this.extents_ = {
        min_lat: d3.min(stream_indexes,
                        function(d) { return d.domain.lat[0]; }),
        max_lat: d3.max(stream_indexes,
                        function(d) { return d.domain.lat[1]; }),
        min_lng: d3.min(stream_indexes,
                        function(d) { return d.domain.lng[0]; }),
        max_lng: d3.max(stream_indexes,
                        function(d) { return d.domain.lng[1]; }),
        min_alt: d3.min(stream_indexes,
                        function(d) { return d.domain.alt[0]; }),
        max_alt: d3.max(stream_indexes,
                        function(d) { return d.domain.alt[1]; }),
    };

    this.scale_z = d3.scale.linear().domain([extents.min_alt,
                                             extents.max_alt]);
    this.generate_proximity_streams(20);
};

StreamSet.prototype.activities = function() {
    return this.activities_;
};

StreamSet.prototype.withGeoData = function(callback) {
    return this.activities_.map(function(activity, activity_index) {
        var coordinates = activity.stream.geojson.geometry.coordinates;
        return coordinates.map(function(coordinates, coord_index) {
            return callback.call(this,
                                 coordinates[1], // lat
                                 coordinates[0], // lng
                                 coordinates[2], // alt
                                 activity_index,
                                 coord_index);
         });
    });
};

  /**
   * A lazy deep dictionary. Pass keys as arguments, and dicts will be
   * created lazily along the way.
   */
  DB = DeepBucket;
  function DeepBucket(bucket) {
    // chop off 'bucket'
    var keys = Array.prototype.slice.call(arguments, 1);
    for (var i = 0; i < keys.length; i++) {
      var key = keys[i];
      if (!(key in bucket)) {
        bucket[key] = {};
      }
      bucket = bucket[key];
    }
    return bucket;
  }

// adds a 'proximity' stream to each activity
// A proximity stream is
StreamSet.prototype.generate_proximity_streams = function(n) {
    // first index the 3d space
    var bucket_lng = d3.scale.linear().domain([this.extents_.min_lng,
                                               this.extents_.max_lng])
            .rangeRound([0,n]).clamp(true);
    var bucket_lat = d3.scale.linear().domain([this.extents_.min_lng,
                                               this.extents_.max_lng])
            .rangeRound([0,n]).clamp(true);
    var bucket_z = this.scale_z.copy().rangeRound([0, n]).clamp(true);

    var bucketCount = {};
    function inc(lat, lng, alt, activity_index, coord_index) {
      var blng = bucket_lng(lng);
      var blat = bucket_lat(lat);
      var bz = bucket_z(alt);
      //var unique_id = activity_index + '_' + coord_index;
      var bucket = DeepBucket(bucketCount, blng, blat, bz,
                              activity_index, coord_index);
      bucket['coord'] = [lng, lat, alt];
    }
    function val(lat, lng, alt) {
        lng = bucket_lng(lng);
        lat = bucket_lat(lat);
        var z = bucket_z(alt);
        return Object.keys(DeepBucket(bucketCount, lng, lat, z)).length;
    }
    // this is kind of a map-reduce style index: count up all
    // lat/lng/altitude combos, then redistribute the counts out to
    // each stream.

    // Accumulate counts (map)
  this.withGeoData(inc);
  this.bucketCount = bucketCount;

  console.log("Used bucket: ", bucketCount);
    // Now summarize (reduce)
    var proximities = this.withGeoData(val);
    console.log("Made proximities: ", proximities);
{'fraction_non_alphanumeric': 0.11990608753982894, 'fraction_numerical': 0.004947174241153781, 'mean_word_length': 1.8601918465227818, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 27, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3676715', 'n_tokens_mistral': 3492, 'n_tokens_neox': 3422, 'n_words': 738}