<commit_before><!DOCTYPE html>
<html>
<head lang="en">
    <meta charset="UTF-8">
    <title></title>
    <style>
        .land {
            fill: #6FC480;
            stroke: #000000;
        }

        .graticule {
            fill: none;
            stroke: #777;
            stroke-opacity: .5;
            stroke-width: .5px;
        }

        .tweet.keep {
            fill: blue;
            stroke: #777;
        }

        .tweet.latest {
            fill: red;
            stroke: #777;
        }
    </style>
</head>
<body>

<svg width="840" height="800">
</svg>
<script src="https://cdnjs.cloudflare.com/ajax/libs/d3/3.5.3/d3.min.js" charset="utf-8"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/topojson/1.6.9/topojson.min.js"></script>
<script src="ngeohash.js"></script>

<script>
    var ngeohash = require('ngeohash');

    var width = 840;
    var height = 800;
    var projection = d3.geo.orthographic()
            .scale(300)
            .translate([width/2, height/2])
            .clipAngle(90);

    d3.json("world110.json", function(err, world) {
        console.log("data", world)
        var land = topojson.feature(world, world.objects.land);

        var svg = d3.select("svg");
        var path = d3.geo.path()
                .projection(projection);
        var graticule = d3.geo.graticule()
        svg.append("path")
                .datum(graticule)
                .attr("class", "graticule")
                .attr("d", path);
        svg.append("path")
                .attr("d", path(land))
                .classed("land", true);

        function featureFromTweet(tweet) {
            var geohash = ngeohash.encode(tweet.location.lat, tweet.location.lon, 3);
            var bbox = ngeohash.decode_bbox(geohash);
            var minlat = bbox[0], minlon = bbox[1], maxlat = bbox[2], maxlon = bbox[3];

            return {
                type: "Feature",
                geometry: {
                    type: "Polygon",
                    properties: {},
                    coordinates: [[
                        [minlon, minlat],
                        [minlon, maxlat],
                        [maxlon, maxlat],
                        [maxlon, minlat],
                        [minlon, minlat] ]]
                }
            };
        }

        function pathFromTweet(tweet) {
            return path(featureFromTweet(tweet));
        }

        var allTweets = [];

        function showTweets(tweets) {
            allTweets = allTweets.concat(tweets);
            svg.selectAll(".tweet.keep")
                .data(allTweets, function(tweet) { return tweet.id; })<commit_msg>only add unique new tweets<commit_after><!DOCTYPE html>
<html>
<head lang="en">
    <meta charset="UTF-8">
    <title></title>
    <style>
        .land {
            fill: #6FC480;
            stroke: #000000;
        }

        .graticule {
            fill: none;
            stroke: #777;
            stroke-opacity: .5;
            stroke-width: .5px;
        }

        .tweet.keep {
            fill: blue;
            stroke: #777;
        }

        .tweet.latest {
            fill: red;
            stroke: #777;
        }
    </style>
</head>
<body>

<svg width="840" height="800">
</svg>
<script src="https://cdnjs.cloudflare.com/ajax/libs/d3/3.5.3/d3.min.js" charset="utf-8"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/topojson/1.6.9/topojson.min.js"></script>
<script src="ngeohash.js"></script>

<script>
    var ngeohash = require('ngeohash');

    var width = 840;
    var height = 800;
    var projection = d3.geo.orthographic()
            .scale(300)
            .translate([width/2, height/2])
            .clipAngle(90);

    d3.json("world110.json", function(err, world) {
        console.log("data", world)
        var land = topojson.feature(world, world.objects.land);

        var svg = d3.select("svg");
        var path = d3.geo.path()
                .projection(projection);
        var graticule = d3.geo.graticule()
        svg.append("path")
                .datum(graticule)
                .attr("class", "graticule")
                .attr("d", path);
        svg.append("path")
                .attr("d", path(land))
                .classed("land", true);

        function featureFromTweet(tweet) {
            var geohash = ngeohash.encode(tweet.location.lat, tweet.location.lon, 3);
            var bbox = ngeohash.decode_bbox(geohash);
            var minlat = bbox[0], minlon = bbox[1], maxlat = bbox[2], maxlon = bbox[3];

            return {
                type: "Feature",
                geometry: {
                    type: "Polygon",
                    properties: {},
                    coordinates: [[
                        [minlon, minlat],
                        [minlon, maxlat],
                        [maxlon, maxlat],
                        [maxlon, minlat],
                        [minlon, minlat] ]]
                }
            };
        }

        function pathFromTweet(tweet) {
            return path(featureFromTweet(tweet));
        }

        var allTweetIds = {};
        var allTweets = [];

        function showTweets(tweets) {
            for(var i = 0; i < tweets.length; i++) {
                var tweet = tweets[i];
                if (typeof allTweetIds[tweet.id] === "undefined") {
                    allTweets.push(tweet);
                    allTweetIds[tweet.id] = tweet;
                }
            }

            svg.selectAll(".tweet.keep")
                .data(allTweets, function(tweet) { return tweet.id; })
{'fraction_non_alphanumeric': 0.14288275367515238, 'fraction_numerical': 0.022768017210469702, 'mean_word_length': 1.4267072640278382, 'pattern_counts': {'":': 0, '<': 42, '<?xml version=': 0, '>': 41, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '494461', 'n_tokens_mistral': 1860, 'n_tokens_neox': 1705, 'n_words': 312}