<commit_before>var reqwest = require('reqwest'),
    qs = require('qs'),
    through = require('through');

var osmStream = (function osmMinutely() {
    var s = {};

    // presets
    var baseUrl = 'http://overpass-api.de/',
        minuteStatePath = 'augmented_diffs/state.txt',
        changePath = 'api/augmented_diff?';

    function minuteStateUrl() {
        return baseUrl + minuteStatePath;
    }

    function changeUrl(id) {
        return baseUrl + changePath + qs.stringify({
            id: id, info: 'no', bbox: '-180,-90,180,90'
        });
    }

    function requestState(cb) {
        reqwest({
            url: minuteStateUrl(),
            crossOrigin: true,
            type: 'text',
            success: function(res) {
                cb(null, parseInt(res.response, 10));
            }
        });
    }

    function requestChangeset(state, cb) {
        reqwest({
            url: changeUrl(state),
            crossOrigin: true,
            type: 'xml',
            success: function(res) {
                cb(null, res);
            }
        });
    }

    function parseNode(x) {
        if (!x) return undefined;
        var o = {
            type: x.tagName,
            lat: +x.getAttribute('lat'),
            lon: +x.getAttribute('lon'),
            user: x.getAttribute('user'),
            timestamp: x.getAttribute('timestamp')
        };
        if (o.type == 'way') {
            var bounds = get(x, ['bounds']);
            o.bounds = [
                +bounds.getAttribute('maxlat'),
                +bounds.getAttribute('maxlon'),
                +bounds.getAttribute('minlat'),
                +bounds.getAttribute('minlon')];
        }
        return o;
    }

    function get(x, y) {
        if (!x) return undefined;
        for (var i = 0; i < y.length; i++) {
            var o = x.getElementsByTagName(y[i])[0];
            if (o) return o;
        }
    }

    function run(id, cb) {
        requestChangeset(id, function(err, xml) {
            if (err) return cb([]);
            var actions = xml.getElementsByTagName('action'), a;
            var items = [];
            for (var i = 0; i < actions.length; i++) {
                var o = {};
                a = actions[i];
                o.type = a.getAttribute('type');
                if (o.type == 'modify') {
                    o.old = parseNode(get(get(a, ['old']), ['node', 'way']));
                    o.neu = parseNode(get(get(a, ['new']), ['node', 'way']));
                } else {
                    o.neu = parseNode(get(a, ['node', 'way']));
                }
                if (o.old || o.neu) {
                    items.push(o);
                }<commit_msg>Put deleted objects in 'old' instead of 'neu'.<commit_after>var reqwest = require('reqwest'),
    qs = require('qs'),
    through = require('through');

var osmStream = (function osmMinutely() {
    var s = {};

    // presets
    var baseUrl = 'http://overpass-api.de/',
        minuteStatePath = 'augmented_diffs/state.txt',
        changePath = 'api/augmented_diff?';

    function minuteStateUrl() {
        return baseUrl + minuteStatePath;
    }

    function changeUrl(id) {
        return baseUrl + changePath + qs.stringify({
            id: id, info: 'no', bbox: '-180,-90,180,90'
        });
    }

    function requestState(cb) {
        reqwest({
            url: minuteStateUrl(),
            crossOrigin: true,
            type: 'text',
            success: function(res) {
                cb(null, parseInt(res.response, 10));
            }
        });
    }

    function requestChangeset(state, cb) {
        reqwest({
            url: changeUrl(state),
            crossOrigin: true,
            type: 'xml',
            success: function(res) {
                cb(null, res);
            }
        });
    }

    function parseNode(x) {
        if (!x) return undefined;
        var o = {
            type: x.tagName,
            lat: +x.getAttribute('lat'),
            lon: +x.getAttribute('lon'),
            user: x.getAttribute('user'),
            timestamp: x.getAttribute('timestamp')
        };
        if (o.type == 'way') {
            var bounds = get(x, ['bounds']);
            o.bounds = [
                +bounds.getAttribute('maxlat'),
                +bounds.getAttribute('maxlon'),
                +bounds.getAttribute('minlat'),
                +bounds.getAttribute('minlon')];
        }
        return o;
    }

    function get(x, y) {
        if (!x) return undefined;
        for (var i = 0; i < y.length; i++) {
            var o = x.getElementsByTagName(y[i])[0];
            if (o) return o;
        }
    }

    function run(id, cb) {
        requestChangeset(id, function(err, xml) {
            if (err) return cb([]);
            var actions = xml.getElementsByTagName('action'), a;
            var items = [];
            for (var i = 0; i < actions.length; i++) {
                var o = {};
                a = actions[i];
                o.type = a.getAttribute('type');
                if (o.type == 'modify') {
                    o.old = parseNode(get(get(a, ['old']), ['node', 'way']));
                    o.neu = parseNode(get(get(a, ['new']), ['node', 'way']));
                } else if (o.type == 'delete') {
                    o.old = parseNode(get(get(a, ['old']), ['node', 'way']));
                } else {
                    o.neu = parseNode(get(a, ['node', 'way']));
                }
                if (o.old || o.neu) {
                    items.push(o);
                }
{'fraction_non_alphanumeric': 0.154985549132948, 'fraction_numerical': 0.00541907514450867, 'mean_word_length': 1.4456713780918728, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '578357', 'n_tokens_mistral': 1673, 'n_tokens_neox': 1605, 'n_words': 356}