<commit_before>import model
import datetime
import urllib2
import StringIO
import gzip
import time
from lxml import etree

def isoToDatetime(s):
    """Parse a ISO8601-formatted string to a Python datetime."""
    if s is None:
        return s
    else:
        return datetime.datetime.strptime(s, "%Y-%m-%dT%H:%M:%SZ")

def maybeInt(s):
    return int(s) if s is not None else s

def maybeFloat(s):
    return float(s) if s is not None else s

def maybeBool(s):
    if s is not None:
        if s == 'true':
            return True
        else:
            return False
    else:
        return s

def readState(state_file, sep='='):
    state = {}

    for line in state_file:
        if line == '---':
            continue
        if line[0] == '#':
            continue
        (k, v) = line.split(sep)
        state[k] = v.strip().replace("\\:", ":")

    return state

def iter_changeset_stream(start_sqn=None, base_url='http://planet.openstreetmap.org/replication/changesets', expected_interval=60, parse_timestamps=True):
    """Start processing an OSM changeset stream and yield one (action, primitive) tuple
    at a time to the caller."""

    # This is a lot like the other osm_stream except there's no
    # state file for each of the diffs, so just push ahead until
    # we run into a 404.

    # If no start_sqn, assume to start from the most recent changeset file
    if not start_sqn:
        u = urllib2.urlopen('%s/state.yaml' % base_url)
        state = readState(u, ': ')
        sequenceNumber = state['sequence']
    else:
        sequenceNumber = start_sqn

    interval_fudge = 0.0
    while True:
        sqnStr = str(sequenceNumber).zfill(9)
        url = '%s/%s/%s/%s.osm.gz' % (base_url, sqnStr[0:3], sqnStr[3:6], sqnStr[6:9])

        delay = 1.0
        while True:
            print 'Fetching %s' % url
            try:
                content = urllib2.urlopen(url)
                content = StringIO.StringIO(content.read())
                gzipper = gzip.GzipFile(fileobj=content)
                interval_fudge -= (interval_fudge / 2.0)
                break
            except urllib2.HTTPError, e:
                if e.code == 404:
                    print "%s doesn't exist yet. Waiting %.1f seconds." % (url, delay)
                    time.sleep(delay)
                    delay = min(delay * 2, 13)
                    interval_fudge += delay

        obj = None
        for event, elem in etree.iterparse(gzipper, events=('start', 'end')):
            if event == 'start':
                if elem.tag == 'changeset':
                    obj = model.Changeset(
                        int(elem.attrib['id']),
                        isoToDatetime(elem.attrib.get('created_at')) if parse_timestamps else elem.attrib.get('created_at'),
                        isoToDatetime(elem.attrib.get('closed_at')) if parse_timestamps else elem.attrib.get('closed_at'),
                        maybeBool(elem.attrib['open']),
                        maybeFloat(elem.get('min_lat')),
                        maybeFloat(elem.get('max_lat')),
                        maybeFloat(elem.get('min_lon')),
                        maybeFloat(elem.get('max_lon')),
                        elem.attrib.get('user'),
                        maybeInt(elem.attrib.get('uid')),
                        []
                    )
                elif elem.tag == 'tag':
                    obj.tags.append(
                        model.Tag(
                            elem.attrib['k'],
                            elem.attrib['v']
                        )
                    )
            elif event == 'end':
                if elem.tag == 'changeset':
                    yield obj
                    obj = None

        sequenceNumber += 1


def iter_osm_change_file(f, parse_timestamps=True):
    action = None
    obj = None
    for event, elem in etree.iterparse(f, events=('start', 'end')):
        if event == 'start':
            if elem.tag == 'node':
                obj = model.Node(
                    int(elem.attrib['id']),
                    int(elem.attrib['version']),
                    int(elem.attrib['changeset']),
                    elem.attrib.get('user'),
                    maybeInt(elem.attrib.get('uid')),
                    maybeBool(elem.attrib.get('visible')),
                    isoToDatetime(elem.attrib.get('timestamp')) if parse_timestamps else elem.attrib.get('timestamp'),
                    float(elem.attrib['lat']),
                    float(elem.attrib['lon']),
                    []
                )
            elif elem.tag == 'way':
                obj = model.Way(
                    int(elem.attrib['id']),
                    int(elem.attrib['version']),
                    int(elem.attrib['changeset']),
                    elem.attrib.get('user'),
                    maybeInt(elem.attrib.get('uid')),
                    maybeBool(elem.attrib.get('visible')),
                    isoToDatetime(elem.attrib.get('timestamp')) if parse_timestamps else elem.attrib.get('timestamp'),
                    [],
                    []<commit_msg>Lat/lon won't be on nodes that are deleted.<commit_after>import model
import datetime
import urllib2
import StringIO
import gzip
import time
from lxml import etree

def isoToDatetime(s):
    """Parse a ISO8601-formatted string to a Python datetime."""
    if s is None:
        return s
    else:
        return datetime.datetime.strptime(s, "%Y-%m-%dT%H:%M:%SZ")

def maybeInt(s):
    return int(s) if s is not None else s

def maybeFloat(s):
    return float(s) if s is not None else s

def maybeBool(s):
    if s is not None:
        if s == 'true':
            return True
        else:
            return False
    else:
        return s

def readState(state_file, sep='='):
    state = {}

    for line in state_file:
        if line == '---':
            continue
        if line[0] == '#':
            continue
        (k, v) = line.split(sep)
        state[k] = v.strip().replace("\\:", ":")

    return state

def iter_changeset_stream(start_sqn=None, base_url='http://planet.openstreetmap.org/replication/changesets', expected_interval=60, parse_timestamps=True):
    """Start processing an OSM changeset stream and yield one (action, primitive) tuple
    at a time to the caller."""

    # This is a lot like the other osm_stream except there's no
    # state file for each of the diffs, so just push ahead until
    # we run into a 404.

    # If no start_sqn, assume to start from the most recent changeset file
    if not start_sqn:
        u = urllib2.urlopen('%s/state.yaml' % base_url)
        state = readState(u, ': ')
        sequenceNumber = state['sequence']
    else:
        sequenceNumber = start_sqn

    interval_fudge = 0.0
    while True:
        sqnStr = str(sequenceNumber).zfill(9)
        url = '%s/%s/%s/%s.osm.gz' % (base_url, sqnStr[0:3], sqnStr[3:6], sqnStr[6:9])

        delay = 1.0
        while True:
            print 'Fetching %s' % url
            try:
                content = urllib2.urlopen(url)
                content = StringIO.StringIO(content.read())
                gzipper = gzip.GzipFile(fileobj=content)
                interval_fudge -= (interval_fudge / 2.0)
                break
            except urllib2.HTTPError, e:
                if e.code == 404:
                    print "%s doesn't exist yet. Waiting %.1f seconds." % (url, delay)
                    time.sleep(delay)
                    delay = min(delay * 2, 13)
                    interval_fudge += delay

        obj = None
        for event, elem in etree.iterparse(gzipper, events=('start', 'end')):
            if event == 'start':
                if elem.tag == 'changeset':
                    obj = model.Changeset(
                        int(elem.attrib['id']),
                        isoToDatetime(elem.attrib.get('created_at')) if parse_timestamps else elem.attrib.get('created_at'),
                        isoToDatetime(elem.attrib.get('closed_at')) if parse_timestamps else elem.attrib.get('closed_at'),
                        maybeBool(elem.attrib['open']),
                        maybeFloat(elem.get('min_lat')),
                        maybeFloat(elem.get('max_lat')),
                        maybeFloat(elem.get('min_lon')),
                        maybeFloat(elem.get('max_lon')),
                        elem.attrib.get('user'),
                        maybeInt(elem.attrib.get('uid')),
                        []
                    )
                elif elem.tag == 'tag':
                    obj.tags.append(
                        model.Tag(
                            elem.attrib['k'],
                            elem.attrib['v']
                        )
                    )
            elif event == 'end':
                if elem.tag == 'changeset':
                    yield obj
                    obj = None

        sequenceNumber += 1


def iter_osm_change_file(f, parse_timestamps=True):
    action = None
    obj = None
    for event, elem in etree.iterparse(f, events=('start', 'end')):
        if event == 'start':
            if elem.tag == 'node':
                obj = model.Node(
                    int(elem.attrib['id']),
                    int(elem.attrib['version']),
                    int(elem.attrib['changeset']),
                    elem.attrib.get('user'),
                    maybeInt(elem.attrib.get('uid')),
                    maybeBool(elem.attrib.get('visible')),
                    isoToDatetime(elem.attrib.get('timestamp')) if parse_timestamps else elem.attrib.get('timestamp'),
                    maybeFloat(elem.attrib.get('lat')),
                    maybeFloat(elem.attrib.get('lon')),
                    []
                )
            elif elem.tag == 'way':
                obj = model.Way(
                    int(elem.attrib['id']),
                    int(elem.attrib['version']),
                    int(elem.attrib['changeset']),
                    elem.attrib.get('user'),
                    maybeInt(elem.attrib.get('uid')),
                    maybeBool(elem.attrib.get('visible')),
                    isoToDatetime(elem.attrib.get('timestamp')) if parse_timestamps else elem.attrib.get('timestamp'),
                    [],
                    []
{'fraction_non_alphanumeric': 0.12181606066498153, 'fraction_numerical': 0.006805366517596733, 'mean_word_length': 1.5053580126643935, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4285991', 'n_tokens_mistral': 3040, 'n_tokens_neox': 2864, 'n_words': 671}