<commit_before>import base64
import hashlib
from scrapy.exporters import JsonLinesItemExporter, JsonItemExporter
from scrapy.utils.python import to_bytes


mapping = (
    ('addr_full', 'addr:full'),
    ('housenumber', 'addr:housenumber'),
    ('street', 'addr:street'),
    ('city', 'addr:city'),
    ('state', 'addr:state'),
    ('postcode', 'addr:postcode'),
    ('country', 'addr:country'),
    ('name', 'name'),
    ('phone', 'phone'),
    ('website', 'website'),
    ('opening_hours', 'opening_hours'),
)


def item_to_properties(item):
    props = {}

    # Ref is required
    props['ref'] = item['ref']

    # Add in the extra bits
    extras = item.get('extras')
    if extras:
        props.update(extras)

    # Bring in the optional stuff
    for map_from, map_to in mapping:
        item_value = item.get(map_from)
        if item_value:
            props[map_to] = item_value

    return props


def compute_hash(item):
    sha1 = hashlib.sha1(item['extras']['@spider'])
    sha1.update(item['ref'])
    return base64.urlsafe_b64encode(sha1.digest())


class LineDelimitedGeoJsonExporter(JsonLinesItemExporter):

    def _get_serialized_fields(self, item, default_value=None, include_empty=None):
        feature = []
        feature.append(('type', 'Feature'))
        feature.append(('id', compute_hash(item)))
        feature.append(('properties', item_to_properties(item)))

        if item.get('lon'):
            feature.append(('geometry', {
                'type': 'Point',
                'coordinates': [
                    item['lon'],
                    item['lat']
                ],
            }))

        return feature


class GeoJsonExporter(JsonItemExporter):

    def _get_serialized_fields(self, item, default_value=None, include_empty=None):
        feature = []
        feature.append(('type', 'Feature'))
        feature.append(('id', compute_hash(item)))
        feature.append(('properties', item_to_properties(item)))

        if item.get('lon'):
            feature.append(('geometry', {
                'type': 'Point',
                'coordinates': [
                    item['lon'],
                    item['lat']
                ],
            }))

        return feature

    def start_exporting(self):
        self.file.write(to_bytes('{"type":"FeatureCollection","features":[', self.encoding))

    def finish_exporting(self):
        self.file.write(to_bytes(']}', self.encoding))<commit_msg>Cast the lat/lon to float always<commit_after>import base64
import hashlib
from scrapy.exporters import JsonLinesItemExporter, JsonItemExporter
from scrapy.utils.python import to_bytes


mapping = (
    ('addr_full', 'addr:full'),
    ('housenumber', 'addr:housenumber'),
    ('street', 'addr:street'),
    ('city', 'addr:city'),
    ('state', 'addr:state'),
    ('postcode', 'addr:postcode'),
    ('country', 'addr:country'),
    ('name', 'name'),
    ('phone', 'phone'),
    ('website', 'website'),
    ('opening_hours', 'opening_hours'),
)


def item_to_properties(item):
    props = {}

    # Ref is required
    props['ref'] = item['ref']

    # Add in the extra bits
    extras = item.get('extras')
    if extras:
        props.update(extras)

    # Bring in the optional stuff
    for map_from, map_to in mapping:
        item_value = item.get(map_from)
        if item_value:
            props[map_to] = item_value

    return props


def compute_hash(item):
    sha1 = hashlib.sha1(item['extras']['@spider'])
    sha1.update(item['ref'])
    return base64.urlsafe_b64encode(sha1.digest())


class LineDelimitedGeoJsonExporter(JsonLinesItemExporter):

    def _get_serialized_fields(self, item, default_value=None, include_empty=None):
        feature = []
        feature.append(('type', 'Feature'))
        feature.append(('id', compute_hash(item)))
        feature.append(('properties', item_to_properties(item)))

        if item.get('lon'):
            feature.append(('geometry', {
                'type': 'Point',
                'coordinates': [
                    float(item['lon']),
                    float(item['lat'])
                ],
            }))

        return feature


class GeoJsonExporter(JsonItemExporter):

    def _get_serialized_fields(self, item, default_value=None, include_empty=None):
        feature = []
        feature.append(('type', 'Feature'))
        feature.append(('id', compute_hash(item)))
        feature.append(('properties', item_to_properties(item)))

        if item.get('lon'):
            feature.append(('geometry', {
                'type': 'Point',
                'coordinates': [
                    float(item['lon']),
                    float(item['lat'])
                ],
            }))

        return feature

    def start_exporting(self):
        self.file.write(to_bytes('{"type":"FeatureCollection","features":[', self.encoding))

    def finish_exporting(self):
        self.file.write(to_bytes(']}', self.encoding))
{'fraction_non_alphanumeric': 0.15267947421638017, 'fraction_numerical': 0.004044489383215369, 'mean_word_length': 2.5814627081824764, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '79311', 'n_tokens_mistral': 1528, 'n_tokens_neox': 1439, 'n_words': 274}