<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
""" Crawling the geocode.

File: geocoder.py
Author: SpaceLis
Email: <EMAIL>
GitHub: http://github.com/spacelis
Description:


"""
from gevent import monkey
monkey.patch_all()

import sys
import re
import requests


from crawler.actors import Controller
from crawler.readers import CSVReader
from crawler.writers import FileWriter

import logging
logging.basicConfig(level=logging.INFO)
COORD = re.compile(r'<gml:pos dimension="2">([\d.]+ [\d.]+)</gml:pos>')


class Worker(object):

    """ A worker. """

    def __init__(self, _):
        """@todo: to be defined1.

        :args: @todo

        """
        pass

    def work_on(self, x):
        """@todo: Docstring for work_on.

        :x: A dict holding rows in csv
        :returns: The coordinates.

        """
        s = x['Straatnaam']
        n = ('' if x['Huisnummer Melding'] == ''
             else str(int(float(x['Huisnummer Melding']))))
        q = {'zoekterm': ' '.join([s, n, 'Rotterdam'])}
        try:
            ret = requests.get(
                'http://geodata.nationaalgeoregister.nl/geocoder/Geocoder',
                params=q, timeout=20
            )
            return ','.join([s, n] +
                            COORD.search(ret.text).group(1).split(' '))
        except Exception as e:
            print ','.join([s, n])
            return ''


def test():
    """ test """
    w = Worker('')
    print w.work_on({'Straatnaam': 'FEIJENOORDHAVEN',
                     'Huisnummer Melding': 44.0})


def console():
    """ running in console
    :returns: @todo

    """
    if len(sys.argv) != 3:
        print >> sys.stderr, 'Usage: tweet_crawler.py <input> <output>'
        sys.exit()

    controller = Controller.start(Worker, [1],
                                  CSVReader(sys.argv[1]),
                                  FileWriter(sys.argv[2]),
                                  poolsize=30)
    controller.actor_stopped.wait()

if __name__ == '__main__':
    console()<commit_msg>Improved geocoder<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
""" Crawling the geocode.

File: geocoder.py
Author: SpaceLis
Email: <EMAIL>
GitHub: http://github.com/spacelis
Description:


"""
from gevent import monkey
monkey.patch_all()

import sys
import re
import requests


from crawler.actors import Controller
from crawler.readers import CSVReader
from crawler.writers import FileWriter

import logging
logging.basicConfig(level=logging.INFO)
logging.getLogger("requests").setLevel(logging.WARNING)
COORD = re.compile(r'<gml:pos dimension="2">([\d.]+ [\d.]+)</gml:pos>')


class Worker(object):

    """ A worker. """

    def __init__(self, _):
        """@todo: to be defined1.

        :args: @todo

        """
        pass

    def work_on(self, x):
        """@todo: Docstring for work_on.

        :x: A dict holding rows in csv
        :returns: The coordinates.

        """
        s = x['Straatnaam']
        n = ('' if x['Huisnummer Melding'] == '0'
             else str(x['Huisnummer Melding']))
        q = {'zoekterm': ' '.join([s, n, 'Rotterdam'])}
        try:
            ret = requests.get(
                'http://geodata.nationaalgeoregister.nl/geocoder/Geocoder',
                params=q
            )

            return ','.join([s, n] +
                            COORD.search(ret.text).group(1).split(' '))
        except Exception as e:
            print ','.join([s, n])
            return ''


def test():
    """ test """
    w = Worker('')
    print w.work_on({'Straatnaam': 'FEIJENOORDHAVEN',
                     '<NAME>': '44'})
    print w.work_on({'Straatnaam': 'FEIJENOORDHAVEN',
                     '<NAME>': '0'})


def console():
    """ running in console
    :returns: @todo

    """
    if len(sys.argv) != 3:
        print >> sys.stderr, 'Usage: tweet_crawler.py <input> <output>'
        sys.exit()

    controller = Controller.start(Worker, [1],
                                  CSVReader(sys.argv[1]),
                                  FileWriter(sys.argv[2]),
                                  poolsize=15)
    controller.actor_stopped.wait()

if __name__ == '__main__':
    console()
    #test()
{'fraction_non_alphanumeric': 0.1541201614818333, 'fraction_numerical': 0.0068867252434101165, 'mean_word_length': 1.9914772727272727, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4291985', 'n_tokens_mistral': 1419, 'n_tokens_neox': 1347, 'n_words': 300}