<commit_before><commit_msg>add geo info<commit_after>#-*-coding: utf-8-*-

from tornado.httpclient import AsyncHTTPClient, HTTPError
from tornado.options import options
import workers
from tornado import gen
from tornado import escape


AsyncHTTPClient.configure("tornado.curl_httpclient.CurlAsyncHTTPClient")


@gen.coroutine
def GeoFetch(keyword):
    client = AsyncHTTPClient()
    url = "view-source:http://api.geonames.org/searchJSON?q=Xiaoshan,%20Zhejiang,%20China&maxRows=10&username=" + options.username
    try:
        resp = yield client.fetch(url)
    except HTTPError, e:
        options.logger.error("fetch geoname error %d %s" % (e.code, e.message))
        resp = e
    raise gen.Return(resp)


@gen.coroutine
def collect_geoname():
    china_map = {}
    for city in options.city_list:
        china_map[city] = {"score": 0, "stateInitColor": 6}
    for user in workers.github_china:
        try:
            location = user["location"].lower()
        except Exception, e:
            options.logger.error("location error: %s" % e)
            continue
        city = match_geoname(location)
        if city:
            china_map[city]['score'] += 1


def match_location(city, location):
    if (city in location or location in city):
        return True
    else:
        return False


@gen.coroutune
def match_geoname(location):
    matched_city = None
    for city in options.city_list:
        if match_location(city, location):
            matched_city = city
            break
    if matched_city is None:
        resp = yield GeoFetch(location)
        if resp.code == 200:
            resp = escape.json_encode(resp.body)
            for geo in resp.get("geonames", []):
                if matched_city is None:
                    for city in options.city_list:
                        if match_location(city, geo.get("adminName1", "NoAdminName")):
                            matched_city = city
                            break
    raise gen.Return(matched_city)
{'fraction_non_alphanumeric': 0.09979939819458375, 'fraction_numerical': 0.007021063189568706, 'mean_word_length': 2.3813559322033897, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '166921', 'n_tokens_mistral': 601, 'n_tokens_neox': 574, 'n_words': 148}