<commit_before>import os
import unicodedata

import pandas as pd
import grequests

from serenata_toolbox.datasets import Datasets

def normalize_string(string):
    if isinstance(string, str):
        nfkd_form = unicodedata.normalize('NFKD', string.lower())
        return nfkd_form.encode('ASCII', 'ignore').decode('utf-8')


def exception_handler(request, exception):
    return type('Response', (object,), {'status_code': None})

def get_status_code(response):
    if not response.status_code:
         return 404
    return response.status_code

def main(data_path='/tmp/serenata-data', cities_file='2017-05-22-brazilian-cities.csv'):
    Datasets(data_path).downloader.download(cities_file)
    br_cities = pd.read_csv(os.path.join(data_path, cities_file))

    br_cities['state'] = br_cities['state'].apply(str.lower)

    br_cities['normalized_name'] = br_cities['name'] \
                                              .apply(lambda x: normalize_string(x))
    br_cities['normalized_name'] = br_cities['normalized_name'] \
                                               .apply(lambda x: x.replace(' ', ''))

    portal_url = 'https://{}-{}.portaltp.com.br/'
    br_cities['transparency_portal_url'] = br_cities \
            .apply(lambda row: portal_url.format(row['normalized_name'],
                                                 row['state']), axis=1)

    rs = (grequests.get(u) for u in list(br_cities['transparency_portal_url']))

    responses = grequests.map(rs, exception_handler=exception_handler)

    br_cities.loc[:,'status_code'] = responses

    br_cities.loc[:,'status_code'] = br_cities \
            .apply(lambda row: get_status_code(row['status_code']), axis=1)

    br_cities.loc[br_cities['status_code'] == 404, 'transparency_portal_url'] = None

    portal_url = 'https://cm{}-{}.portaltp.com.br/'
    br_cities_cm = br_cities[br_cities['status_code'] == 404]

    br_cities_cm.loc[:,'transparency_portal_url'] = br_cities_cm \
            .apply(lambda row: portal_url.format(row['normalized_name'],
                                                 row['state']), axis=1)

    rs = (grequests.get(u) for u in list(br_cities_cm['transparency_portal_url']))

    responses = grequests.map(rs, exception_handler=exception_handler)

    br_cities_cm.loc[:,'status_code'] = responses

    br_cities_cm.loc[:,'status_code'] = br_cities_cm \
            .apply(lambda row: get_status_code(row['status_code']), axis=1)
    br_cities_cm.loc[br_cities_cm['status_code'] == 404, 'transparency_portal_url'] = None

    unnecessary_columns = ['normalized_name', 'status_code']
    br_cities = pd.merge(br_cities.drop(unnecessary_columns, axis=1),
		                 br_cities_cm.drop(unnecessary_columns, axis=1),
		                 on=['code', 'name', 'state'], how='left')

    br_cities['transparency_portal_url'] = br_cities \
	  .apply(lambda row: row['transparency_portal_url_x'] or row['transparency_portal_url_y'], axis=1)

    unnecessary_columns = ['transparency_portal_url_x', 'transparency_portal_url_y']
    br_cities = br_cities.drop(unnecessary_columns, axis=1)
    br_cities['state'] = br_cities['state'].apply(str.upper)

    br_cities.to_csv(os.path.join(data_path, 'cities-with-tp-url.xz'),
                     compression='xz', index=False)


if __name__ == '__main__':
    main()<commit_msg>Initial refactor on cities transparency fetch<commit_after>import os
import unicodedata

import grequests
import numpy as np
import pandas as pd

from serenata_toolbox.datasets import Datasets


def normalize_string(string):
    if isinstance(string, str):
        nfkd_form = unicodedata.normalize('NFKD', string.lower())
        return nfkd_form.encode('ASCII', 'ignore').decode('utf-8').replace(' ', '')


def exception_handler(request, exception):
    return type('Response', (object,), {'status_code': None})


def get_status_code(response):
    if not response.status_code:
         return 404
    return response.status_code


def format_row(row, url):
    if row['transparency_portal_url'] == 'None':
        return url.format(row['normalized_name'], row['state'].lower())
    return row['transparency_portal_url']


def check_transparency_portal_existance(dataset, portal_urls):
    dataset['transparency_portal_url'] = 'None'

    for url in portal_urls:
        dataset['transparency_portal_url'] = dataset.apply(format_row, axis=1, args=(url,))
        # TODO: Also filter by status_code
        rs = (grequests.get(u) for u in list(dataset['transparency_portal_url']))
        responses = grequests.map(rs, exception_handler=exception_handler)
        dataset.loc[:,'status_code'] = responses
        dataset.loc[:,'status_code'] = dataset \
               .apply(lambda row: get_status_code(row['status_code']), axis=1)
        dataset.loc[dataset['status_code'] == 404, 'transparency_portal_url'] = 'None'


def main(data_path='/tmp/serenata-data', cities_file='2017-05-22-brazilian-cities.csv'):
    Datasets(data_path).downloader.download(cities_file)
    cities = pd.read_csv(os.path.join(data_path, cities_file))
    cities = cities.head(50).copy()

    cities['normalized_name'] = cities['name'].apply(normalize_string)

    portal_urls = ['https://{}-{}.portaltp.com.br/',
                  'https://cm{}-{}.portaltp.com.br/']
    check_transparency_portal_existance(cities, portal_urls)

    unnecessary_columns = ['normalized_name', 'status_code']
    cities = cities.drop(unnecessary_columns, axis=1)

    cities.to_csv(os.path.join(data_path, 'cities-with-tp-url.xz'),
                     compression='xz', index=False)


if __name__ == '__main__':
    main()
{'fraction_non_alphanumeric': 0.13950177935943062, 'fraction_numerical': 0.008718861209964414, 'mean_word_length': 3.6763727121464225, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '437', 'n_tokens_mistral': 1881, 'n_tokens_neox': 1838, 'n_words': 290}