<commit_before>import csv
import datetime
from lxml import html
import os
import requests
import grequests

CAMARA_URL = 'http://www2.camara.leg.br/transparencia/recursos-humanos/quadro-remuneratorio/consulta-secretarios-parlamentares/layouts_transpar_quadroremuner_consultaSecretariosParlamentares'
USERAGENT = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.86 Safari/537.36'

BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
DATA_PATH = os.path.join(BASE_DIR, 'data')
DATE = datetime.date.today().strftime('%Y-%m-%d')
FILE_BASE_NAME = '{}-deputies-advisors.csv'.format(DATE)

def run():
    print("Fetching deputies data...")
    deputies_data = fetch_deputies_data()

    print("Preparing requests to fetch advisors data...")
    requests = [get_request_to_page_of_advisors_from_deputy(deputy) for deputy in deputies_data]

    for page_data in send_requests(requests):
        deputy_with_advisors = page_data["data"]
        deputy_information = organize_deputy_data({"deputy_name": deputy_with_advisors["deputy_name"], "deputy_number": deputy_with_advisors["deputy_number"]}, deputy_with_advisors["deputy_advisors"])
        write_to_csv(deputy_information, os.path.join(DATA_PATH, FILE_BASE_NAME))

    print("\nFinished! The file can be found at data/{}".format(FILE_BASE_NAME))


def send_requests(reqs):
    """
    Send all the requests in :reqs: and reads the response data to extract the deputies data.
    It will check if a deputy has more than one page of advisors and send new requests if True
    """
    request_buffer = list()
<commit_msg>Organize imports<commit_after>import csv
import datetime
import os
from lxml import html

import grequests
import requests

CAMARA_URL = 'http://www2.camara.leg.br/transparencia/recursos-humanos/quadro-remuneratorio/consulta-secretarios-parlamentares/layouts_transpar_quadroremuner_consultaSecretariosParlamentares'
USERAGENT = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.86 Safari/537.36'

BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
DATA_PATH = os.path.join(BASE_DIR, 'data')
DATE = datetime.date.today().strftime('%Y-%m-%d')
FILE_BASE_NAME = '{}-deputies-advisors.csv'.format(DATE)

def run():
    print("Fetching deputies data...")
    deputies_data = fetch_deputies_data()

    print("Preparing requests to fetch advisors data...")
    requests = [get_request_to_page_of_advisors_from_deputy(deputy) for deputy in deputies_data]

    for page_data in send_requests(requests):
        deputy_with_advisors = page_data["data"]
        deputy_information = organize_deputy_data({"deputy_name": deputy_with_advisors["deputy_name"], "deputy_number": deputy_with_advisors["deputy_number"]}, deputy_with_advisors["deputy_advisors"])
        write_to_csv(deputy_information, os.path.join(DATA_PATH, FILE_BASE_NAME))

    print("\nFinished! The file can be found at data/{}".format(FILE_BASE_NAME))


def send_requests(reqs):
    """
    Send all the requests in :reqs: and reads the response data to extract the deputies data.
    It will check if a deputy has more than one page of advisors and send new requests if True
    """
    request_buffer = list()

{'fraction_non_alphanumeric': 0.11076923076923077, 'fraction_numerical': 0.01723076923076923, 'mean_word_length': 5.239923224568138, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '496603', 'n_tokens_mistral': 1166, 'n_tokens_neox': 1071, 'n_words': 235}