<commit_before>import asyncio
import os

import aiofiles
import aiohttp
from tqdm import tqdm

MAX_REQUESTS = 4


class Downloader:

    LATEST = (
        '2016-08-08-ceap-datasets.md',
        '2016-08-08-current-year.xz',
        '2016-08-08-datasets-format.html',
        '2016-08-08-last-year.xz',
        '2016-08-08-previous-years.xz',
        '2016-09-03-companies.xz',
        '2016-11-11-congressperson-relatives.xz',
        '2016-11-19-current-year.xz',
        '2016-11-19-last-year.xz',
        '2016-11-19-previous-years.xz',
        '2016-11-28-congressperson-civil-names.xz',
        '2016-11-29-yelp-companies.xz',
        '2016-12-02-foursquare-companies.xz',
        '2016-12-15-speeches.xz',
        '2016-12-20-impeded-non-profit-entities.xz',
        '2016-12-21-deputies.xz',
        '2016-12-21-inident-and-suspended-companies.xz',
        '2016-12-21-national-register-punished-companies.xz',
        '2016-12-21-presences.xz',
        '2016-12-21-sessions.xz',
        '2016-12-21-speeches.xz',
        '2016-12-22-agreements.xz',
        '2016-12-22-amendments.xz',
        '2017-02-15-receipts-texts.xz',
        '2017-03-15-reimbursements.xz',
        '2017-03-20-purchase-suppliers.xz',
        '2017-04-27-companies-no-geolocation.xz',
        '2017-05-10-tse-candidates.xz',
        '2017-05-21-companies-no-geolocation.xz'
    )

    def __init__(self, target, **kwargs):
        self.bucket = kwargs.get('bucket')
        self.region = kwargs.get('region_name')
        if not all((self.bucket, self.region)):
            raise RuntimeError('No bucket and/or region_name kwargs provided')

        self.target = os.path.abspath(target)
        if not all((os.path.exists(self.target), os.path.isdir(self.target))):
            msg = '{} does not exist or is not a directory.'
            raise FileNotFoundError(msg.format(self.target))

        self.semaphore = asyncio.Semaphore(MAX_REQUESTS)
        self.total = 0

    def download(self, files):
        if isinstance(files, str):
            files = [files]

        files = tuple(filter(bool, files))
        if not files:
            return

        loop = asyncio.get_event_loop()
        loop.run_until_complete(self.main(loop, files))<commit_msg>Remove old companies dataset (wo/ geolocaion) from latest<commit_after>import asyncio
import os

import aiofiles
import aiohttp
from tqdm import tqdm

MAX_REQUESTS = 4


class Downloader:

    LATEST = (
        '2016-08-08-ceap-datasets.md',
        '2016-08-08-current-year.xz',
        '2016-08-08-datasets-format.html',
        '2016-08-08-last-year.xz',
        '2016-08-08-previous-years.xz',
        '2016-09-03-companies.xz',
        '2016-11-11-congressperson-relatives.xz',
        '2016-11-19-current-year.xz',
        '2016-11-19-last-year.xz',
        '2016-11-19-previous-years.xz',
        '2016-11-28-congressperson-civil-names.xz',
        '2016-11-29-yelp-companies.xz',
        '2016-12-02-foursquare-companies.xz',
        '2016-12-15-speeches.xz',
        '2016-12-20-impeded-non-profit-entities.xz',
        '2016-12-21-deputies.xz',
        '2016-12-21-inident-and-suspended-companies.xz',
        '2016-12-21-national-register-punished-companies.xz',
        '2016-12-21-presences.xz',
        '2016-12-21-sessions.xz',
        '2016-12-21-speeches.xz',
        '2016-12-22-agreements.xz',
        '2016-12-22-amendments.xz',
        '2017-02-15-receipts-texts.xz',
        '2017-03-15-reimbursements.xz',
        '2017-03-20-purchase-suppliers.xz',
        '2017-05-10-tse-candidates.xz',
        '2017-05-21-companies-no-geolocation.xz'
    )

    def __init__(self, target, **kwargs):
        self.bucket = kwargs.get('bucket')
        self.region = kwargs.get('region_name')
        if not all((self.bucket, self.region)):
            raise RuntimeError('No bucket and/or region_name kwargs provided')

        self.target = os.path.abspath(target)
        if not all((os.path.exists(self.target), os.path.isdir(self.target))):
            msg = '{} does not exist or is not a directory.'
            raise FileNotFoundError(msg.format(self.target))

        self.semaphore = asyncio.Semaphore(MAX_REQUESTS)
        self.total = 0

    def download(self, files):
        if isinstance(files, str):
            files = [files]

        files = tuple(filter(bool, files))
        if not files:
            return

        loop = asyncio.get_event_loop()
        loop.run_until_complete(self.main(loop, files))
{'fraction_non_alphanumeric': 0.15192135835567472, 'fraction_numerical': 0.10277033065236818, 'mean_word_length': 2.197857142857143, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4318740', 'n_tokens_mistral': 1965, 'n_tokens_neox': 1575, 'n_words': 215}