<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
release.py
Retrieve latest compressed wheels from GitHub


Created by <NAME> on 2016-06-19
"""

import io
import tarfile
import zipfile
import requests
from subprocess import check_output
from multiprocessing import Pool

path = 'dist/'
url = "https://github.com/urschrei/simplification/releases/download/{tag}/simplification-{tag}-{target}.{extension}"
# get latest tag
tag = check_output(["git", "describe", "--abbrev=0", "--tags"]).strip()
releases = [
    {
        'tag': tag,
        'target': 'x86_64-apple-darwin-cp27',
        'extension': 'tar.gz'
        },
    {
        'tag': tag,
        'target': 'x86_64-apple-darwin-cp36',
        'extension': 'tar.gz'
        },
    {
        'tag': tag,
        'target': 'x86_64-unknown-linux-gnu',
        'extension': 'tar.gz'
        },
    {
        'tag': tag,
        'target': 'x86_64-pc-windows-msvc-cp27',
        'extension': 'zip'
        },
    {
        'tag': tag,
        'target': 'i686-pc-windows-msvc-cp27',
        'extension': 'zip'
        },
    {
        'tag': tag,
        'target': 'x86_64-pc-windows-msvc-cp34',
        'extension': 'zip'
        },
    {
        'tag': tag,
        'target': 'i686-pc-windows-msvc-cp34',
        'extension': 'zip'
        },
    {
        'tag': tag,
        'target': 'x86_64-pc-windows-msvc-cp35',
        'extension': 'zip'
        },
    {
        'tag': tag,
        'target': 'i686-pc-windows-msvc-cp35',
        'extension': 'zip'
        },
    {
        'tag': tag,
        'target': 'x86_64-pc-windows-msvc-cp36',
        'extension': 'zip'
        },
    {
        'tag': tag,
        'target': 'i686-pc-windows-msvc-cp36',
        'extension': 'zip'
        },
]

def retrieve(url):
    sess = requests.Session()
    retrieved = sess.get(url, stream=True)
    # don't continue if something's wrong
    retrieved.raise_for_status()
    try:
        raw_zip = zipfile.ZipFile(io.BytesIO(retrieved.content))
        raw_zip.extractall(path)
    except zipfile.BadZipfile:
        # it's a tar
        tar = tarfile.open(mode="r:gz", fileobj=io.BytesIO(retrieved.content))
        tar.extractall(path)

urls = (url.format(**release) for release in releases)

# let's do this in parallel
pool = Pool(processes=5)
pool.map(retrieve, urls)<commit_msg>Use async<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
release.py
Retrieve latest compressed wheels from GitHub


Created by <NAME> on 2016-06-19
"""

import io
import tarfile
import zipfile
import requests
from subprocess import check_output
from multiprocessing import Pool

path = 'dist/'
url = "https://github.com/urschrei/simplification/releases/download/{tag}/simplification-{tag}-{target}.{extension}"
# get latest tag
tag = check_output(["git", "describe", "--abbrev=0", "--tags"]).strip()
releases = [
    {
        'tag': tag,
        'target': 'x86_64-apple-darwin-cp27',
        'extension': 'tar.gz'
        },
    {
        'tag': tag,
        'target': 'x86_64-apple-darwin-cp36',
        'extension': 'tar.gz'
        },
    {
        'tag': tag,
        'target': 'x86_64-unknown-linux-gnu',
        'extension': 'tar.gz'
        },
    {
        'tag': tag,
        'target': 'x86_64-pc-windows-msvc-cp27',
        'extension': 'zip'
        },
    {
        'tag': tag,
        'target': 'i686-pc-windows-msvc-cp27',
        'extension': 'zip'
        },
    {
        'tag': tag,
        'target': 'x86_64-pc-windows-msvc-cp34',
        'extension': 'zip'
        },
    {
        'tag': tag,
        'target': 'i686-pc-windows-msvc-cp34',
        'extension': 'zip'
        },
    {
        'tag': tag,
        'target': 'x86_64-pc-windows-msvc-cp35',
        'extension': 'zip'
        },
    {
        'tag': tag,
        'target': 'i686-pc-windows-msvc-cp35',
        'extension': 'zip'
        },
    {
        'tag': tag,
        'target': 'x86_64-pc-windows-msvc-cp36',
        'extension': 'zip'
        },
    {
        'tag': tag,
        'target': 'i686-pc-windows-msvc-cp36',
        'extension': 'zip'
        },
]

def retrieve(url):
    sess = requests.Session()
    retrieved = sess.get(url, stream=True)
    # don't continue if something's wrong
    retrieved.raise_for_status()
    try:
        raw_zip = zipfile.ZipFile(io.BytesIO(retrieved.content))
        raw_zip.extractall(path)
    except zipfile.BadZipfile:
        # it's a tar
        tar = tarfile.open(mode="r:gz", fileobj=io.BytesIO(retrieved.content))
        tar.extractall(path)

urls = (url.format(**release) for release in releases)

# let's do this in parallel
pool = Pool(processes=5)
# we could use map, but it consumes the entire iterable (doesn't matter for small n)
res = pool.map_async(retrieve, urls)
# need these if we use _async
pool.close()
pool.join()
{'fraction_non_alphanumeric': 0.1668048072938251, 'fraction_numerical': 0.029423953584749276, 'mean_word_length': 2.161100196463654, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '851839', 'n_tokens_mistral': 1778, 'n_tokens_neox': 1654, 'n_words': 328}