<commit_before>#!/usr/bin/env python2
# coding: utf-8

import os
import re
import requests
import subprocess
import sys

re_distfile = re.compile(r'(blogc[^\'"]+)\.sha512')
base_url = 'https://distfiles.rgm.io/blogc'
cwd = os.path.dirname(os.path.abspath(__file__))


def download_release(version):
    release_url = '%s/blogc-%s' % (base_url, version)
    r = requests.get('%s/' % release_url)
    r.raise_for_status()

    for distfile in set(re_distfile.findall(r.content)):
        file_url = '%s/%s' % (release_url, distfile)
        dest_path = os.path.join(cwd, 'releases', version)
        subprocess.check_call(['wget', '-c', '-P', dest_path, file_url,
                               '%s.sha512' % file_url])
        subprocess.check_call(['sha512sum', '-c', '%s.sha512' % distfile],
                              cwd=dest_path)


if __name__ == '__main__':
    download_release(sys.argv[1])<commit_msg>maint: download latest build by default<commit_after>#!/usr/bin/env python2
# coding: utf-8

import os
import re
import requests
import shutil
import subprocess
import sys

re_version = re.compile(r'blogc-([^\'"]+)\.tar\.gz')
re_distfile = re.compile(r'(blogc[^\'"]+)\.sha512')
base_url = 'https://distfiles.rgm.io/blogc'
cwd = os.path.dirname(os.path.abspath(__file__))


def download_release(version):
    if version is None:
        release_url = '%s/LATEST' % (base_url,)
        r = requests.get(release_url)
        r.raise_for_status()
        match = re_version.search(r.content)
        if match is None:
            raise RuntimeError('Could not guess version')
        version = match.group(1)
    else:
        release_url = '%s/blogc-%s' % (base_url, version)
        r = requests.get(release_url)
        r.raise_for_status()

    dest_path = os.path.join(cwd, 'releases', version)
    if os.path.exists(dest_path):
        shutil.rmtree(dest_path)
    os.makedirs(dest_path)

    for distfile in set(re_distfile.findall(r.content)):
        file_url = '%s/%s' % (release_url, distfile)
        subprocess.check_call(['wget', '-P', dest_path, file_url,
                               '%s.sha512' % file_url])
        subprocess.check_call(['sha512sum', '-c', '%s.sha512' % distfile],
                              cwd=dest_path)


if __name__ == '__main__':
    download_release(sys.argv[1] if len(sys.argv) > 1 else None)
{'fraction_non_alphanumeric': 0.15289608177172062, 'fraction_numerical': 0.013628620102214651, 'mean_word_length': 2.8634868421052633, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2941299', 'n_tokens_mistral': 851, 'n_tokens_neox': 819, 'n_words': 155}