<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# File: fs.py
# Author: <NAME> <<EMAIL>>

import os, sys
from six.moves import urllib

def mkdir_p(dirname):
    assert dirname is not None
    if dirname == '':
        return
    try:
        os.makedirs(dirname)
    except OSError as e:
        if e.errno != 17:
            raise e


def download(url, dir):
    mkdir_p(dir)
    fname = url.split('/')[-1]
    fpath = os.path.join(dir, fname)

    def _progress(count, block_size, total_size):
        sys.stdout.write('\r>> Downloading %s %.1f%%' %
                         (fname, float(count * block_size) / float(total_size) * 100.0))
        sys.stdout.flush()
    fpath, _ = urllib.request.urlretrieve(url, fpath, reporthook=_progress)
    statinfo = os.stat(fpath)
    sys.stdout.write('\n')
    print('Succesfully downloaded ' + fname + " " + str(statinfo.st_size) + ' bytes.')
    return fpath

if __name__ == '__main__':
    download('http://dl.caffe.berkeleyvision.org/caffe_ilsvrc12.tar.gz', '.')
<commit_msg>fix permission requirements on TENSORSPACK_DATASET<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# File: fs.py
# Author: <NAME> <<EMAIL>>

import os, sys
from six.moves import urllib

def mkdir_p(dirname):
    """ make a dir recursively, but do nothing if the dir exists"""
    assert dirname is not None
    if dirname == '' or os.path.isdir(dirname):
        return
    try:
        os.makedirs(dirname)
    except OSError as e:
        if e.errno != 17:
            raise e


def download(url, dir):
    mkdir_p(dir)
    fname = url.split('/')[-1]
    fpath = os.path.join(dir, fname)

    def _progress(count, block_size, total_size):
        sys.stdout.write('\r>> Downloading %s %.1f%%' %
                         (fname, float(count * block_size) / float(total_size) * 100.0))
        sys.stdout.flush()
    fpath, _ = urllib.request.urlretrieve(url, fpath, reporthook=_progress)
    statinfo = os.stat(fpath)
    sys.stdout.write('\n')
    print('Succesfully downloaded ' + fname + " " + str(statinfo.st_size) + ' bytes.')
    return fpath

if __name__ == '__main__':
    download('http://dl.caffe.berkeleyvision.org/caffe_ilsvrc12.tar.gz', '.')

{'fraction_non_alphanumeric': 0.15206536541080345, 'fraction_numerical': 0.009986382206082615, 'mean_word_length': 2.69179229480737, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1840857', 'n_tokens_mistral': 804, 'n_tokens_neox': 786, 'n_words': 173}