<commit_before>#!/usr/bin/python

import logging
from glob import glob
from json import loads as load_json
from hashlib import md5
from urllib2 import urlopen, URLError
from urlparse import urljoin
from argparse import ArgumentParser
from pkg_resources import find_distributions


def md5check(path):
    logger = logging.getLogger(__name__)
    with open(path) as fh:
        md5_hash = md5()
        while True:
            chunk = fh.read(128)
            if chunk == '':
                break
            md5_hash.update(chunk)
    file_hash = md5_hash.hexdigest()
    logger.debug('file md5: {0}'.format(file_hash))
    return file_hash


def get_pypi_hash(baseurl):
    logger = logging.getLogger(__name__)
    dist = find_distributions('.').next()
    logger.debug('dist found: {0}'.format(dist))
    targer_url = urljoin(baseurl, 'pypi/{0}/json'.format(dist.project_name))
    logger.debug('opening url: {0}'.format(targer_url))
    response = urlopen(targer_url)
    pypi_json = load_json(response.read())
    md5_digest = pypi_json['releases'][dist.version][0]['md5_digest']
    logger.debug('pypi md5: {0}'.format(md5_digest))
    return md5_digest


def main():
    logger = logging.getLogger(__name__)
    logger.setLevel(logging.DEBUG)
    ch = logging.StreamHandler()
    logger.addHandler(ch)

    parser = ArgumentParser()
    parser.add_argument('-f', '--file',
                        nargs=1,
                        default=glob('dist/*.tar.gz'))
    parser.add_argument('-u', '--url',
                        nargs=1,
                        default=['https://pypi.python.org/'])
    parser.add_argument('-v', '--verbose', action='store_true')
    args = parser.parse_args()

    ch.setLevel(logging.DEBUG if args.verbose else logging.INFO)

    try:
        file_hash = md5check(args.file[0])
        pypi_hash = get_pypi_hash(args.url[0])
    except (IndexError, IOError):
        logger.error('File not found')
        exit(1)
    except StopIteration:
        logger.error('Distribution not found')
        exit(1)
    except URLError:
        logger.error('Unable to open url')
        exit(1)

    if file_hash == pypi_hash:
        logger.info('Package succesfully uploaded to pypi')
        exit(0)
    else:
        logger.info('Error, package upload failed')
        exit(1)


if __name__ == '__main__':
    main()<commit_msg>add pypi url by branch<commit_after>#!/usr/bin/python

import logging
from glob import glob
from json import loads as load_json
from hashlib import md5
from urllib2 import urlopen, URLError
from urlparse import urljoin
from argparse import ArgumentParser
from pkg_resources import find_distributions


def md5check(path):
    logger = logging.getLogger(__name__)
    with open(path) as fh:
        md5_hash = md5()
        while True:
            chunk = fh.read(128)
            if chunk == '':
                break
            md5_hash.update(chunk)
    file_hash = md5_hash.hexdigest()
    logger.debug('file md5: {0}'.format(file_hash))
    return file_hash


def get_pypi_hash(baseurl):
    logger = logging.getLogger(__name__)
    dist = find_distributions('.').next()
    logger.debug('dist found: {0}'.format(dist))
    targer_url = urljoin(baseurl, 'pypi/{0}/json'.format(dist.project_name))
    logger.debug('opening url: {0}'.format(targer_url))
    response = urlopen(targer_url)
    pypi_json = load_json(response.read())
    md5_digest = pypi_json['releases'][dist.version][0]['md5_digest']
    logger.debug('pypi md5: {0}'.format(md5_digest))
    return md5_digest


def get_pypi_url(branch):
    if branch == 'pypi-release':
        return 'https://pypi.python.org/'
    elif branch == 'pypi-test':
        return 'https://testpypi.python.org/'
    else:
        raise ValueError('Unknown branch')


def main():
    logger = logging.getLogger(__name__)
    logger.setLevel(logging.DEBUG)
    ch = logging.StreamHandler()
    logger.addHandler(ch)

    parser = ArgumentParser()
    parser.add_argument('-f', '--file',
                        nargs=1,
                        default=glob('dist/*.tar.gz'))

    urlgroup = parser.add_argument_group('PyPi repository url')
    urlgroup.add_argument('-u', '--url',
                          nargs=1,
                          default=['https://pypi.python.org/'])

    branchgroup = parser.add_argument_group('PyPi repository url by branch')
    branchgroup.add_argument('-b', '--branch',
                             nargs=1)
    parser.add_argument('-v', '--verbose', action='store_true')
    args = parser.parse_args()

    ch.setLevel(logging.DEBUG if args.verbose else logging.INFO)

    try:
        file_hash = md5check(args.file[0])
        if args.branch:
            baseurl = get_pypi_url(args.branch[0])
        else:
            baseurl = args.url
        logger.info('pypi url: {}'.format(baseurl))
        pypi_hash = get_pypi_hash(baseurl)
    except (IndexError, IOError):
        logger.error('File not found')
        exit(1)
    except StopIteration:
        logger.error('Distribution not found')
        exit(1)
    except URLError:
        logger.error('Unable to open url')
        exit(1)

    if file_hash == pypi_hash:
        logger.info('Package succesfully uploaded to pypi')
        exit(0)
    else:
        logger.info('Error, package upload failed')
        exit(1)


if __name__ == '__main__':
    main()
{'fraction_non_alphanumeric': 0.12534974818130945, 'fraction_numerical': 0.01212460361872785, 'mean_word_length': 2.6205266711681294, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4963873', 'n_tokens_mistral': 1766, 'n_tokens_neox': 1671, 'n_words': 356}