<commit_before>import xmlrpclib
import urllib2
import hashlib
import warnings

def myprint(mystr,fd=None):
    if fd is None:
        print mystr
    else:
        print >> fd, mystr

USER_AGENT = 'pypi-install/0.6.0+git ( http://github.com/astraw/stdeb )'

def find_tar_gz(package_name, pypi_url = 'http://python.org/pypi',verbose=0):
    transport = xmlrpclib.Transport()
    transport.user_agent = USER_AGENT
    pypi = xmlrpclib.ServerProxy(pypi_url, transport=transport)

    download_url = None
    expected_md5_digest = None

    if verbose >= 2:
        myprint( 'querying PyPI (%s) for package name "%s"' % (pypi_url,
                                                               package_name) )
    releases = pypi.package_releases(package_name)
    if verbose >= 2:
        myprint( 'found releases: %s' % (', '.join(releases),) )
    if len(releases) > 1:
        # XXX how to sort versions?
        raise NotImplementedError('no ability to handle more than one release')
    for version in releases:

        urls = pypi.release_urls( package_name,version)
        for url in urls:
            if url['packagetype']=='sdist':
                assert url['python_version']=='source', 'how can an sdist not be a source?'
                if url['url'].endswith('.tar.gz'):
                    download_url = url['url']
                    if 'md5_digest' in url:
                        expected_md5_digest = url['md5_digest']
                    break

        if download_url is None:
            # PyPI doesn't have package. Is download URL provided?
            result = pypi.release_data(package_name,version)
            if result['download_url'] != 'UNKNOWN':
                download_url = result['download_url']
                # no download URL provided, see if PyPI itself has download
                urls = pypi.release_urls( result['name'], result['version'] )
    if download_url is None:
        raise ValueError('no package "%s" was found'%package_name)
    return download_url, expected_md5_digest

def get_source_tarball(package_name,verbose=0):
    download_url, expected_md5_digest = find_tar_gz(package_name,
                                                    verbose=verbose)
    if verbose >= 1:
        myprint( 'downloading %s' % download_url )
    request = urllib2.Request(download_url)
    request.add_header('User-Agent', USER_AGENT )
    opener = urllib2.build_opener()
    package_tar_gz = opener.open(request).read()
    if verbose >= 1:
        myprint( 'done downloading %d bytes.' % ( len(package_tar_gz), ) )
    if expected_md5_digest is not None:
        m = hashlib.md5()
        m.update(package_tar_gz)
        actual_md5_digest = m.hexdigest()
        if verbose >= 2:
            myprint( 'md5:   actual %s\n     expected %s' % (actual_md5_digest,
                                                             expected_md5_digest))
        if actual_md5_digest != expected_md5_digest:
            raise ValueError('actual and expected md5 digests do not match')
    else:
        warnings.warn('no md5 digest found -- cannot verify source file')
    fname = download_url.split('/')[-1]
    fd = open(fname,mode='wb')
    fd.write( package_tar_gz )
    fd.close()
    return fname<commit_msg>PyPI: skip download if existing file OK<commit_after>import os
import xmlrpclib
import urllib2
import hashlib
import warnings

def myprint(mystr,fd=None):
    if fd is None:
        print mystr
    else:
        print >> fd, mystr

USER_AGENT = 'pypi-install/0.6.0+git ( http://github.com/astraw/stdeb )'

def find_tar_gz(package_name, pypi_url = 'http://python.org/pypi',verbose=0):
    transport = xmlrpclib.Transport()
    transport.user_agent = USER_AGENT
    pypi = xmlrpclib.ServerProxy(pypi_url, transport=transport)

    download_url = None
    expected_md5_digest = None

    if verbose >= 2:
        myprint( 'querying PyPI (%s) for package name "%s"' % (pypi_url,
                                                               package_name) )
    releases = pypi.package_releases(package_name)
    if verbose >= 2:
        myprint( 'found releases: %s' % (', '.join(releases),) )
    if len(releases) > 1:
        # XXX how to sort versions?
        raise NotImplementedError('no ability to handle more than one release')
    for version in releases:

        urls = pypi.release_urls( package_name,version)
        for url in urls:
            if url['packagetype']=='sdist':
                assert url['python_version']=='source', 'how can an sdist not be a source?'
                if url['url'].endswith('.tar.gz'):
                    download_url = url['url']
                    if 'md5_digest' in url:
                        expected_md5_digest = url['md5_digest']
                    break

        if download_url is None:
            # PyPI doesn't have package. Is download URL provided?
            result = pypi.release_data(package_name,version)
            if result['download_url'] != 'UNKNOWN':
                download_url = result['download_url']
                # no download URL provided, see if PyPI itself has download
                urls = pypi.release_urls( result['name'], result['version'] )
    if download_url is None:
        raise ValueError('no package "%s" was found'%package_name)
    return download_url, expected_md5_digest

def get_source_tarball(package_name,verbose=0):
    download_url, expected_md5_digest = find_tar_gz(package_name,
                                                    verbose=verbose)
    fname = download_url.split('/')[-1]
    if expected_md5_digest is not None:
        if os.path.exists(fname):
            m = hashlib.md5()
            m.update(open(fname,mode='r').read())
            actual_md5_digest = m.hexdigest()
            if actual_md5_digest == expected_md5_digest:
                if verbose >= 1:
                    myprint( 'Download URL: %s' % download_url )
                    myprint( 'File "%s" already exists with correct checksum.' % fname )
                return fname
            else:
                raise ValueError('File "%s" exists but has wrong checksum.'%fname)
    if verbose >= 1:
        myprint( 'downloading %s' % download_url )
    request = urllib2.Request(download_url)
    request.add_header('User-Agent', USER_AGENT )
    opener = urllib2.build_opener()
    package_tar_gz = opener.open(request).read()
    if verbose >= 1:
        myprint( 'done downloading %d bytes.' % ( len(package_tar_gz), ) )
    if expected_md5_digest is not None:
        m = hashlib.md5()
        m.update(package_tar_gz)
        actual_md5_digest = m.hexdigest()
        if verbose >= 2:
            myprint( 'md5:   actual %s\n     expected %s' % (actual_md5_digest,
                                                             expected_md5_digest))
        if actual_md5_digest != expected_md5_digest:
            raise ValueError('actual and expected md5 digests do not match')
    else:
        warnings.warn('no md5 digest found -- cannot verify source file')

    fd = open(fname,mode='wb')
    fd.write( package_tar_gz )
    fd.close()
    return fname
{'fraction_non_alphanumeric': 0.10153499507111674, 'fraction_numerical': 0.009576116039994366, 'mean_word_length': 2.1410880141530297, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 49, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5353812', 'n_tokens_mistral': 2188, 'n_tokens_neox': 2059, 'n_words': 530}