<commit_before>#!/usr/bin/python
from urlparse import urlparse
import urllib
import urllib2
import click
import os
import hashlib
import logging
import cargoport
from cargoport.utils import yield_packages, package_name, PACKAGE_SERVER, depot_url
logging.basicConfig(level=logging.DEBUG)
log = logging.getLogger()


@click.command()
@click.option('--package_id', help='Package ID', required=True)
@click.option('--package_version', help="Package version, downloads all versions if not specified", default=None, required=False)
@click.option('--download_location', default='./',
              help='Location for the downloaded file')
def get(package_id, package_version, download_location):
    package_found = False
    database = PACKAGE_SERVER + 'urls.tsv'
    log.info("Searching for package: "+str(package_id)+" in "+str(database))
    if not os.path.exists(download_location):
        os.makedirs(download_location)
    
    for ld in yield_packages(urllib2.urlopen(database)):
        # TODO: check platform/architecture, failover to all if available?
        # iid, version, platform, architecture, upstream_url, checksum, alternate_url = line.split('\t')
        if ld['id'] == package_id.strip() and ld['platform']== 'src' and (package_version == None or ld['version'] == package_version):
            package_found = True
            # I worry about this being unreliable. TODO: add target filename column?
            pkg_name = package_name(ld)
            storage_path = os.path.join(download_location, pkg_name)
            url = depot_url(ld)
            urllib.urlretrieve(url, storage_path)
            download_checksum = hashlib.sha256(open(storage_path, 'rb').read()).hexdigest()
            if ld['sha256sum'] != download_checksum:
                print ('Checksum does not match, something seems to be wrong.\n'
                       '{expected}\t(expected)\n{actual}\t(downloaded)').format(
                           expected=ld['sha'],<commit_msg>Small change to gsl.py that uses log.error when checksums don't match<commit_after>#!/usr/bin/python
from urlparse import urlparse
import urllib
import urllib2
import click
import os
import hashlib
import logging
import cargoport
from cargoport.utils import yield_packages, package_name, PACKAGE_SERVER, depot_url
logging.basicConfig(level=logging.DEBUG)
log = logging.getLogger()


@click.command()
@click.option('--package_id', help='Package ID', required=True)
@click.option('--package_version', help="Package version, downloads all versions if not specified", default=None, required=False)
@click.option('--download_location', default='./',
              help='Location for the downloaded file')
def get(package_id, package_version, download_location):
    package_found = False
    database = PACKAGE_SERVER + 'urls.tsv'
    log.info("Searching for package: "+str(package_id)+" in "+str(database))
    if not os.path.exists(download_location):
        os.makedirs(download_location)
    
    for ld in yield_packages(urllib2.urlopen(database)):
        # TODO: check platform/architecture, failover to all if available?
        # iid, version, platform, architecture, upstream_url, checksum, alternate_url = line.split('\t')
        if ld['id'] == package_id.strip() and ld['platform']== 'src' and (package_version == None or ld['version'] == package_version):
            package_found = True
            # I worry about this being unreliable. TODO: add target filename column?
            pkg_name = package_name(ld)
            storage_path = os.path.join(download_location, pkg_name)
            url = depot_url(ld)
            urllib.urlretrieve(url, storage_path)
            download_checksum = hashlib.sha256(open(storage_path, 'rb').read()).hexdigest()
            if ld['sha256sum'] != download_checksum:
                log.error('Checksum does not match, something seems to be wrong.\n'
                       '{expected}\t(expected)\n{actual}\t(downloaded)').format(
                           expected=ld['sha256sum'],
{'fraction_non_alphanumeric': 0.11673734098278872, 'fraction_numerical': 0.004739336492890996, 'mean_word_length': 3.531073446327684, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6829504', 'n_tokens_mistral': 1172, 'n_tokens_neox': 1135, 'n_words': 294}