<commit_before># Copyright 2015-2016 The Rust Project Developers. See the COPYRIGHT
# file at the top-level directory of this distribution and at
# http://rust-lang.org/COPYRIGHT.
#
# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
# option. This file may not be copied, modified, or distributed
# except according to those terms.

import argparse
import contextlib
import os
import shutil
import subprocess
import sys
import tarfile

def get(url, path, verbose=False):
    print("downloading " + url)
    # see http://serverfault.com/questions/301128/how-to-download
    if sys.platform == 'win32':
        run(["PowerShell.exe", "/nologo", "-Command",
             "(New-Object System.Net.WebClient).DownloadFile('" + url +
                "', '" + path + "')"], verbose=verbose)
    else:
        run(["curl", "-o", path, url], verbose=verbose)

def unpack(tarball, dst, verbose=False, match=None):
    print("extracting " + tarball)
    fname = os.path.basename(tarball).replace(".tar.gz", "")
    with contextlib.closing(tarfile.open(tarball)) as tar:
        for p in tar.getnames():
            if "/" not in p:
                continue<commit_msg>rustbuild: Verify sha256 of downloaded tarballs<commit_after># Copyright 2015-2016 The Rust Project Developers. See the COPYRIGHT
# file at the top-level directory of this distribution and at
# http://rust-lang.org/COPYRIGHT.
#
# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
# option. This file may not be copied, modified, or distributed
# except according to those terms.

import argparse
import contextlib
import hashlib
import os
import shutil
import subprocess
import sys
import tarfile

def get(url, path, verbose=False):
    print("downloading " + url)
    sha_url = url + ".sha256"
    sha_path = path + ".sha256"
    for _url, _path in ((url, path), (sha_url, sha_path)):
        # see http://serverfault.com/questions/301128/how-to-download
        if sys.platform == 'win32':
            run(["PowerShell.exe", "/nologo", "-Command",
                 "(New-Object System.Net.WebClient)"
                 ".DownloadFile('{}', '{}')".format(_url, _path)],
                verbose=verbose)
        else:
            run(["curl", "-o", _path, _url], verbose=verbose)
    print("verifying " + path)
    with open(path, "rb") as f:
        found = hashlib.sha256(f.read()).hexdigest()
    with open(sha_path, "r") as f:
        expected, _ = f.readline().split()
    if found != expected:
        err = ("invalid checksum:\n"
               "    found:    {}\n"
               "    expected: {}".format(found, expected))
        if verbose:
            raise RuntimeError(err)
        sys.exit(err)

def unpack(tarball, dst, verbose=False, match=None):
    print("extracting " + tarball)
    fname = os.path.basename(tarball).replace(".tar.gz", "")
    with contextlib.closing(tarfile.open(tarball)) as tar:
        for p in tar.getnames():
            if "/" not in p:
                continue
{'fraction_non_alphanumeric': 0.1360418590335488, 'fraction_numerical': 0.01600492459218221, 'mean_word_length': 2.8644470868014267, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 9, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2103331', 'n_tokens_mistral': 1031, 'n_tokens_neox': 965, 'n_words': 285}