<commit_before>import argparse
import os, sys, subprocess, json, re

def run(*args):
    ret = subprocess.call(args,  stdout=sys.stdout, stderr=sys.stderr)
    if ret != 0:
        exit(ret)
def run_output(*args):
    try:
        output = subprocess.check_output(args,
                                         stderr=sys.stderr)
    except subprocess.CalledProcessError as e:
        print(e.output.decode())
        exit(e.returncode)
    return output.decode()

def target_binary_name(target):
    return target['name'].replace('-', '_') + target['metadata']['extra_filename']

class Manifest(object):
    def __init__(self, dir):
        self.manifest = json.loads(run_output('cargo', 'read-manifest', '--manifest-path', dir))
    def targets(self):
        return self.manifest['targets']
    def lib_name(self):
        for target in self.targets():
            if 'lib' in target['kind']:
                return target['name'].replace('-', '_')
        return None


def add_features(cargo_args, version):
    nightly_feature = os.environ.get('TRAVIS_CARGO_NIGHTLY_FEATURE', 'unstable')
    if version == 'nightly':
        for i in range(0, len(cargo_args)):
            if cargo_args[i] == '--features':
                cargo_args[i + 1] += ' ' + nightly_feature
                break
        else:
            cargo_args += ['--features', nightly_feature]


def cargo_raw(feature, version, manifest, args):
    cargo_args = args.cargo_args
    subcommand = args.subcommand
    if subcommand == 'bench' and version != 'nightly':
        print('skipping `cargo bench` on non-nightly version')
        return

    if feature:
        add_features(cargo_args, version)

    if not args.quiet and '--verbose' not in args and '-v' not in cargo_args:
        cargo_args.append('--verbose')

    run('cargo', subcommand, *cargo_args)

def cargo_feature(version, manifest, args):
    cargo_raw(True, version, manifest, args)
def cargo_no_feature(version, manifest, args):
    cargo_raw(False, version, manifest, args)

def doc_upload(version, manifest, args):
    branch = os.environ['TRAVIS_BRANCH']
    pr = os.environ['TRAVIS_PULL_REQUEST']
    token = os.environ['GH_TOKEN']
    repo = os.environ['TRAVIS_REPO_SLUG']

    lib_name = manifest.lib_name()
    if lib_name is None:
        sys.stderr.write('error: uploading docs for package with no library')
        exit(1)

    if branch == 'master' and pr == 'false':
        print('uploading docs...')
        sys.stdout.flush()
        with open('target/doc/index.html', 'w') as f:
            f.write('<meta http-equiv=refresh content=0;url=%s/index.html>' % lib_name)

        run('git', 'clone', 'https://github.com/davisp/ghp-import')
        run('./ghp-import/ghp-import', '-n', 'target/doc')
        run('git', 'push', '-fq', 'https://%s@github.com/%s.git' % (token, repo), 'gh-pages')

def build_kcov(use_sudo):
    deps = ''
    if use_sudo:
        deps = 'sudo apt-get install libcurl4-openssl-dev libelf-dev libdw-dev cmake'
    init = deps + '''
    wget https://github.com/SimonKagstrom/kcov/archive/master.zip
    unzip master.zip
    mv kcov-master kcov
    mkdir kcov/build
    '''
    for line in init.split('\n'):
        line = line.strip()
        if line:
            print('Running: %s' % line)
            run(*line.split())
    current = os.getcwd()
    os.chdir('kcov/build')
    build = '''
    cmake ..
    make
    make install DESTDIR=../built
    '''
    for line in build.split('\n'):
        line = line.strip()
        if line:
            print('Running: %s' % line)
            run(*line.split())
    os.chdir(current)
    return os.path.join(current, 'kcov/built/usr/local/bin/kcov')

def raw_coverage(use_sudo, test_args, merge_msg, kcov_merge_args, kcov_merge_dir):
    kcov = build_kcov(use_sudo)

    test_binaries = []
    # look through the output of `cargo test` to find the test
    # binaries.
    # FIXME: the information cargo feeds us is
    # inconsistent/inaccurate, so using the output of read-manifest is
    # far too much trouble.

    output = run_output('cargo', 'test', *test_args)
    running = re.compile('^     Running target/debug/(.*)$', re.M)
    for line in running.finditer(output):
        test_binaries.append(line.group(1))


    # record coverage for each binary
    for binary in test_binaries:
        print('Recording %s' % binary)
        run(kcov, '--exclude-pattern=/.cargo', 'target/kcov-' + binary,
            'target/debug/' + binary)
    # merge all the coverages and upload in one go
    print(merge_msg)
    kcov_args = [kcov, '--merge'] + kcov_merge_args + [kcov_merge_dir]<commit_msg>Add `--verify` to kcov args.<commit_after>import argparse
import os, sys, subprocess, json, re

def run(*args):
    ret = subprocess.call(args,  stdout=sys.stdout, stderr=sys.stderr)
    if ret != 0:
        exit(ret)
def run_output(*args):
    try:
        output = subprocess.check_output(args,
                                         stderr=sys.stderr)
    except subprocess.CalledProcessError as e:
        print(e.output.decode())
        exit(e.returncode)
    return output.decode()

def target_binary_name(target):
    return target['name'].replace('-', '_') + target['metadata']['extra_filename']

class Manifest(object):
    def __init__(self, dir):
        self.manifest = json.loads(run_output('cargo', 'read-manifest', '--manifest-path', dir))
    def targets(self):
        return self.manifest['targets']
    def lib_name(self):
        for target in self.targets():
            if 'lib' in target['kind']:
                return target['name'].replace('-', '_')
        return None


def add_features(cargo_args, version):
    nightly_feature = os.environ.get('TRAVIS_CARGO_NIGHTLY_FEATURE', 'unstable')
    if version == 'nightly':
        for i in range(0, len(cargo_args)):
            if cargo_args[i] == '--features':
                cargo_args[i + 1] += ' ' + nightly_feature
                break
        else:
            cargo_args += ['--features', nightly_feature]


def cargo_raw(feature, version, manifest, args):
    cargo_args = args.cargo_args
    subcommand = args.subcommand
    if subcommand == 'bench' and version != 'nightly':
        print('skipping `cargo bench` on non-nightly version')
        return

    if feature:
        add_features(cargo_args, version)

    if not args.quiet and '--verbose' not in args and '-v' not in cargo_args:
        cargo_args.append('--verbose')

    run('cargo', subcommand, *cargo_args)

def cargo_feature(version, manifest, args):
    cargo_raw(True, version, manifest, args)
def cargo_no_feature(version, manifest, args):
    cargo_raw(False, version, manifest, args)

def doc_upload(version, manifest, args):
    branch = os.environ['TRAVIS_BRANCH']
    pr = os.environ['TRAVIS_PULL_REQUEST']
    token = os.environ['GH_TOKEN']
    repo = os.environ['TRAVIS_REPO_SLUG']

    lib_name = manifest.lib_name()
    if lib_name is None:
        sys.stderr.write('error: uploading docs for package with no library')
        exit(1)

    if branch == 'master' and pr == 'false':
        print('uploading docs...')
        sys.stdout.flush()
        with open('target/doc/index.html', 'w') as f:
            f.write('<meta http-equiv=refresh content=0;url=%s/index.html>' % lib_name)

        run('git', 'clone', 'https://github.com/davisp/ghp-import')
        run('./ghp-import/ghp-import', '-n', 'target/doc')
        run('git', 'push', '-fq', 'https://%s@github.com/%s.git' % (token, repo), 'gh-pages')

def build_kcov(use_sudo):
    deps = ''
    if use_sudo:
        deps = 'sudo apt-get install libcurl4-openssl-dev libelf-dev libdw-dev cmake'
    init = deps + '''
    wget https://github.com/SimonKagstrom/kcov/archive/master.zip
    unzip master.zip
    mv kcov-master kcov
    mkdir kcov/build
    '''
    for line in init.split('\n'):
        line = line.strip()
        if line:
            print('Running: %s' % line)
            run(*line.split())
    current = os.getcwd()
    os.chdir('kcov/build')
    build = '''
    cmake ..
    make
    make install DESTDIR=../built
    '''
    for line in build.split('\n'):
        line = line.strip()
        if line:
            print('Running: %s' % line)
            run(*line.split())
    os.chdir(current)
    return os.path.join(current, 'kcov/built/usr/local/bin/kcov')

def raw_coverage(use_sudo, test_args, merge_msg, kcov_merge_args, kcov_merge_dir):
    kcov = build_kcov(use_sudo)

    test_binaries = []
    # look through the output of `cargo test` to find the test
    # binaries.
    # FIXME: the information cargo feeds us is
    # inconsistent/inaccurate, so using the output of read-manifest is
    # far too much trouble.

    output = run_output('cargo', 'test', *test_args)
    running = re.compile('^     Running target/debug/(.*)$', re.M)
    for line in running.finditer(output):
        test_binaries.append(line.group(1))


    # record coverage for each binary
    for binary in test_binaries:
        print('Recording %s' % binary)
        run(kcov, '--verify --exclude-pattern=/.cargo', 'target/kcov-' + binary,
            'target/debug/' + binary)
    # merge all the coverages and upload in one go
    print(merge_msg)
    kcov_args = [kcov, '--merge'] + kcov_merge_args + [kcov_merge_dir]
{'fraction_non_alphanumeric': 0.14093163683416002, 'fraction_numerical': 0.0015095967220185465, 'mean_word_length': 2.664559462662979, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2706700', 'n_tokens_mistral': 3048, 'n_tokens_neox': 2985, 'n_words': 714}