<commit_before>from __future__ import print_function
import argparse
import os, sys, subprocess, json, re

def run(*args):
    ret = subprocess.call(args,  stdout=sys.stdout, stderr=sys.stderr)
    if ret != 0:
        exit(ret)

def run_filter(filter, *args):
    replacement = 'X' * len(filter)
    try:
        output = subprocess.check_output(args,
                                         stderr = subprocess.STDOUT)
    except subprocess.CalledProcessError as e:
        print(e.output.decode('utf-8').replace(filter, replacement))
        exit(e.returncode)
    print(output.decode('utf-8').replace(filter, replacement))

def run_output(*args):
    try:
        output = subprocess.check_output(args,
                                         stderr=sys.stderr)
    except subprocess.CalledProcessError as e:
        print(e.output.decode('utf-8'))
        exit(e.returncode)
    return output.decode('utf-8')

def target_binary_name(target):
    return target['name'].replace('-', '_') + target['metadata']['extra_filename']

class Manifest(object):
    def __init__(self, dir, version):
        # the --manifest-path behaviour changed in
        # https://github.com/rust-lang/cargo/pull/1955, so we need to
        # be careful to handle both
        path_file = os.path.join(dir, 'Cargo.toml')
        path_dir = dir

        try:
            output = subprocess.check_output(['cargo', 'read-manifest', '--manifest-path',
                                              path_file],
                                             stderr=subprocess.STDOUT)
        except subprocess.CalledProcessError as e:
            try:
                output = subprocess.check_output(['cargo', 'read-manifest', '--manifest-path',
                                                  path_dir],
                                                 stderr=subprocess.STDOUT)
            except subprocess.CalledProcessError as e2:
                print('Cargo failed to read `--manifest-path {}`'
                      ' and `--manifest-path {}`:' % (path_file, path_dir),
                      file = sys.sdterr)
                print(e.output.decode('utf-8'))
                print(e2.output.decode('utf-8'))
                exit(e.returncode)

        self.manifest = json.loads(output.decode('utf-8'))

    def targets(self):
        return self.manifest['targets']
    def lib_name(self):
        for target in self.targets():
            if 'lib' in target['kind']:
                return target['name'].replace('-', '_')
        return None


def add_features(cargo_args, version):
    nightly_feature = os.environ.get('TRAVIS_CARGO_NIGHTLY_FEATURE', 'unstable')
    if version == 'nightly' and nightly_feature != '':
        # only touch feature arguments when we are actually going to
        # add something non-trivial, avoids problems like that in
        # issue #14 (can't use -p ... on nightly even with an empty<commit_msg>Use new-style formatter<commit_after>from __future__ import print_function
import argparse
import os, sys, subprocess, json, re

def run(*args):
    ret = subprocess.call(args,  stdout=sys.stdout, stderr=sys.stderr)
    if ret != 0:
        exit(ret)

def run_filter(filter, *args):
    replacement = 'X' * len(filter)
    try:
        output = subprocess.check_output(args,
                                         stderr = subprocess.STDOUT)
    except subprocess.CalledProcessError as e:
        print(e.output.decode('utf-8').replace(filter, replacement))
        exit(e.returncode)
    print(output.decode('utf-8').replace(filter, replacement))

def run_output(*args):
    try:
        output = subprocess.check_output(args,
                                         stderr=sys.stderr)
    except subprocess.CalledProcessError as e:
        print(e.output.decode('utf-8'))
        exit(e.returncode)
    return output.decode('utf-8')

def target_binary_name(target):
    return target['name'].replace('-', '_') + target['metadata']['extra_filename']

class Manifest(object):
    def __init__(self, dir, version):
        # the --manifest-path behaviour changed in
        # https://github.com/rust-lang/cargo/pull/1955, so we need to
        # be careful to handle both
        path_file = os.path.join(dir, 'Cargo.toml')
        path_dir = dir

        try:
            output = subprocess.check_output(['cargo', 'read-manifest', '--manifest-path',
                                              path_file],
                                             stderr=subprocess.STDOUT)
        except subprocess.CalledProcessError as e:
            try:
                output = subprocess.check_output(['cargo', 'read-manifest', '--manifest-path',
                                                  path_dir],
                                                 stderr=subprocess.STDOUT)
            except subprocess.CalledProcessError as e2:
                print('Cargo failed to read `--manifest-path {}`'
                      ' and `--manifest-path {}`:'.format(path_file, path_dir),
                      file = sys.sdterr)
                print(e.output.decode('utf-8'))
                print(e2.output.decode('utf-8'))
                exit(e.returncode)

        self.manifest = json.loads(output.decode('utf-8'))

    def targets(self):
        return self.manifest['targets']
    def lib_name(self):
        for target in self.targets():
            if 'lib' in target['kind']:
                return target['name'].replace('-', '_')
        return None


def add_features(cargo_args, version):
    nightly_feature = os.environ.get('TRAVIS_CARGO_NIGHTLY_FEATURE', 'unstable')
    if version == 'nightly' and nightly_feature != '':
        # only touch feature arguments when we are actually going to
        # add something non-trivial, avoids problems like that in
        # issue #14 (can't use -p ... on nightly even with an empty
{'fraction_non_alphanumeric': 0.11821903787103377, 'fraction_numerical': 0.005458887751620608, 'mean_word_length': 1.9447513812154695, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1626636', 'n_tokens_mistral': 1639, 'n_tokens_neox': 1579, 'n_words': 367}