<commit_before>#! /usr/bin/env python

from __future__ import print_function

import os
import sys
import argparse
import shutil
import contextlib
import locale
import random
import textwrap
try:
    from subprocess import check_call, check_output
except ImportError:
    from subprocess import check_call, Popen, PIPE # py2.6 compatibility
    check_output = lambda *args: Popen(*args, stdout=PIPE).communicate()[0]
from glob import glob

locale.setlocale(locale.LC_ALL, '')


def update_args_dict():
    global args
    args = dict(enumerate(sys.argv))

update_args_dict()


def which_win(fn):
    def _access_check(fn):
        return (os.path.exists(fn) and os.access(fn, os.F_OK | os.X_OK)
                and not os.path.isdir(fn))
    pathext = os.environ.get("PATHEXT", "").split(os.pathsep)
    files = [fn + ext.lower() for ext in pathext]
    path = os.environ.get("PATH", os.defpath).split(os.pathsep)
    seen = set()
    for dir in map(os.path.normcase, path):
        if dir not in seen:
            seen.add(dir)
            for name in map(lambda f: os.path.join(dir, f), files):
                if _access_check(name):
                    return name


def resolve_path(f):
    if sys.platform != 'win32':
        return f<commit_msg>Make check_output fallback raise an exception when needed<commit_after>#! /usr/bin/env python

from __future__ import print_function

import os
import sys
import argparse
import shutil
import contextlib
import locale
import random
import textwrap
try:
    from subprocess import check_call, check_output
except ImportError:
    from subprocess import check_call, Popen, PIPE  # py2.6 compatibility

    def check_output(cmd, *args, **kwargs):
        popen = Popen(cmd, *args, stdout=PIPE, **kwargs)
        output = popen.communicate()[0]
        return_code = popen.poll()
        if return_code != 0:
            raise Exception('Command %r failed with return code %r' % (
                cmd, return_code))
        return output
from glob import glob

locale.setlocale(locale.LC_ALL, '')


def update_args_dict():
    global args
    args = dict(enumerate(sys.argv))

update_args_dict()


def which_win(fn):
    def _access_check(fn):
        return (os.path.exists(fn) and os.access(fn, os.F_OK | os.X_OK)
                and not os.path.isdir(fn))
    pathext = os.environ.get("PATHEXT", "").split(os.pathsep)
    files = [fn + ext.lower() for ext in pathext]
    path = os.environ.get("PATH", os.defpath).split(os.pathsep)
    seen = set()
    for dir in map(os.path.normcase, path):
        if dir not in seen:
            seen.add(dir)
            for name in map(lambda f: os.path.join(dir, f), files):
                if _access_check(name):
                    return name


def resolve_path(f):
    if sys.platform != 'win32':
        return f
{'fraction_non_alphanumeric': 0.10826210826210826, 'fraction_numerical': 0.003917378917378918, 'mean_word_length': 2.8321964529331516, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2613677', 'n_tokens_mistral': 911, 'n_tokens_neox': 897, 'n_words': 257}