<commit_before>#!/usr/bin/env python

###################################################################
#  cpuinfo - Get information about CPU
#
#      License: BSD
#      Author:  <NAME> <<EMAIL>>
#
#  See LICENSES/cpuinfo.txt for details about copyright and
#  rights to use.
####################################################################

"""
cpuinfo

Copyright 2002 <NAME> all rights reserved,
<NAME> <<EMAIL>>
Permission to use, modify, and distribute this software is given under the
terms of the NumPy (BSD style) license.  See LICENSE.txt that came with
this distribution for specifics.

NO WARRANTY IS EXPRESSED OR IMPLIED.  USE AT YOUR OWN RISK.
<NAME>
"""

__all__ = ['cpu']

import sys, re, types
import os
import subprocess
import warnings
import platform

def getoutput(cmd, successful_status=(0,), stacklevel=1):
    try:
        p = subprocess.Popen(cmd, stdout=subprocess.PIPE)
        output, _ = p.communicate()
        status = p.returncode
    except EnvironmentError, e:
        warnings.warn(str(e), UserWarning, stacklevel=stacklevel)
        return False, ''
    if os.WIFEXITED(status) and os.WEXITSTATUS(status) in successful_status:
        return True, output
    return False, output

def command_info(successful_status=(0,), stacklevel=1, **kw):
    info = {}
    for key in kw:
        ok, output = getoutput(kw[key], successful_status=successful_status,
                               stacklevel=stacklevel+1)
        if ok:
            info[key] = output.strip()
    return info

def command_by_line(cmd, successful_status=(0,), stacklevel=1):
    ok, output = getoutput(cmd, successful_status=successful_status,
                           stacklevel=stacklevel+1)
    if not ok:
        return
    for line in output.splitlines():
        yield line.strip()

def key_value_from_command(cmd, sep, successful_status=(0,),
                           stacklevel=1):
    d = {}
    for line in command_by_line(cmd, successful_status=successful_status,
                                stacklevel=stacklevel+1):
        l = [s.strip() for s in line.split(sep, 1)]
        if len(l) == 2:
            d[l[0]] = l[1]
    return d

class CPUInfoBase(object):
    """Holds CPU information and provides methods for requiring
    the availability of various CPU features.
    """

    def _try_call(self,func):
        try:
            return func()
        except:<commit_msg>Decode subprocess output<commit_after>#!/usr/bin/env python

###################################################################
#  cpuinfo - Get information about CPU
#
#      License: BSD
#      Author:  <NAME> <<EMAIL>>
#
#  See LICENSES/cpuinfo.txt for details about copyright and
#  rights to use.
####################################################################

"""
cpuinfo

Copyright 2002 <NAME> all rights reserved,
<NAME> <<EMAIL>>
Permission to use, modify, and distribute this software is given under the
terms of the NumPy (BSD style) license.  See LICENSE.txt that came with
this distribution for specifics.

NO WARRANTY IS EXPRESSED OR IMPLIED.  USE AT YOUR OWN RISK.
<NAME>
"""

__all__ = ['cpu']

import sys, re, types
import os
import subprocess
import warnings
import platform

def getoutput(cmd, successful_status=(0,), stacklevel=1):
    try:
        p = subprocess.Popen(cmd, stdout=subprocess.PIPE)
        output, _ = p.communicate()
        status = p.returncode
    except EnvironmentError, e:
        warnings.warn(str(e), UserWarning, stacklevel=stacklevel)
        return False, ''
    if os.WIFEXITED(status) and os.WEXITSTATUS(status) in successful_status:
        return True, output
    return False, output

def command_info(successful_status=(0,), stacklevel=1, **kw):
    info = {}
    for key in kw:
        ok, output = getoutput(kw[key], successful_status=successful_status,
                               stacklevel=stacklevel+1)
        if ok:
            info[key] = output.strip()
    return info

def command_by_line(cmd, successful_status=(0,), stacklevel=1):
    ok, output = getoutput(cmd, successful_status=successful_status,
                           stacklevel=stacklevel+1)
    if not ok:
        return

    # XXX: check
    output = output.decode('ascii')

    for line in output.splitlines():
        yield line.strip()

def key_value_from_command(cmd, sep, successful_status=(0,),
                           stacklevel=1):
    d = {}
    for line in command_by_line(cmd, successful_status=successful_status,
                                stacklevel=stacklevel+1):
        l = [s.strip() for s in line.split(sep, 1)]
        if len(l) == 2:
            d[l[0]] = l[1]
    return d

class CPUInfoBase(object):
    """Holds CPU information and provides methods for requiring
    the availability of various CPU features.
    """

    def _try_call(self,func):
        try:
            return func()
        except:
{'fraction_non_alphanumeric': 0.15122950819672132, 'fraction_numerical': 0.00778688524590164, 'mean_word_length': 2.886146496815287, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7085928', 'n_tokens_mistral': 1409, 'n_tokens_neox': 1385, 'n_words': 413}