<commit_before>import collections
import functools
import locale
import os
import sys

try:
    from StringIO import StringIO
except ImportError:
    from io import StringIO

from .. import config

__all__ = [
    'StringIO', 'get_decoded', 'get_sorted', 'DefaultLocale', 'get_files',
    'get_dict', 'num', 'read_lines', 'exit_with_failure', 'CapturedStream'
]

try:
    STRING_TYPES = (str, bytes, unicode)
except NameError:
    STRING_TYPES = (str, bytes)

def get_decoded(items, encoding):
    """
    Return an iterator that yields all elements of `items` as unicode-strings.
    If `items` contains byte-strings then each of these strings is decoded to
    unicode by using the codec name specified by `encoding`. Items that are
    already unicode-strings are left unchanged. A `TypeError` is raised if
    `items` contains non-string elements.
    """
    for item in items:
        if not isinstance(item, STRING_TYPES):
            raise TypeError('all items must be strings')
        if isinstance(item, bytes):
            item = item.decode(encoding)
        yield item

def get_sorted(items, sortkey=None):
    """
    Sort given `items` in a locale-aware manner (i.e. ordering with respect to
    non-ascii characters that are specific to the current locale). Note that
    calling this function may temporary change the interpreter's global locale
    configuration and thus is not thread-safe.

    Use `sortkey` if you want to provide your own key to be used for sorting.
    """
    if sortkey is None:
        sortkey = functools.cmp_to_key(locale.strcoll)
    with DefaultLocale(locale.LC_COLLATE):
        return sorted(items, key=sortkey)


class DefaultLocale(object):
    def __init__(self, category, fail_on_locale_error=False):
        self.category = category
        self.fail_on_locale_error = fail_on_locale_error
        self.old_locale = None

    def __enter__(self):
        self.old_locale = locale.getlocale(self.category)
        try:
            locale.setlocale(self.category, '')
        except locale.Error as err:
            if self.fail_on_locale_error:
                raise err

    def __exit__(self, *unused):
        if self.old_locale is not None:
            locale.setlocale(self.category, self.old_locale)
        self.old_locale = None


def get_files(path, hide_dotted):
    """
    Return a sequence of the filenames in `path`. Note that this function does
    shell-like expansion of symbols such as "*", "?" or even "~" (user's home).

    `hide_dotted` defines whether to exclude filenames starting with a ".".
    """
    path = os.path.expanduser(os.path.expandvars(path))
    try:
        filenames = os.listdir(path)
    except OSError as err:
        filenames = glob.glob(path)
        if not filenames:
            raise err
    if hide_dotted:
        filenames = [fn for fn in filenames if not fn.startswith('.')]
    return filenames


def get_dict(mapping):
    """
    Return `mapping` as a dictionary. If mapping is already a `Mapping`-type
    then it is returned unchanged. Otherwise it is converted to an `OrderedDict`
    to preserve the ordering of its items. Typical candidates for this function
    are sequences of 2-element tuples (defining the mapping's items). In fact,
    this function is just a shorthand for ``collections.OrderedDict(mapping)``
    but with the pre-check mentioned above.
    """
    if isinstance(mapping, collections.Mapping):
        return mapping
    return collections.OrderedDict(mapping)


def num(s):
    """
    Return string `s` converted to a number. `s` is expected to represent an
    integer >= 0. An exception is raised if conversion failed or if the result
    is a negative number.
    """
    number = int(s)
    if number < 0:
        raise ValueError('number must be non-negative')
    return number

def read_lines(stream, column_index=None):
    """
    Return an iterator that yields all lines from `stream` removing any trailing
    "\n"-characters per line. `column_index` may be used to restrict reading to
    a specific column per line. Note that the index value is interpreted as an
    ordinary Python index. A column is defined as a sequence of non-whitespace
    characters. The column seperator is whitespace.
    """
    lines = (line.rstrip('\n') for line in stream)
    if column_index is not None:
        lines = (line.split()[column_index] for line in lines)
    return lines


def exit_with_failure(msg=None):
    """
    Exit the application with exit code 1. If `msg` is given then its text is
    printed to stderr.
    """
    if msg is not None:
        sys.stderr.write(msg + '\n')
    sys.exit(1)


class CapturedStream(object):
    """
    A class to temporary redirect data for a stream.
    """
    stream_locations = {
        'stdin': sys,
        'stdout': sys,
        'stderr': sys,
        'PRINT_STREAM': config,
    }

    def __init__(self, stream_name):
        """
        Create a new `CapturedStream`. Use `stream_name` to define the name of
        the stream whose data should be captured. This must be one of "stdin",
        "stdout", "stderr" or "PRINT_STREAM" in order to work.
        """
        if stream_name not in self.stream_locations:
            raise ValueError('Unknown stream name: {}'.format(stream_name))
        self.stream_name = stream_name
        self.location = self.stream_locations[stream_name]
        self.original_stream = getattr(self.location, stream_name)

    def capture(self):
        """
        Start capturing. A new stream that fetches all data for the desired
        standard stream is created and returned.
        """
        pseudo_stream = StringIO()
        setattr(self.location, self.stream_name, pseudo_stream)
        return pseudo_stream

    def uncapture(self):
        """
        Stop capturing. This will restore control by the original stream.
        """
        setattr(self.location, self.stream_name, self.original_stream)

    def __enter__(self):
        return self.capture()

    def __exit__(self, *unused):
        self.uncapture()<commit_msg>Mention that `CapturedStream` returns a `StringIO`-instance.<commit_after>import collections
import functools
import locale
import os
import sys

try:
    from StringIO import StringIO
except ImportError:
    from io import StringIO

from .. import config

__all__ = [
    'StringIO', 'get_decoded', 'get_sorted', 'DefaultLocale', 'get_files',
    'get_dict', 'num', 'read_lines', 'exit_with_failure', 'CapturedStream'
]

try:
    STRING_TYPES = (str, bytes, unicode)
except NameError:
    STRING_TYPES = (str, bytes)

def get_decoded(items, encoding):
    """
    Return an iterator that yields all elements of `items` as unicode-strings.
    If `items` contains byte-strings then each of these strings is decoded to
    unicode by using the codec name specified by `encoding`. Items that are
    already unicode-strings are left unchanged. A `TypeError` is raised if
    `items` contains non-string elements.
    """
    for item in items:
        if not isinstance(item, STRING_TYPES):
            raise TypeError('all items must be strings')
        if isinstance(item, bytes):
            item = item.decode(encoding)
        yield item

def get_sorted(items, sortkey=None):
    """
    Sort given `items` in a locale-aware manner (i.e. ordering with respect to
    non-ascii characters that are specific to the current locale). Note that
    calling this function may temporary change the interpreter's global locale
    configuration and thus is not thread-safe.

    Use `sortkey` if you want to provide your own key to be used for sorting.
    """
    if sortkey is None:
        sortkey = functools.cmp_to_key(locale.strcoll)
    with DefaultLocale(locale.LC_COLLATE):
        return sorted(items, key=sortkey)


class DefaultLocale(object):
    def __init__(self, category, fail_on_locale_error=False):
        self.category = category
        self.fail_on_locale_error = fail_on_locale_error
        self.old_locale = None

    def __enter__(self):
        self.old_locale = locale.getlocale(self.category)
        try:
            locale.setlocale(self.category, '')
        except locale.Error as err:
            if self.fail_on_locale_error:
                raise err

    def __exit__(self, *unused):
        if self.old_locale is not None:
            locale.setlocale(self.category, self.old_locale)
        self.old_locale = None


def get_files(path, hide_dotted):
    """
    Return a sequence of the filenames in `path`. Note that this function does
    shell-like expansion of symbols such as "*", "?" or even "~" (user's home).

    `hide_dotted` defines whether to exclude filenames starting with a ".".
    """
    path = os.path.expanduser(os.path.expandvars(path))
    try:
        filenames = os.listdir(path)
    except OSError as err:
        filenames = glob.glob(path)
        if not filenames:
            raise err
    if hide_dotted:
        filenames = [fn for fn in filenames if not fn.startswith('.')]
    return filenames


def get_dict(mapping):
    """
    Return `mapping` as a dictionary. If mapping is already a `Mapping`-type
    then it is returned unchanged. Otherwise it is converted to an `OrderedDict`
    to preserve the ordering of its items. Typical candidates for this function
    are sequences of 2-element tuples (defining the mapping's items). In fact,
    this function is just a shorthand for ``collections.OrderedDict(mapping)``
    but with the pre-check mentioned above.
    """
    if isinstance(mapping, collections.Mapping):
        return mapping
    return collections.OrderedDict(mapping)


def num(s):
    """
    Return string `s` converted to a number. `s` is expected to represent an
    integer >= 0. An exception is raised if conversion failed or if the result
    is a negative number.
    """
    number = int(s)
    if number < 0:
        raise ValueError('number must be non-negative')
    return number

def read_lines(stream, column_index=None):
    """
    Return an iterator that yields all lines from `stream` removing any trailing
    "\n"-characters per line. `column_index` may be used to restrict reading to
    a specific column per line. Note that the index value is interpreted as an
    ordinary Python index. A column is defined as a sequence of non-whitespace
    characters. The column seperator is whitespace.
    """
    lines = (line.rstrip('\n') for line in stream)
    if column_index is not None:
        lines = (line.split()[column_index] for line in lines)
    return lines


def exit_with_failure(msg=None):
    """
    Exit the application with exit code 1. If `msg` is given then its text is
    printed to stderr.
    """
    if msg is not None:
        sys.stderr.write(msg + '\n')
    sys.exit(1)


class CapturedStream(object):
    """
    A class to temporary redirect data for a stream.
    """
    stream_locations = {
        'stdin': sys,
        'stdout': sys,
        'stderr': sys,
        'PRINT_STREAM': config,
    }

    def __init__(self, stream_name):
        """
        Create a new `CapturedStream`. Use `stream_name` to define the name of
        the stream whose data should be captured. This must be one of "stdin",
        "stdout", "stderr" or "PRINT_STREAM" in order to work.
        """
        if stream_name not in self.stream_locations:
            raise ValueError('Unknown stream name: {}'.format(stream_name))
        self.stream_name = stream_name
        self.location = self.stream_locations[stream_name]
        self.original_stream = getattr(self.location, stream_name)

    def capture(self):
        """
        Start capturing. A new `StringIO`-instance that fetches all data for the
        desired stream is created and returned.
        """
        pseudo_stream = StringIO()
        setattr(self.location, self.stream_name, pseudo_stream)
        return pseudo_stream

    def uncapture(self):
        """
        Stop capturing. This will restore control by the original stream.
        """
        setattr(self.location, self.stream_name, self.original_stream)

    def __enter__(self):
        return self.capture()

    def __exit__(self, *unused):
        self.uncapture()
{'fraction_non_alphanumeric': 0.08621960206337509, 'fraction_numerical': 0.0008187996397281585, 'mean_word_length': 2.7790841584158414, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1033071', 'n_tokens_mistral': 3474, 'n_tokens_neox': 3400, 'n_words': 1259}