<commit_before># -*- coding: utf-8 -*-

"""
I/O routines.

io.py
(c) <NAME>  23 June, 2017
"""

import gzip
import sys
import zipfile
from collections import Counter

from .compat import basestring

try:
    import lzma
except ImportError:
    try:
        import backports.lzma as lzma
    except ImportError:
        pass


def open_file(filename, mode='rb'):
    """
    Open a file for access with the given mode.

    This function transparently wraps gzip and xz files as well as
    normal files.  You can also open zip files using syntax like::

        f = utils.open_file('../semcor-parsed.zip:semcor000.txt')

    :param str filename: The name of the file to open
    :param str mode: The mode to open the file in (defaults to 'rb')
    """
    if (('r' not in mode or hasattr(filename, 'read')) and
        (('a' not in mode and 'w' not in mode) or
         hasattr(filename, 'write')) and hasattr(filename, '__iter__')):
        return filename
    elif isinstance(filename, basestring):
        if filename == '-' and 'r' in mode:
            return sys.stdin
        elif filename == '-' and ('w' in mode or 'a' in mode):
            return sys.stdout
        if filename.lower().count('.zip:'):
            if 'r' not in mode:
                raise Exception(
                    'zip file syntax only supports reading from files')
            mode = mode.replace('b', '')
            assert filename.count(':') == 1
            zipped_file = zipfile.ZipFile(filename.split(':')[0])
            unzipped_file = zipped_file.open(filename.split(':')[1], 'r')
            zipped_file.close()
            return unzipped_file
        elif filename.lower().endswith('.gz'):
            return gzip.open(filename, mode)
        elif filename.lower().endswith('.xz'):
            tmp = lzma.LZMAFile(filename, mode)
            dir(tmp)
            return tmp
        else:
            return open(filename, mode)
    else:
        raise Exception('Unknown type for argument filename')


def lines(filename, encoding='utf-8'):
    """
    Open the named file and yield the lines inside it.

    :param str filename: The name of the file to open
    :param str encoding: The encoding of the file (defaults to utf-8)
    """
    decode = lambda line: line  # noqa E731
    if encoding:
        decode = lambda line: line.decode(encoding)  # noqa E731
    with open(filename, 'rb') as input_file:
        for line in input_file:
            yield decode(line)


def read_counts(filename, encoding='utf-8'):
    """
    Load a tab-separated count file into a Counter structure.<commit_msg>wkr.io.lines: use codecs to properly decode encoded text<commit_after># -*- coding: utf-8 -*-

"""
I/O routines.

io.py
(c) <NAME>  23 June, 2017
"""

import codecs
import gzip
import sys
import zipfile
from collections import Counter

from .compat import basestring

try:
    import lzma
except ImportError:
    try:
        import backports.lzma as lzma
    except ImportError:
        pass


def open_file(filename, mode='rb'):
    """
    Open a file for access with the given mode.

    This function transparently wraps gzip and xz files as well as
    normal files.  You can also open zip files using syntax like::

        f = utils.open_file('../semcor-parsed.zip:semcor000.txt')

    :param str filename: The name of the file to open
    :param str mode: The mode to open the file in (defaults to 'rb')
    """
    if (('r' not in mode or hasattr(filename, 'read')) and
        (('a' not in mode and 'w' not in mode) or
         hasattr(filename, 'write')) and hasattr(filename, '__iter__')):
        return filename
    elif isinstance(filename, basestring):
        if filename == '-' and 'r' in mode:
            return sys.stdin
        elif filename == '-' and ('w' in mode or 'a' in mode):
            return sys.stdout
        if filename.lower().count('.zip:'):
            if 'r' not in mode:
                raise Exception(
                    'zip file syntax only supports reading from files')
            mode = mode.replace('b', '')
            assert filename.count(':') == 1
            zipped_file = zipfile.ZipFile(filename.split(':')[0])
            unzipped_file = zipped_file.open(filename.split(':')[1], 'r')
            zipped_file.close()
            return unzipped_file
        elif filename.lower().endswith('.gz'):
            return gzip.open(filename, mode)
        elif filename.lower().endswith('.xz'):
            tmp = lzma.LZMAFile(filename, mode)
            dir(tmp)
            return tmp
        else:
            return open(filename, mode)
    else:
        raise Exception('Unknown type for argument filename')


def lines(filename, encoding='utf-8'):
    """
    Open the named file and yield the lines inside it.

    :param str filename: The name of the file to open
    :param str encoding: The encoding of the file (defaults to utf-8)
    """
    with open(filename, 'rb') as input_file:
        if encoding is not None:
            stream = codecs.getreader(encoding)(input_file)
        else:
            stream = input_file
        for line in stream:
            yield line


def read_counts(filename, encoding='utf-8'):
    """
    Load a tab-separated count file into a Counter structure.
{'fraction_non_alphanumeric': 0.10865421273351125, 'fraction_numerical': 0.0072436141822340835, 'mean_word_length': 2.263059701492537, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1612332', 'n_tokens_mistral': 1544, 'n_tokens_neox': 1493, 'n_words': 546}