# Path: cjktools/common.py
# def sopen(filename, mode='rb', encoding='utf8'):
#     """
#     Transparently uses compression on the given file based on file
#     extension.
# 
#     :param filename:
#         The filename to use for the file handle.
#     
#     :param mode:
#         The mode to open the file in, e.g. ``'r'`` for read, ``'w'`` for
#         write, ``'a'`` for append.
#     
#     :param encoding:
#         The encoding to use. Can be set to None to avoid
#         using unicode at all.
#     """
#     read_mode = 'r' in mode
#     if read_mode and 'w' in mode:
#         raise Exception("Must be either read mode or write, but not both")
# 
#     # Do this before any streams are opened to hopefully minimize the potential
#     # for failing while resources are open.
#     if encoding not in (None, 'byte') and six.PY2:
#         if read_mode:
#             streamhandler = codecs.getreader(encoding)
#         else:
#             streamhandler = codecs.getwriter(encoding)
#     else:
#         streamhandler = lambda x: x
# 
#     if filename.endswith('.bz2'):
#         stream = bz2.BZ2File(filename, mode)
#     elif filename.endswith('.gz'):
#         stream = gzip.GzipFile(filename, mode)
#     elif filename == '-':
#         if read_mode:
#             stream = sys.stdin
#         else:
#             stream = sys.stdout
#     else:
#         stream = open(filename, mode)
# 
#     return streamhandler(stream)
# 
# class _NullContextWrapper(object):
#     """
#     Class for wrapping contexts so that they are passed through in a
#     with statement.
#     """
#     def __init__(self, context):
#         self.context = context
# 
#     def __enter__(self):
#         return self.context
# 
#     def __exit__(*args, **kwargs):
#         pass


import re
import itertools
import csv
from datetime import datetime
from collections import defaultdict, Mapping
from six import raise_from, iteritems
from six import PY2
from cjktools.common import sopen, _NullContextWrapper
    This is an abstract base class for reader classes which read Tatoeba data
    and expose it in a dictionary-like interface.
    """
    def __init__(self, *args, **kwargs):
        raise NotImplementedError('This is an abstract base class '
                                  'and should not be instantiated')

    def load_file(self, fpath_or_buf, **csv_kwargs):
        """
        A generator reading a given file line by line.

        :param fpath_or_buf:
            This can either be a file path or open file buffer.

        :param csv_kwargs:
            By default, the kwargs passed to :py:func:`csv.reader` are those for
            a standard Tatoeba file. You can pass additional keyword arguments
            here.
        """
        reader_kwargs = dict(delimiter='\t')
        reader_kwargs.update(csv_kwargs)

        if PY2:
            encoding = None
            encode_row = lambda row: [col.decode('utf-8') for col in row]
        else:
            encoding = 'utf-8'
            encode_row = lambda row: row

        if getattr(fpath_or_buf, 'read', None) is None:

# Path: cjktools/common.py
# def sopen(filename, mode='rb', encoding='utf8'):
#     """
#     Transparently uses compression on the given file based on file
#     extension.
# 
#     :param filename:
#         The filename to use for the file handle.
#     
#     :param mode:
#         The mode to open the file in, e.g. ``'r'`` for read, ``'w'`` for
#         write, ``'a'`` for append.
#     
#     :param encoding:
#         The encoding to use. Can be set to None to avoid
#         using unicode at all.
#     """
#     read_mode = 'r' in mode
#     if read_mode and 'w' in mode:
#         raise Exception("Must be either read mode or write, but not both")
# 
#     # Do this before any streams are opened to hopefully minimize the potential
#     # for failing while resources are open.
#     if encoding not in (None, 'byte') and six.PY2:
#         if read_mode:
#             streamhandler = codecs.getreader(encoding)
#         else:
#             streamhandler = codecs.getwriter(encoding)
#     else:
#         streamhandler = lambda x: x
# 
#     if filename.endswith('.bz2'):
#         stream = bz2.BZ2File(filename, mode)
#     elif filename.endswith('.gz'):
#         stream = gzip.GzipFile(filename, mode)
#     elif filename == '-':
#         if read_mode:
#             stream = sys.stdin
#         else:
#             stream = sys.stdout
#     else:
#         stream = open(filename, mode)
# 
#     return streamhandler(stream)
# 
# class _NullContextWrapper(object):
#     """
#     Class for wrapping contexts so that they are passed through in a
#     with statement.
#     """
#     def __init__(self, context):
#         self.context = context
# 
#     def __enter__(self):
#         return self.context
# 
#     def __exit__(*args, **kwargs):
#         pass


# Path: cjktools/resources/tatoeba.py
import re
import itertools
import csv
from datetime import datetime
from collections import defaultdict, Mapping
from six import raise_from, iteritems
from six import PY2
from cjktools.common import sopen, _NullContextWrapper

    This is an abstract base class for reader classes which read Tatoeba data
    and expose it in a dictionary-like interface.
    """
    def __init__(self, *args, **kwargs):
        raise NotImplementedError('This is an abstract base class '
                                  'and should not be instantiated')

    def load_file(self, fpath_or_buf, **csv_kwargs):
        """
        A generator reading a given file line by line.

        :param fpath_or_buf:
            This can either be a file path or open file buffer.

        :param csv_kwargs:
            By default, the kwargs passed to :py:func:`csv.reader` are those for
            a standard Tatoeba file. You can pass additional keyword arguments
            here.
        """
        reader_kwargs = dict(delimiter='\t')
        reader_kwargs.update(csv_kwargs)

        if PY2:
            encoding = None
            encode_row = lambda row: [col.decode('utf-8') for col in row]
        else:
            encoding = 'utf-8'
            encode_row = lambda row: row

        if getattr(fpath_or_buf, 'read', None) is None:

            cfile = sopen(fpath_or_buf, mode='r', encoding=encoding)