<commit_before>__all__ = ['savetxt', 'loadtxt', 'genfromtxt', 'ndfromtxt', 'mafromtxt',
        'recfromtxt', 'recfromcsv', 'load', 'loads', 'save', 'savez',
        'packbits', 'unpackbits', 'fromregex', 'DataSource']

import numpy as np
import format
import sys
import os
import sys
import itertools
import warnings
from operator import itemgetter

from cPickle import load as _cload, loads
from _datasource import DataSource
from _compiled_base import packbits, unpackbits

from _iotools import LineSplitter, NameValidator, StringConverter, \
                     ConverterError, ConverterLockError, ConversionWarning, \
                     _is_string_like, has_nested_fields, flatten_dtype, \
                     easy_dtype, _bytes_to_name

from numpy.compat import asbytes, asstr, asbytes_nested, bytes

if sys.version_info[0] >= 3:
    from io import BytesIO
else:
    from cStringIO import StringIO as BytesIO

_string_like = _is_string_like

def seek_gzip_factory(f):
    """Use this factory to produce the class so that we can do a lazy
    import on gzip.

    """
    import gzip

    def seek(self, offset, whence=0):
        # figure out new position (we can only seek forwards)
        if whence == 1:
            offset = self.offset + offset

        if whence not in [0, 1]:
            raise IOError, "Illegal argument"

        if offset < self.offset:
            # for negative seek, rewind and do positive seek
            self.rewind()
            count = offset - self.offset
            for i in range(count // 1024):
                self.read(1024)
            self.read(count % 1024)

    def tell(self):
        return self.offset

    if isinstance(f, str):
        f = gzip.GzipFile(f)

    if sys.version_info[0] >= 3:
        import types
        f.seek = types.MethodType(seek, f)
        f.tell = types.MethodType(tell, f)
    else:
        import new
        f.seek = new.instancemethod(seek, f)
        f.tell = new.instancemethod(tell, f)

    return f



class BagObj(object):
    """
    BagObj(obj)

    Convert attribute lookups to getitems on the object passed in.

    Parameters
    ----------
    obj : class instance
        Object on which attribute lookup is performed.

    Examples
    --------
    >>> class BagDemo(object):
    ...     def __getitem__(self, key):
    ...         return key
    ...
    >>> demo_obj = BagDemo()
    >>> bagobj = np.lib.npyio.BagObj(demo_obj)
    >>> bagobj.some_item
    'some_item'

    """
    def __init__(self, obj):
        self._obj = obj
    def __getattribute__(self, key):
        try:
            return object.__getattribute__(self, '_obj')[key]
        except KeyError:
            raise AttributeError, key



class NpzFile(object):
    """
    NpzFile(fid)

    A dictionary-like object with lazy-loading of files in the zipped
    archive provided on construction.

    `NpzFile` is used to load files in the NumPy ``.npz`` data archive
    format. It assumes that files in the archive have a ".npy" extension,
    other files are ignored.

    The arrays and file strings are lazily loaded on either
    getitem access using ``obj['key']`` or attribute lookup using
    ``obj.f.key``. A list of all files (without ".npy" extensions) can
    be obtained with ``obj.files`` and the ZipFile object itself using
    ``obj.zip``.

    Attributes
    ----------
    files : list of str
        List of all files in the archive with a ".npy" extension.
    zip : ZipFile instance
        The ZipFile object initialized with the zipped archive.
    f : BagObj instance
        An object on which attribute can be performed as an alternative
        to getitem access on the `NpzFile` instance itself.

    Parameters
    ----------
    fid : file or str
        The zipped archive to open. This is either a file-like object
        or a string containing the path to the archive.

    Examples
    --------
    >>> from tempfile import TemporaryFile
    >>> outfile = TemporaryFile()
    >>> x = np.arange(10)
    >>> y = np.sin(x)
    >>> np.savez(outfile, x=x, y=y)
    >>> outfile.seek(0)

    >>> npz = np.load(outfile)
    >>> isinstance(npz, np.lib.npyio.NpzFile)
    True
    >>> npz.files<commit_msg>DOC: merge wiki edits for module npyio.<commit_after>__all__ = ['savetxt', 'loadtxt', 'genfromtxt', 'ndfromtxt', 'mafromtxt',
        'recfromtxt', 'recfromcsv', 'load', 'loads', 'save', 'savez',
        'packbits', 'unpackbits', 'fromregex', 'DataSource']

import numpy as np
import format
import sys
import os
import sys
import itertools
import warnings
from operator import itemgetter

from cPickle import load as _cload, loads
from _datasource import DataSource
from _compiled_base import packbits, unpackbits

from _iotools import LineSplitter, NameValidator, StringConverter, \
                     ConverterError, ConverterLockError, ConversionWarning, \
                     _is_string_like, has_nested_fields, flatten_dtype, \
                     easy_dtype, _bytes_to_name

from numpy.compat import asbytes, asstr, asbytes_nested, bytes

if sys.version_info[0] >= 3:
    from io import BytesIO
else:
    from cStringIO import StringIO as BytesIO

_string_like = _is_string_like

def seek_gzip_factory(f):
    """Use this factory to produce the class so that we can do a lazy
    import on gzip.

    """
    import gzip

    def seek(self, offset, whence=0):
        # figure out new position (we can only seek forwards)
        if whence == 1:
            offset = self.offset + offset

        if whence not in [0, 1]:
            raise IOError, "Illegal argument"

        if offset < self.offset:
            # for negative seek, rewind and do positive seek
            self.rewind()
            count = offset - self.offset
            for i in range(count // 1024):
                self.read(1024)
            self.read(count % 1024)

    def tell(self):
        return self.offset

    if isinstance(f, str):
        f = gzip.GzipFile(f)

    if sys.version_info[0] >= 3:
        import types
        f.seek = types.MethodType(seek, f)
        f.tell = types.MethodType(tell, f)
    else:
        import new
        f.seek = new.instancemethod(seek, f)
        f.tell = new.instancemethod(tell, f)

    return f



class BagObj(object):
    """
    BagObj(obj)

    Convert attribute look-ups to getitems on the object passed in.

    Parameters
    ----------
    obj : class instance
        Object on which attribute look-up is performed.

    Examples
    --------
    >>> from numpy.lib.npyio import BagObj as BO
    >>> class BagDemo(object):
    ...     def __getitem__(self, key): # An instance of BagObj(BagDemo)
    ...                                 # will call this method when any
    ...                                 # attribute look-up is required
    ...         result = "Doesn't matter what you want, "
    ...         return result + "you're gonna get this"
    ...
    >>> demo_obj = BagDemo()
    >>> bagobj = BO(demo_obj)
    >>> bagobj.hello_there
    "Doesn't matter what you want, you're gonna get this"
    >>> bagobj.I_can_be_anything
    "Doesn't matter what you want, you're gonna get this"

    """
    def __init__(self, obj):
        self._obj = obj
    def __getattribute__(self, key):
        try:
            return object.__getattribute__(self, '_obj')[key]
        except KeyError:
            raise AttributeError, key



class NpzFile(object):
    """
    NpzFile(fid)

    A dictionary-like object with lazy-loading of files in the zipped
    archive provided on construction.

    `NpzFile` is used to load files in the NumPy ``.npz`` data archive
    format. It assumes that files in the archive have a ".npy" extension,
    other files are ignored.

    The arrays and file strings are lazily loaded on either
    getitem access using ``obj['key']`` or attribute lookup using
    ``obj.f.key``. A list of all files (without ".npy" extensions) can
    be obtained with ``obj.files`` and the ZipFile object itself using
    ``obj.zip``.

    Attributes
    ----------
    files : list of str
        List of all files in the archive with a ".npy" extension.
    zip : ZipFile instance
        The ZipFile object initialized with the zipped archive.
    f : BagObj instance
        An object on which attribute can be performed as an alternative
        to getitem access on the `NpzFile` instance itself.

    Parameters
    ----------
    fid : file or str
        The zipped archive to open. This is either a file-like object
        or a string containing the path to the archive.

    Examples
    --------
    >>> from tempfile import TemporaryFile
    >>> outfile = TemporaryFile()
    >>> x = np.arange(10)
    >>> y = np.sin(x)
    >>> np.savez(outfile, x=x, y=y)
    >>> outfile.seek(0)

    >>> npz = np.load(outfile)
    >>> isinstance(npz, np.lib.io.NpzFile)
    True
    >>> npz.files
{'fraction_non_alphanumeric': 0.10064312309601715, 'fraction_numerical': 0.005190116213471737, 'mean_word_length': 2.243322356384925, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 91, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1412256', 'n_tokens_mistral': 2677, 'n_tokens_neox': 2550, 'n_words': 910}