<commit_before># -*- coding: utf-8 -*-

# Copyright (c) 2020 shmilee

'''
Contains Npz pickled file loader class.
'''

import numpy
import zipfile

from ..glogger import getGLogger
from .base import BasePckLoader

__all__ = ['NpzPckLoader']
log = getGLogger('L')


class NpzPckLoader(BasePckLoader):
    '''
    Load pickled data from ``.npz`` file. Return a dictionary-like object.

    Notes
    -----
    Q: How to read data from .npz file?
    A: npzfile[datakey]
    >>> npzfile = numpy.load('/tmp/test.npz')
    >>> datakey = 'group/key'
    >>> npzfile[datakey]
    '''
    __slots__ = []
    loader_type = '.npz'

    def _special_check_path(self):
        if zipfile.is_zipfile(self.path):
            return True
        else:
            log.error("'%s' is not a ZIP file!" % self.path)
            return False

    def _special_open(self):
        return numpy.load(self.path)

    def _special_close(self, pathobj):
        pathobj.close()<commit_msg>fix NpzPckLoader cannot load pickled Object arrays<commit_after># -*- coding: utf-8 -*-

# Copyright (c) 2020 shmilee

'''
Contains Npz pickled file loader class.
'''

import numpy
import zipfile

from ..glogger import getGLogger
from .base import BasePckLoader

__all__ = ['NpzPckLoader']
log = getGLogger('L')


class NpzPckLoader(BasePckLoader):
    '''
    Load pickled data from ``.npz`` file. Return a dictionary-like object.

    Notes
    -----
    Q: How to read data from .npz file?
    A: npzfile[datakey]
    >>> npzfile = numpy.load('/tmp/test.npz')
    >>> datakey = 'group/key'
    >>> npzfile[datakey]
    '''
    __slots__ = []
    loader_type = '.npz'

    def _special_check_path(self):
        if zipfile.is_zipfile(self.path):
            return True
        else:
            log.error("'%s' is not a ZIP file!" % self.path)
            return False

    def _special_open(self):
        return numpy.load(self.path, allow_pickle=True)

    def _special_close(self, pathobj):
        pathobj.close()
{'fraction_non_alphanumeric': 0.1299093655589124, 'fraction_numerical': 0.005035246727089627, 'mean_word_length': 2.5866425992779782, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7182441', 'n_tokens_mistral': 688, 'n_tokens_neox': 679, 'n_words': 175}