<commit_before># -*- coding: utf-8 -*-
"""
This module provides alternative implementations of array interfaces defined in
the :mod:`allel.model` module, using `bcolz <http://bcolz.blosc.org>`_
compressed arrays (:class:`bcolz.carray`) instead of numpy arrays for data
storage. Compressed arrays can use either main memory or be stored on disk.
In either case, the use of compressed arrays enables analysis of data that
are too large to fit uncompressed into main memory.

"""
from __future__ import absolute_import, print_function, division


import operator
from allel.compat import range


import numpy as np
import bcolz


# monkey-patch bcolz.ctable.dtype property because it's broken for
# multi-dimensional columns

def _ctable_dtype(self):
    """The data type of this object (numpy dtype)."""
    names, cols = self.names, self.cols
    l = []
    for name in names:
        col = cols[name]
        if col.ndim == 1:
            t = (name, col.dtype)
        else:
            t = (name, (col.dtype, col.shape[1:]))
        l.append(t)
    return np.dtype(l)

bcolz.ctable.dtype = property(_ctable_dtype)


from allel.model import GenotypeArray, HaplotypeArray, AlleleCountsArray, \
    SortedIndex, SortedMultiIndex, subset, VariantTable
from allel.constants import DIM_PLOIDY
from allel.util import asarray_ndim
from allel.io import write_vcf_header, write_vcf_data


__all__ = ['GenotypeCArray', 'HaplotypeCArray', 'AlleleCountsCArray']


def carray_block_map(carr, f, out=None, blen=None, **kwargs):
    kwargs.setdefault('expectedlen', carr.shape[0])
    if blen is None:
        blen = carr.chunklen<commit_msg>monkey-patch bcolz.ctable.addcol<commit_after># -*- coding: utf-8 -*-
"""
This module provides alternative implementations of array interfaces defined in
the :mod:`allel.model` module, using `bcolz <http://bcolz.blosc.org>`_
compressed arrays (:class:`bcolz.carray`) instead of numpy arrays for data
storage. Compressed arrays can use either main memory or be stored on disk.
In either case, the use of compressed arrays enables analysis of data that
are too large to fit uncompressed into main memory.

"""
from __future__ import absolute_import, print_function, division


import os
import operator
from allel.compat import range


import numpy as np
import bcolz


from allel.model import GenotypeArray, HaplotypeArray, AlleleCountsArray, \
    SortedIndex, SortedMultiIndex, subset, VariantTable
from allel.constants import DIM_PLOIDY
from allel.util import asarray_ndim
from allel.io import write_vcf_header, write_vcf_data


__all__ = ['GenotypeCArray', 'HaplotypeCArray', 'AlleleCountsCArray',
           'VariantCTable']


# monkey-patch bcolz.ctable.dtype property because it's broken for
# multi-dimensional columns

def _ctable_dtype(self):
    """The data type of this object (numpy dtype)."""
    names, cols = self.names, self.cols
    l = []
    for name in names:
        col = cols[name]
        if col.ndim == 1:
            t = (name, col.dtype)
        else:
            t = (name, (col.dtype, col.shape[1:]))
        l.append(t)
    return np.dtype(l)

bcolz.ctable.dtype = property(_ctable_dtype)


# monkey-patch bcolz.ctable.addcol because it's broken for persistent ctables

_ctable_addcol_original = bcolz.ctable.addcol


def _ctable_addcol(self, newcol, name, **kwargs):
    # require name to simplify monkey-patch
    if self.rootdir is not None:
        rootdir = os.path.join(self.rootdir, name)
        kwargs['rootdir'] = rootdir
        kwargs['mode'] = 'w'
        _ctable_addcol_original(self, newcol, name=name, **kwargs)
        if isinstance(newcol, bcolz.carray):
            # ensure carrays are actually written to disk
            newcol.copy(**kwargs)
    else:
        _ctable_addcol_original(self, newcol, name=name, **kwargs)

bcolz.ctable.addcol = _ctable_addcol


def carray_block_map(carr, f, out=None, blen=None, **kwargs):
    kwargs.setdefault('expectedlen', carr.shape[0])
    if blen is None:
        blen = carr.chunklen
{'fraction_non_alphanumeric': 0.09745127436281859, 'fraction_numerical': 0.001999000499750125, 'mean_word_length': 3.7038777908343126, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2917462', 'n_tokens_mistral': 1302, 'n_tokens_neox': 1273, 'n_words': 393}