<commit_before>#!/usr/bin/env python
r"""HDF5 demultiplexed DDL

Attributes off of ./ for the full file:

    n         : int, the number of sequences
    max       : int, the max sequence length
    min       : int, the min sequence length
    mean      : float, the mean sequence length
    std       : float, the standard deviation of sequence length
    median    : float, the median sequence length
    hist      : np.array of int, 10 bin histogram of sequence lengths
    hist_edge : np.array of int, left edge of each bin

Each sample has its own group with the following structure:

    ./<sample_name>/sequence          : (N,) of str where N is the number of \
sequences in the sample
    ./<sample_name>/qual              : (N, M) of int where N is the number \
of sequences in the sample, and M is the max sequence length (file-wide)
    ./<sample_name>/barcode/corrected : (N,) of str where N is the number of \
sequences in the sample
    ./<sample_name>/barcode/original  : (N,) of str where N is the number of \
sequences in the sample
    ./<sample_name>/barcode/error     : (N,) of int where N is the number of
sequences in the sample

Each sample additionally has the following attributes described on the
sample group:

    n         : int, the number of sequences
    max       : int, the max sequence length
    min       : int, the min sequence length
    mean      : float, the mean sequence length
    std       : float, the standard deviation of sequence length
    median    : float, the median sequence length
    hist      : np.array of int, 10 bin histogram of sequence lengths
    hist_edge : np.array of int, left edge of each bin

"""
from __future__ import division

# -----------------------------------------------------------------------------
# Copyright (c) 2014--, The Qiita Development Team.
#
# Distributed under the terms of the BSD 3-clause License.
#
# The full license is in the file LICENSE, distributed with this software.
# -----------------------------------------------------------------------------

import os
from functools import partial
from itertools import repeat
from collections import defaultdict, namedtuple

import numpy as np
from numpy.random import shuffle
from future.utils import viewitems, viewvalues
from future.builtins import zip
from skbio.parse.sequences import load
from skbio.format.sequences import format_fastq_record

# track some basic stats about the samples
stat = namedtuple('stat', 'n max min mean median std hist hist_edge')

# centralized incase paths change
dset_paths = {'sequence': 'sequence',
              'barcode_original': 'barcode/original',
              'barcode_corrected': 'barcode/corrected',
              'barcode_error': 'barcode/error',
              'qual': 'qual'}


class _buffer(object):
    """Buffer baseclass that sits on top of an HDF5 dataset

    Notes
    -----
    The intent of the buffer is to minimize direct writes against HDF5 datasets
    and to instead perform bulk operations against the h5py API. The 'stdio'
    driver for h5py also enables system buffers, but in practice, there is
    still a large amount of overhead when writing small pieces of data
    incrementally to h5py datasets.
    """
    def __init__(self, dset, max_fill=10000):
        """Construct thy self

        Parameters
        ----------
        dset : h5py.Dataset
            The dataset to buffer
        max_fill : unsigned int
            The maximum fill for the buffer
        """
        self.dset = dset

        self._current_fill = 0
        self._n = 0
        self._idx = 0
        self._max_fill = max_fill
        self._alloc()

    def __del__(self):
        """Flush when the buffer is deconstructed"""
        if self._n > 0:
            self.flush()

    def write(self, data):
        """Deposit into the buffer, write to dataset if necessary

        Parameters
        ----------
        data : scalar or np.array
            The data is dependent on the underlying buffer
        """
        self._write(data)
        self._n += 1

        if self.is_full():
            self.flush()

    def _write(self, data):<commit_msg>STY: removed extra variable (excited for flake8...), thanks @adamrp<commit_after>#!/usr/bin/env python
r"""HDF5 demultiplexed DDL

Attributes off of ./ for the full file:

    n         : int, the number of sequences
    max       : int, the max sequence length
    min       : int, the min sequence length
    mean      : float, the mean sequence length
    std       : float, the standard deviation of sequence length
    median    : float, the median sequence length
    hist      : np.array of int, 10 bin histogram of sequence lengths
    hist_edge : np.array of int, left edge of each bin

Each sample has its own group with the following structure:

    ./<sample_name>/sequence          : (N,) of str where N is the number of \
sequences in the sample
    ./<sample_name>/qual              : (N, M) of int where N is the number \
of sequences in the sample, and M is the max sequence length (file-wide)
    ./<sample_name>/barcode/corrected : (N,) of str where N is the number of \
sequences in the sample
    ./<sample_name>/barcode/original  : (N,) of str where N is the number of \
sequences in the sample
    ./<sample_name>/barcode/error     : (N,) of int where N is the number of
sequences in the sample

Each sample additionally has the following attributes described on the
sample group:

    n         : int, the number of sequences
    max       : int, the max sequence length
    min       : int, the min sequence length
    mean      : float, the mean sequence length
    std       : float, the standard deviation of sequence length
    median    : float, the median sequence length
    hist      : np.array of int, 10 bin histogram of sequence lengths
    hist_edge : np.array of int, left edge of each bin

"""
from __future__ import division

# -----------------------------------------------------------------------------
# Copyright (c) 2014--, The Qiita Development Team.
#
# Distributed under the terms of the BSD 3-clause License.
#
# The full license is in the file LICENSE, distributed with this software.
# -----------------------------------------------------------------------------

import os
from functools import partial
from itertools import repeat
from collections import defaultdict, namedtuple

import numpy as np
from numpy.random import shuffle
from future.utils import viewitems, viewvalues
from future.builtins import zip
from skbio.parse.sequences import load
from skbio.format.sequences import format_fastq_record

# track some basic stats about the samples
stat = namedtuple('stat', 'n max min mean median std hist hist_edge')

# centralized incase paths change
dset_paths = {'sequence': 'sequence',
              'barcode_original': 'barcode/original',
              'barcode_corrected': 'barcode/corrected',
              'barcode_error': 'barcode/error',
              'qual': 'qual'}


class _buffer(object):
    """Buffer baseclass that sits on top of an HDF5 dataset

    Notes
    -----
    The intent of the buffer is to minimize direct writes against HDF5 datasets
    and to instead perform bulk operations against the h5py API. The 'stdio'
    driver for h5py also enables system buffers, but in practice, there is
    still a large amount of overhead when writing small pieces of data
    incrementally to h5py datasets.
    """
    def __init__(self, dset, max_fill=10000):
        """Construct thy self

        Parameters
        ----------
        dset : h5py.Dataset
            The dataset to buffer
        max_fill : unsigned int
            The maximum fill for the buffer
        """
        self.dset = dset

        self._n = 0
        self._idx = 0
        self._max_fill = max_fill
        self._alloc()

    def __del__(self):
        """Flush when the buffer is deconstructed"""
        if self._n > 0:
            self.flush()

    def write(self, data):
        """Deposit into the buffer, write to dataset if necessary

        Parameters
        ----------
        data : scalar or np.array
            The data is dependent on the underlying buffer
        """
        self._write(data)
        self._n += 1

        if self.is_full():
            self.flush()

    def _write(self, data):
{'fraction_non_alphanumeric': 0.10871918220084185, 'fraction_numerical': 0.006253758268190018, 'mean_word_length': 2.126315789473684, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6469015', 'n_tokens_mistral': 2196, 'n_tokens_neox': 2085, 'n_words': 959}