<commit_before>import logging

from libc.stdint cimport int_fast64_t as int64
from libc.string cimport strcmp, strcpy
from cython cimport boundscheck, wraparound

from .bitarray cimport bit_index_t, bitarray
from PyMaSC.reader.bx.bigwig_file cimport BigWigFile

from PyMaSC.core.ncc import ReadUnsortedError
from PyMaSC.utils.progress import ProgressBar

logger = logging.getLogger(__name__)


cdef class CCBitArrayCalculator(object):
    cdef public double MAPPABILITY_THRESHOLD

    cdef readonly:
        int64 max_shift
        dict ref2genomelen
        int64 genomelen
        int64 forward_sum, reverse_sum
        dict ref2forward_sum, ref2reverse_sum
        dict ref2mappable_forward_sum,  ref2mappable_reverse_sum
        int64 forward_read_len_sum, reverse_read_len_sum
        dict ref2ccbins, ref2mascbins, ref2mappable_len
        bint skip_ncc
    cdef:
        str _chr
        list _solved_chr
        bitarray _forward_array, _reverse_array
        object _bwfeeder, logger_lock, _progress

        int64 _last_pos, _last_forward_pos, _last_reverse_pos
        bint _buff_flashed, _allow_bitarray_progress

    def __init__(self, int64 max_shift, references, lengths,
                 bwfeeder=None, skip_ncc=False, logger_lock=None, progress_bar=None):
        self.MAPPABILITY_THRESHOLD = 1.

        self.max_shift = max_shift
        self.ref2genomelen = dict(zip(references, lengths))
        self.genomelen = sum(lengths)
        # stats
        self.forward_sum = self.reverse_sum = 0
        self.ref2forward_sum = {ref: 0 for ref in references}
        self.ref2reverse_sum = {ref: 0 for ref in references}
        # {forward,reverse}_sum[d[0...i]] = Number of reads in Md
        self.ref2mappable_forward_sum = {ref: None for ref in references}
        self.ref2mappable_reverse_sum = {ref: None for ref in references}
        self.forward_read_len_sum = self.reverse_read_len_sum = 0
        self.ref2ccbins = {ref: None for ref in references}
        self.ref2mascbins = {ref: None for ref in references}
        self.ref2mappable_len = {ref: None for ref in references}

        # internal buff
        self._chr = ''
        self._solved_chr = []
        self._buff_flashed = False

        #
        self._bwfeeder = bwfeeder
        self.skip_ncc = skip_ncc
        self.logger_lock = logger_lock
        self._progress = progress_bar if progress_bar else ProgressBar()
        # Issue: bitarray progress stacks in multiprocessing
        self._allow_bitarray_progress = progress_bar is None

    def _logging_info(self, msg):
        if self.logger_lock:
            self.logger_lock.acquire()
        logger.info(msg)
        if self.logger_lock:
            self.logger_lock.release()

    def _init_progress(self, body, prefix, suffix, maxval):
        self._progress.body = body
        self._progress.fmt = "\r" + prefix + "{:<" + str(len(body)) + "}" + suffix
        self._progress.set(self._chr, maxval)

    def _init_mappability_progress(self):
        self._progress.enable_bar()
        self._init_progress("@@@@@^" * 12, '^', ' ', self.ref2genomelen[self._chr])

    def _init_bitarray_progress(self):
        if self._allow_bitarray_progress:
            self._init_progress("xxxxxX" * 12, 'X', ' ', self.max_shift)
        else:
            self._progress.disable_bar()

    def _init_buff(self):
        # To access by 1-based index, allocate arrays with 1 more size.
        chromsize = self.ref2genomelen[self._chr] + 1
        self._forward_array = bitarray(chromsize)
        self._reverse_array = bitarray(chromsize)

        self._last_pos = 0
        self._last_forward_pos = self._last_reverse_pos = 0

    def flush(self):
        if self._buff_flashed:
            return None

        self._calc_correlation()
        self._buff_flashed = True

    cdef inline int _calc_correlation(self) except -1:<commit_msg>fix some reverse reads become out of range<commit_after>import logging

from libc.stdint cimport int_fast64_t as int64
from libc.string cimport strcmp, strcpy
from cython cimport boundscheck, wraparound

from .bitarray cimport bit_index_t, bitarray
from PyMaSC.reader.bx.bigwig_file cimport BigWigFile

from PyMaSC.core.ncc import ReadUnsortedError
from PyMaSC.utils.progress import ProgressBar

logger = logging.getLogger(__name__)
MAX_READLEN = 1000


cdef class CCBitArrayCalculator(object):
    cdef public double MAPPABILITY_THRESHOLD

    cdef readonly:
        int64 max_shift
        dict ref2genomelen
        int64 genomelen
        int64 forward_sum, reverse_sum
        dict ref2forward_sum, ref2reverse_sum
        dict ref2mappable_forward_sum,  ref2mappable_reverse_sum
        int64 forward_read_len_sum, reverse_read_len_sum
        dict ref2ccbins, ref2mascbins, ref2mappable_len
        bint skip_ncc
    cdef:
        str _chr
        list _solved_chr
        bitarray _forward_array, _reverse_array
        object _bwfeeder, logger_lock, _progress

        int64 _last_pos, _last_forward_pos, _last_reverse_pos
        bint _buff_flashed, _allow_bitarray_progress

    def __init__(self, int64 max_shift, references, lengths,
                 bwfeeder=None, skip_ncc=False, logger_lock=None, progress_bar=None):
        self.MAPPABILITY_THRESHOLD = 1.

        self.max_shift = max_shift
        self.ref2genomelen = dict(zip(references, lengths))
        self.genomelen = sum(lengths)
        # stats
        self.forward_sum = self.reverse_sum = 0
        self.ref2forward_sum = {ref: 0 for ref in references}
        self.ref2reverse_sum = {ref: 0 for ref in references}
        # {forward,reverse}_sum[d[0...i]] = Number of reads in Md
        self.ref2mappable_forward_sum = {ref: None for ref in references}
        self.ref2mappable_reverse_sum = {ref: None for ref in references}
        self.forward_read_len_sum = self.reverse_read_len_sum = 0
        self.ref2ccbins = {ref: None for ref in references}
        self.ref2mascbins = {ref: None for ref in references}
        self.ref2mappable_len = {ref: None for ref in references}

        # internal buff
        self._chr = ''
        self._solved_chr = []
        self._buff_flashed = False

        #
        self._bwfeeder = bwfeeder
        self.skip_ncc = skip_ncc
        self.logger_lock = logger_lock
        self._progress = progress_bar if progress_bar else ProgressBar()
        # Issue: bitarray progress stacks in multiprocessing
        self._allow_bitarray_progress = progress_bar is None

    def _logging_info(self, msg):
        if self.logger_lock:
            self.logger_lock.acquire()
        logger.info(msg)
        if self.logger_lock:
            self.logger_lock.release()

    def _init_progress(self, body, prefix, suffix, maxval):
        self._progress.body = body
        self._progress.fmt = "\r" + prefix + "{:<" + str(len(body)) + "}" + suffix
        self._progress.set(self._chr, maxval)

    def _init_mappability_progress(self):
        self._progress.enable_bar()
        self._init_progress("@@@@@^" * 12, '^', ' ', self.ref2genomelen[self._chr])

    def _init_bitarray_progress(self):
        if self._allow_bitarray_progress:
            self._init_progress("xxxxxX" * 12, 'X', ' ', self.max_shift)
        else:
            self._progress.disable_bar()

    def _init_buff(self):
        # To access by 1-based index, allocate arrays with 1 more size.
        # Additionally, extra spaces same or more than (read_length - 1)
        # is needed to contain reverse strand reads.
        chromsize = self.ref2genomelen[self._chr] + MAX_READLEN
        self._forward_array = bitarray(chromsize)
        self._reverse_array = bitarray(chromsize)

        self._last_pos = 0
        self._last_forward_pos = self._last_reverse_pos = 0

    def flush(self):
        if self._buff_flashed:
            return None

        self._calc_correlation()
        self._buff_flashed = True

    cdef inline int _calc_correlation(self) except -1:
{'fraction_non_alphanumeric': 0.07738845475170154, 'fraction_numerical': 0.013108142172926645, 'mean_word_length': 2.75, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3789596', 'n_tokens_mistral': 2560, 'n_tokens_neox': 2450, 'n_words': 582}