<commit_before># -*- coding: utf-8 -*-

## Copyright 2015 <NAME>
##
## Licensed under the Apache License, Version 2.0 (the "License"); you may
## not use this file except in compliance with the License.  You may obtain
## a copy of the License at
##
##        http://www.apache.org/licenses/LICENSE-2.0
##
## Unless required by applicable law or agreed to in writing, software
## distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
## WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
## License for the specific language governing permissions and limitations
## under the License.


'''
Normalization of alleles relative to a reference sequence
'''


from __future__       import division, print_function

import sys

from collections import namedtuple
from itertools   import chain


class NormalizationError(ValueError):
    pass


class ReferenceMismatch(NormalizationError):
    pass


normalized_alleles = namedtuple('shuffled_alleles', 'start stop alleles')


cpdef trim_common_suffixes(strs, int min_len=0, int max_trim=0):
    '''trim common suffixes'''

    if len(strs) < 2:
        return 0, strs

    rev_strs = [ s[::-1] for s in strs ]

    trimmed, rev_strs = trim_common_prefixes(rev_strs, min_len, max_trim)

    if trimmed:
        strs = [ s[::-1] for s in rev_strs ]

    return trimmed, strs


cpdef trim_common_prefixes(strs, int min_len=0, int max_trim=0):
    '''trim common prefixes'''

    cdef int i, trimmed = 0
    cdef bytes s1, s2

    if len(strs) > 1:
        s1 = min(strs)
        s2 = max(strs)

        for i in range(len(s1) - min_len):
            if s1[i] != s2[i]:
                break
            trimmed = i + 1

    if 0 < max_trim < trimmed:
        trimmed = max_trim

    if trimmed > 0:
        strs = [ s[trimmed:] for s in strs ]

    return trimmed, strs


cdef shuffle_left(bytes ref, int *start, int *stop, alleles, int bound, int ref_step):
    cdef int trimmed, step, left, n = len(alleles)

    while 0 < alleles.count('') < n and start[0] > bound:
        step = min(ref_step, start[0] - bound)

        r = ref[start[0] - step:start[0]].upper()
        new_alleles = [ r+a for a in alleles ]

        trimmed, new_alleles = trim_common_suffixes(new_alleles)

        if not trimmed:
            break

        start[0] -= trimmed
        stop[0]  -= trimmed

        if trimmed == step:
            alleles = new_alleles
        else:
            left    = step - trimmed
            alleles = [ a[left:] for a in new_alleles ]
            break

    return alleles


cdef shuffle_right(bytes ref, int *start, int *stop, alleles, int bound, int ref_step):
    cdef int trimmed, step, left, n = len(alleles)

    while 0 < alleles.count('') < n and stop[0] < bound:
        step = min(ref_step, bound - stop[0])
        r = ref[stop[0]:stop[0]+step].upper()
        new_alleles = [ a+r for a in alleles ]

        trimmed, new_alleles = trim_common_prefixes(new_alleles)

        if not trimmed:
            break

        start[0] += trimmed
        stop[0]  += trimmed

        if trimmed == step:
            alleles = new_alleles
        else:
            left    = step - trimmed
            alleles = [ a[:-left] for a in new_alleles ]
            break

    return alleles


cpdef normalize_alleles(bytes ref, int start, int stop, alleles, int bound=-1, int ref_step=24, left=True, bint shuffle=True):
    if left:
        if bound < 0:
            bound = 0
        return normalize_alleles_left(ref, start, stop, alleles, bound, ref_step, shuffle)
    else:
        if bound < 0:
            bound = len(ref)
        return normalize_alleles_right(ref, start, stop, alleles, bound, ref_step, shuffle)


cdef normalize_alleles_left(bytes ref, int start, int stop, alleles, int bound, int ref_step=24, bint shuffle=True):
    '''Normalize loci by removing extraneous reference padding'''
    cdef int trimmed

    if alleles[0] != ref[start:stop]:
        raise ReferenceMismatch('Reference alleles does not match reference sequence: {} != {}'.format(alleles[0], ref[start:stop]))

    if len(alleles) < 2 or start <= 0 or start <= 0:
        return normalized_alleles(start, stop, alleles)

    # STEP 0: Trim prefixes if needed to clear bound
    if start < bound:
        trimmed, alleles = trim_common_prefixes(alleles, max_trim=bound - start)
        start += trimmed

    # STEP 1: Trim common suffix
    trimmed, alleles = trim_common_suffixes(alleles)<commit_msg>Fix typo in bound check<commit_after># -*- coding: utf-8 -*-

## Copyright 2015 <NAME>
##
## Licensed under the Apache License, Version 2.0 (the "License"); you may
## not use this file except in compliance with the License.  You may obtain
## a copy of the License at
##
##        http://www.apache.org/licenses/LICENSE-2.0
##
## Unless required by applicable law or agreed to in writing, software
## distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
## WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
## License for the specific language governing permissions and limitations
## under the License.


'''
Normalization of alleles relative to a reference sequence
'''


from __future__       import division, print_function

import sys

from collections import namedtuple
from itertools   import chain


class NormalizationError(ValueError):
    pass


class ReferenceMismatch(NormalizationError):
    pass


normalized_alleles = namedtuple('shuffled_alleles', 'start stop alleles')


cpdef trim_common_suffixes(strs, int min_len=0, int max_trim=0):
    '''trim common suffixes'''

    if len(strs) < 2:
        return 0, strs

    rev_strs = [ s[::-1] for s in strs ]

    trimmed, rev_strs = trim_common_prefixes(rev_strs, min_len, max_trim)

    if trimmed:
        strs = [ s[::-1] for s in rev_strs ]

    return trimmed, strs


cpdef trim_common_prefixes(strs, int min_len=0, int max_trim=0):
    '''trim common prefixes'''

    cdef int i, trimmed = 0
    cdef bytes s1, s2

    if len(strs) > 1:
        s1 = min(strs)
        s2 = max(strs)

        for i in range(len(s1) - min_len):
            if s1[i] != s2[i]:
                break
            trimmed = i + 1

    if 0 < max_trim < trimmed:
        trimmed = max_trim

    if trimmed > 0:
        strs = [ s[trimmed:] for s in strs ]

    return trimmed, strs


cdef shuffle_left(bytes ref, int *start, int *stop, alleles, int bound, int ref_step):
    cdef int trimmed, step, left, n = len(alleles)

    while 0 < alleles.count('') < n and start[0] > bound:
        step = min(ref_step, start[0] - bound)

        r = ref[start[0] - step:start[0]].upper()
        new_alleles = [ r+a for a in alleles ]

        trimmed, new_alleles = trim_common_suffixes(new_alleles)

        if not trimmed:
            break

        start[0] -= trimmed
        stop[0]  -= trimmed

        if trimmed == step:
            alleles = new_alleles
        else:
            left    = step - trimmed
            alleles = [ a[left:] for a in new_alleles ]
            break

    return alleles


cdef shuffle_right(bytes ref, int *start, int *stop, alleles, int bound, int ref_step):
    cdef int trimmed, step, left, n = len(alleles)

    while 0 < alleles.count('') < n and stop[0] < bound:
        step = min(ref_step, bound - stop[0])
        r = ref[stop[0]:stop[0]+step].upper()
        new_alleles = [ a+r for a in alleles ]

        trimmed, new_alleles = trim_common_prefixes(new_alleles)

        if not trimmed:
            break

        start[0] += trimmed
        stop[0]  += trimmed

        if trimmed == step:
            alleles = new_alleles
        else:
            left    = step - trimmed
            alleles = [ a[:-left] for a in new_alleles ]
            break

    return alleles


cpdef normalize_alleles(bytes ref, int start, int stop, alleles, int bound=-1, int ref_step=24, left=True, bint shuffle=True):
    if left:
        if bound < 0:
            bound = 0
        return normalize_alleles_left(ref, start, stop, alleles, bound, ref_step, shuffle)
    else:
        if bound < 0:
            bound = len(ref)
        return normalize_alleles_right(ref, start, stop, alleles, bound, ref_step, shuffle)


cdef normalize_alleles_left(bytes ref, int start, int stop, alleles, int bound, int ref_step=24, bint shuffle=True):
    '''Normalize loci by removing extraneous reference padding'''
    cdef int trimmed

    if alleles[0] != ref[start:stop]:
        raise ReferenceMismatch('Reference alleles does not match reference sequence: {} != {}'.format(alleles[0], ref[start:stop]))

    if len(alleles) < 2 or start <= 0 or bound <= 0:
        return normalized_alleles(start, stop, alleles)

    # STEP 0: Trim prefixes if needed to clear bound
    if start < bound:
        trimmed, alleles = trim_common_prefixes(alleles, max_trim=bound - start)
        start += trimmed

    # STEP 1: Trim common suffix
    trimmed, alleles = trim_common_suffixes(alleles)
{'fraction_non_alphanumeric': 0.10049019607843138, 'fraction_numerical': 0.012923351158645277, 'mean_word_length': 2.5864962045545346, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2045682', 'n_tokens_mistral': 2956, 'n_tokens_neox': 2720, 'n_words': 935}