<commit_before>import re


whitespace = {'\t', '\n', '\x0b', '\x0c', '\r', '\x1c', '\x1d', '\x1e', '\x1f', ' ',
              '\x85', '\xa0', '\u1680', '\u2000', '\u2001', '\u2002', '\u2003',
              '\u2004', '\u2005', '\u2006', '\u2007', '\u2008', '\u2009', '\u200a',
              '\u2028', '\u2029', '\u202f', '\u205f', '\u3000'}
label_pattern = re.compile(r'\[(.+?)(?<!\])\]', re.DOTALL)
punctuation = {'!', '"', '#', '$', '%', '&', '\'', '(', ')', '*', '+', ',',
               '-', '.', '/', ':', ';', '<', '=', '>', '?', '@', '[', '\\',
               ']', '^', '_', '`', '{', '|', '}', '~'}


def find_core_tokens(string):
    delimiters = []
    matches = []
    escaped = False
    in_delimiter_run = None
    in_image = False
    start = 0
    i = 0
    while i < len(string):
        c = string[i]
        if c == '\\':
            escaped = True
            i += 1
            continue
        if in_delimiter_run is not None and c != in_delimiter_run or escaped:
            delimiters.append(Delimiter(start, i if not escaped else i-1, string))
            in_delimiter_run = None
        if in_delimiter_run is None and (c == '*' or c == '_') and not escaped:
            in_delimiter_run = c
            start = i
        if not escaped:
            if c == '[':
                if not in_image:
                    delimiters.append(Delimiter(i, i+1, string))
                else:
                    delimiters.append(Delimiter(i-1, i+1, string))
                    in_image = False
            elif c == '!':
                in_image = True
            elif c == ']':
                i = find_link_image(string, i, delimiters, matches)
            elif in_image:
                in_image = False
        else:
            escaped = False
        i += 1
    if in_delimiter_run:
        delimiters.append(Delimiter(start, i, string))
    process_emphasis(string, 0, delimiters, matches)
    return matches


def find_link_image(string, offset, delimiters, matches):
    i = len(delimiters) - 1
    for delimiter in delimiters[::-1]:
        # found a link/image delimiter
        if delimiter.type in ('[', '!['):
            # not active, remove delimiter
            if not delimiter.active:
                delimiters.remove(delimiter)
                return offset
            match = match_link_image(string, offset, delimiter)
            # found match
            if match:
                # parse for emphasis
                process_emphasis(string, i, delimiters, matches)
                # append current match
                matches.append(match)
                # if match is a link, set all previous links to be inactive
                if delimiter.type == '[':
                    deactivate_delimiters(delimiters, i, '[')
                # shift index till end of match
                return match.end() - 1
            # no match, remove delimiter
            delimiters.remove(delimiter)
            return offset
        i -= 1
    # no link/image delimiter
    return offset


def process_emphasis(string, stack_bottom, delimiters, matches):
    star_bottom = stack_bottom
    underscore_bottom = stack_bottom
    curr_pos = next_closer(stack_bottom, delimiters)
    while curr_pos is not None:
        closer = delimiters[curr_pos]
        bottom = star_bottom if closer.type[0] == '*' else underscore_bottom
        open_pos = matching_opener(curr_pos, delimiters, bottom)
        if open_pos is not None:
            opener = delimiters[open_pos]
            n = 2 if closer.number >= 2 and opener.number >= 2 else 1
            start = opener.end - n
            end = closer.start + n
            match = MatchObj(start, end, (start+n, end-n, string[start+n:end-n]))
            match.type = 'Strong' if n == 2 else 'Emphasis'
            matches.append(match)
            if not opener.remove(n, left=False):
                delimiters.remove(opener)
                curr_pos -= 2
                if curr_pos < 0: curr_pos = 0
            if not closer.remove(n, left=True):
                delimiters.remove(closer)
                curr_pos -= 2
                if curr_pos < 0: curr_pos = 0
        else:
            if closer.type[0] == '*':
                star_bottom = curr_pos - 1
            else:
                underscore_bottom = curr_pos - 1
            if not closer.open:
                delimiters.remove(closer)
                curr_pos -= 2
                if curr_pos < 0: curr_pos = 0
        curr_pos = next_closer(curr_pos, delimiters)
    del delimiters[stack_bottom:]


def match_link_image(string, offset, delimiter):
    image = delimiter.type == '!['
    start = delimiter.start
    text_start = start + delimiter.number
    text_end = offset
    text = string[text_start:text_end]
    # inline link
    if follows(string, offset, '('):
        # link destination
        match_info = match_link_dest(string, offset+1)
        if match_info is None:
            return None
        dest_start, dest_end, dest = match_info
        # link title<commit_msg>removing from delimiter always decrement curr_pos<commit_after>import re


whitespace = {'\t', '\n', '\x0b', '\x0c', '\r', '\x1c', '\x1d', '\x1e', '\x1f', ' ',
              '\x85', '\xa0', '\u1680', '\u2000', '\u2001', '\u2002', '\u2003',
              '\u2004', '\u2005', '\u2006', '\u2007', '\u2008', '\u2009', '\u200a',
              '\u2028', '\u2029', '\u202f', '\u205f', '\u3000'}
label_pattern = re.compile(r'\[(.+?)(?<!\])\]', re.DOTALL)
punctuation = {'!', '"', '#', '$', '%', '&', '\'', '(', ')', '*', '+', ',',
               '-', '.', '/', ':', ';', '<', '=', '>', '?', '@', '[', '\\',
               ']', '^', '_', '`', '{', '|', '}', '~'}


def find_core_tokens(string):
    delimiters = []
    matches = []
    escaped = False
    in_delimiter_run = None
    in_image = False
    start = 0
    i = 0
    while i < len(string):
        c = string[i]
        if c == '\\':
            escaped = True
            i += 1
            continue
        if in_delimiter_run is not None and c != in_delimiter_run or escaped:
            delimiters.append(Delimiter(start, i if not escaped else i-1, string))
            in_delimiter_run = None
        if in_delimiter_run is None and (c == '*' or c == '_') and not escaped:
            in_delimiter_run = c
            start = i
        if not escaped:
            if c == '[':
                if not in_image:
                    delimiters.append(Delimiter(i, i+1, string))
                else:
                    delimiters.append(Delimiter(i-1, i+1, string))
                    in_image = False
            elif c == '!':
                in_image = True
            elif c == ']':
                i = find_link_image(string, i, delimiters, matches)
            elif in_image:
                in_image = False
        else:
            escaped = False
        i += 1
    if in_delimiter_run:
        delimiters.append(Delimiter(start, i, string))
    process_emphasis(string, 0, delimiters, matches)
    return matches


def find_link_image(string, offset, delimiters, matches):
    i = len(delimiters) - 1
    for delimiter in delimiters[::-1]:
        # found a link/image delimiter
        if delimiter.type in ('[', '!['):
            # not active, remove delimiter
            if not delimiter.active:
                delimiters.remove(delimiter)
                return offset
            match = match_link_image(string, offset, delimiter)
            # found match
            if match:
                # parse for emphasis
                process_emphasis(string, i, delimiters, matches)
                # append current match
                matches.append(match)
                # if match is a link, set all previous links to be inactive
                if delimiter.type == '[':
                    deactivate_delimiters(delimiters, i, '[')
                # shift index till end of match
                return match.end() - 1
            # no match, remove delimiter
            delimiters.remove(delimiter)
            return offset
        i -= 1
    # no link/image delimiter
    return offset


def process_emphasis(string, stack_bottom, delimiters, matches):
    star_bottom = stack_bottom
    underscore_bottom = stack_bottom
    curr_pos = next_closer(stack_bottom, delimiters)
    while curr_pos is not None:
        closer = delimiters[curr_pos]
        bottom = star_bottom if closer.type[0] == '*' else underscore_bottom
        open_pos = matching_opener(curr_pos, delimiters, bottom)
        if open_pos is not None:
            opener = delimiters[open_pos]
            n = 2 if closer.number >= 2 and opener.number >= 2 else 1
            start = opener.end - n
            end = closer.start + n
            match = MatchObj(start, end, (start+n, end-n, string[start+n:end-n]))
            match.type = 'Strong' if n == 2 else 'Emphasis'
            matches.append(match)
            if not opener.remove(n, left=False):
                delimiters.remove(opener)
                curr_pos -= 2
            else:
                curr_pos -= 1
            if not closer.remove(n, left=True):
                delimiters.remove(closer)
                curr_pos -= 2
            else:
                curr_pos -= 1
            if curr_pos < 0:
                curr_pos = 0
        else:
            if closer.type[0] == '*':
                star_bottom = curr_pos - 1
            else:
                underscore_bottom = curr_pos - 1
            if not closer.open:
                delimiters.remove(closer)
                curr_pos -= 2
                if curr_pos < 0: curr_pos = 0
        curr_pos = next_closer(curr_pos, delimiters)
    del delimiters[stack_bottom:]


def match_link_image(string, offset, delimiter):
    image = delimiter.type == '!['
    start = delimiter.start
    text_start = start + delimiter.number
    text_end = offset
    text = string[text_start:text_end]
    # inline link
    if follows(string, offset, '('):
        # link destination
        match_info = match_link_dest(string, offset+1)
        if match_info is None:
            return None
        dest_start, dest_end, dest = match_info
        # link title
{'fraction_non_alphanumeric': 0.12964419107528996, 'fraction_numerical': 0.0208374287399253, 'mean_word_length': 1.681159420289855, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1969856', 'n_tokens_mistral': 3262, 'n_tokens_neox': 3044, 'n_words': 773}