<commit_before>import re
import lib.base_token as base_token
from lib.block_token import *
import lib.leaf_token as leaf_token
from lib.reader import *

def tokenize(lines):
    tokens = []
    index = 0

    def shift_token(token_type, reader_func):
        end_index = reader_func(index, lines)
        tokens.append(token_type(lines[index:end_index]))
        return end_index

    def shift_line_token(token_type=None):
        if token_type:
            tokens.append(token_type(lines[index]))
        return index + 1

    while index < len(lines):
        if lines[index].startswith('#'):        # heading
            index = shift_line_token(Heading)
        elif lines[index].startswith('> '):     # quote
            index = shift_token(Quote, read_quote)
        elif lines[index].startswith('```'):    # block code
            index = shift_token(BlockCode, read_block_code)
        elif lines[index] == '---\n':           # separator
            index = shift_line_token(Separator)
        elif lines[index].startswith('- '):     # list
            index = shift_token(build_list, read_list)
        elif lines[index] == '\n':              # skip empty line
            index = shift_line_token()
        else:                                   # paragraph
            index = shift_token(Paragraph, read_paragraph)
    return tokens

def tokenize_inner(content):
    tokens = []

    def append_token(token_type, content, index):
        tokens.append(token_type(content[:index]))
        tokenize_inner_helper(content[index:])

    def tokenize_inner_helper(content):
        if content == '':                                 # base case
            return
        if re.match(r"\*\*(.+?)\*\*", content):           # bold
            i = content.index('**', 1) + 2
            append_token(leaf_token.Bold, content, i)
        elif re.match(r"\*(.+?)\*", content):             # italics
            i = content.index('*', 1) + 1
            append_token(leaf_token.Italic, content, i)
        elif re.match(r"`(.+?)`", content):               # inline code
            i = content.index('`', 1) + 1
            append_token(leaf_token.InlineCode, content, i)
        elif re.match(r"\[(.+?)\]\((.+?)\)", content):    # link
            i = content.index(')') + 1
            append_token(leaf_token.Link, content, i)
        else:                                             # raw text
            try:                      # next token
                p = r"(`(.+?)`)|(\*\*(.+?)\*\*)|(\*(.+?)\*)|\[(.+?)\]\((.+?)\)"
                i = re.search(p, content).start()
            except AttributeError:    # no more tokens
                i = len(content)
            append_token(leaf_token.RawText, content, i)
    tokenize_inner_helper(content)
    return tokens

def build_list(lines, level=0):
    l = List()
    index = 0
    while index < len(lines):
        if lines[index][level*4:].startswith('- '):
            l.add(ListItem(lines[index]))
        else:
            curr_level = level + 1
            end_index = read_list(index, lines, curr_level)
            l.add(build_list(lines[index:end_index], curr_level))
            index = end_index - 1
        index += 1
    return l
<commit_msg>☕️ refactoring: simplified dependency in parser module<commit_after>import re
from lib.block_token import *
from lib.leaf_token import *
from lib.reader import *

def tokenize(lines):
    tokens = []
    index = 0

    def shift_token(token_type, reader_func):
        end_index = reader_func(index, lines)
        tokens.append(token_type(lines[index:end_index]))
        return end_index

    def shift_line_token(token_type=None):
        if token_type:
            tokens.append(token_type(lines[index]))
        return index + 1

    while index < len(lines):
        if lines[index].startswith('#'):        # heading
            index = shift_line_token(Heading)
        elif lines[index].startswith('> '):     # quote
            index = shift_token(Quote, read_quote)
        elif lines[index].startswith('```'):    # block code
            index = shift_token(BlockCode, read_block_code)
        elif lines[index] == '---\n':           # separator
            index = shift_line_token(Separator)
        elif lines[index].startswith('- '):     # list
            index = shift_token(build_list, read_list)
        elif lines[index] == '\n':              # skip empty line
            index = shift_line_token()
        else:                                   # paragraph
            index = shift_token(Paragraph, read_paragraph)
    return tokens

def tokenize_inner(content):
    tokens = []

    def append_token(token_type, content, index):
        tokens.append(token_type(content[:index]))
        tokenize_inner_helper(content[index:])

    def tokenize_inner_helper(content):
        if content == '':                                 # base case
            return
        if re.match(r"\*\*(.+?)\*\*", content):           # bold
            i = content.index('**', 1) + 2
            append_token(Bold, content, i)
        elif re.match(r"\*(.+?)\*", content):             # italics
            i = content.index('*', 1) + 1
            append_token(Italic, content, i)
        elif re.match(r"`(.+?)`", content):               # inline code
            i = content.index('`', 1) + 1
            append_token(InlineCode, content, i)
        elif re.match(r"\[(.+?)\]\((.+?)\)", content):    # link
            i = content.index(')') + 1
            append_token(Link, content, i)
        else:                                             # raw text
            try:                      # next token
                p = r"(`(.+?)`)|(\*\*(.+?)\*\*)|(\*(.+?)\*)|\[(.+?)\]\((.+?)\)"
                i = re.search(p, content).start()
            except AttributeError:    # no more tokens
                i = len(content)
            append_token(RawText, content, i)
    tokenize_inner_helper(content)
    return tokens

def build_list(lines, level=0):
    l = List()
    index = 0
    while index < len(lines):
        if lines[index][level*4:].startswith('- '):
            l.add(ListItem(lines[index]))
        else:
            curr_level = level + 1
            end_index = read_list(index, lines, curr_level)
            l.add(build_list(lines[index:end_index], curr_level))
            index = end_index - 1
        index += 1
    return l

{'fraction_non_alphanumeric': 0.13884971007679048, 'fraction_numerical': 0.004701457451810061, 'mean_word_length': 1.8028107158541942, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2031578', 'n_tokens_mistral': 1905, 'n_tokens_neox': 1840, 'n_words': 401}