<commit_before>import re

from idl.lexer.EnumBeginToken import EnumBeginToken
from idl.lexer.EnumFieldToken import EnumFieldToken
from idl.lexer.InterfaceBeginToken import InterfaceBeginToken
from idl.lexer.MethodToken import MethodToken
from idl.lexer.StructBeginToken import StructBeginToken
from idl.lexer.StructFieldToken import StructFieldToken
from idl.lexer.Token import Token
from idl.lexer.TokenType import TokenType
from idl.lexer.Utils import *

from idl.lexer.AnnotationToken import AnnotationToken


class Lexer:
    def __init__(self, source):
        pass

    @staticmethod
    def tokenize(source):
        # Preprocess source
        source = Lexer.__preprocess(source)
        
        # Tokenize it
        return Lexer.__tokenize(source)
    
    @staticmethod
    def __tokenize(string):
        tokens = []
        
        # Empty string?
        if not string:
            return tokens
        
        lines = string.split('\n')
        
        for line in lines:
            TokenType = Lexer.__getTokenType(line)
            
            if TokenType == None:
                raise RuntimeError('Unrecognized token \"%s\"' % line)
            else:
                tokens.append( TokenType.instantiate(line) )
                
        return tokens
        
    @staticmethod
    def __preprocess(string):
        # Replace CR LF with LF
        string = string.replace('\r\n', '\n')
        
        res = ''
        
        # Remove comment blocks
        blockComment = re.compile(  r'/\*' + r'.*?' + r'\*/', re.DOTALL )
        
        match = True
        
        while match:
            match = blockComment.search(string)
            
            if match:
                span = match.span()
                
                string = string[:span[0]] + string[span[1]:]

        for line in string.split('\n'):
            # Remove comments<commit_msg>line escaping added<commit_after>import re

from idl.lexer.EnumBeginToken import EnumBeginToken
from idl.lexer.EnumFieldToken import EnumFieldToken
from idl.lexer.InterfaceBeginToken import InterfaceBeginToken
from idl.lexer.MethodToken import MethodToken
from idl.lexer.StructBeginToken import StructBeginToken
from idl.lexer.StructFieldToken import StructFieldToken
from idl.lexer.Token import Token
from idl.lexer.TokenType import TokenType
from idl.lexer.Utils import *

from idl.lexer.AnnotationToken import AnnotationToken


class Lexer:
    def __init__(self, source):
        pass

    @staticmethod
    def tokenize(source):
        # Preprocess source
        source = Lexer.__preprocess(source)
        
        # Tokenize it
        return Lexer.__tokenize(source)
    
    @staticmethod
    def __tokenize(string):
        tokens = []
        
        # Empty string?
        if not string:
            return tokens
        
        lines = string.split('\n')
        
        for line in lines:
            TokenType = Lexer.__getTokenType(line)
            
            if TokenType == None:
                raise RuntimeError('Unrecognized token \"%s\"' % line)
            else:
                tokens.append( TokenType.instantiate(line) )
                
        return tokens
        
    @staticmethod
    def __preprocess(string):
        # Replace CR LF with LF
        string = string.replace('\r\n', '\n')
        
        res = ''
        
        # Remove escaped new lines
        string = string.replace('\\\n', '')
        
        # Remove comment blocks
        blockComment = re.compile(  r'/\*' + r'.*?' + r'\*/', re.DOTALL )
        
        match = True
        
        while match:
            match = blockComment.search(string)
            
            if match:
                span = match.span()
                
                string = string[:span[0]] + string[span[1]:]

        for line in string.split('\n'):
            # Remove comments
{'fraction_non_alphanumeric': 0.08526532715095311, 'fraction_numerical': 0.0010303967027305513, 'mean_word_length': 1.7676407697790448, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6372908', 'n_tokens_mistral': 1062, 'n_tokens_neox': 1039, 'n_words': 267}