<commit_before>require 'strscan'
  
class String
  def is_empty_token?
    self.size == 0 or self == "\n" or self == "\r\n"
  end
end


class MediaWikiLexer
  
  INLINE_ELEMENTS = [:LINK, :INTLINK, :BOLD, :ITALIC]
  BLOCK_ELEMENTS = [:PARA, :PRE, :UL, :OL, :DL, :LI, :SECTION, :TABLE, :ROW, :CELL, :HEAD]
  PARA_BREAK_ELEMENTS = [:UL, :OL, :DL, :PRE, :SECTION, :TABLE, :HLINE, :KEYWORD]
  
  NAME_CHAR_TABLE = (0 .. 255).collect{|n| n.chr =~ /[a-zA-Z0-9_\-]/ ? true : false}
  TOKEN_CHAR_TABLE = (0 .. 255).collect{|n| n.chr =~ /[a-zA-Z0-9_\-.;?&~=#%\/]/ ? true : false}

  
  def initialize
    # Current position in token list
    @position = 0
    
    # Lexer table of methods that handle only formatting, e.g. bold or italicized
    # text; or spans of XHTML, or wiki-escape, markup
    @formatting_lexer_table = {}
    @formatting_lexer_table["'"] = method(:match_quote)
    @formatting_lexer_table["<"] = method(:match_left_angle)
    @formatting_lexer_table["&"] = method(:match_ampersand)
    @formatting_lexer_table["{"] = method(:match_left_curly)
    
    # Lexer table of methods that handle everything that may occur in-line in
    # addition to formatting, i.e. links and signatures
    @inline_lexer_table = @formatting_lexer_table.dup
    @inline_lexer_table["["] = method(:match_left_square)
    @inline_lexer_table["~"] = method(:match_tilde)
    @inline_lexer_table["h"] = method(:match_h_char)
        
    # Default lexer table, which includes all in-line formatting and links, plus
    # methods that handle constructs that begin on a newline
    @default_lexer_table = @inline_lexer_table.dup
    @default_lexer_table[" "] = method(:match_space)
    @default_lexer_table["="] = method(:match_equal)
    @default_lexer_table["*"] = method(:match_star)
    @default_lexer_table["#"] = method(:match_hash)
    @default_lexer_table[":"] = method(:match_colon)
    @default_lexer_table[";"] = method(:match_semicolon)
    @default_lexer_table["-"] = method(:match_dash)
    @default_lexer_table["_"] = method(:match_underscore)
    @default_lexer_table["\n"] = method(:match_newline)
    @default_lexer_table["\r"] = method(:match_newline)
    
    # Lexer table used inside spans of markup, wherein spans of newlines are not
    # automatically treated as paragraphs.
    @markup_lexer_table = @default_lexer_table.dup
    @markup_lexer_table["\n"] = nil
    @markup_lexer_table["\r"] = nil
    
    # Lexer table used inside of headings
    @heading_lexer_table = @inline_lexer_table.dup
    @heading_lexer_table["="] = method(:match_equal_in_heading)
    @heading_lexer_table["\n"] = method(:match_newline_in_heading)
    
    # Lexer table used inside the left half of an external link
    @link_lexer_table = {}
    @link_lexer_table["]"] = method(:match_right_square_in_link)
    @link_lexer_table["\n"] = method(:match_newline_in_link)
    @link_lexer_table["\r"] = method(:match_newline_in_link)
    @link_lexer_table[" "] = method(:match_space_in_link)
    
    # Lexer table used inside the right half of an external link, or the right
    # half of an internal link
    @link_opt_lexer_table = @inline_lexer_table.dup
    @link_opt_lexer_table["]"] = method(:match_right_square_in_link)
    @link_opt_lexer_table["\n"] = method(:match_newline_in_link)
    @link_opt_lexer_table["\r"] = method(:match_newline_in_link)
    
    # Lexer table used inside the left half of an internal link or internal
    # resource link
    @intlink_lexer_table = {}
    @intlink_lexer_table["]"] = method(:match_right_square_in_intlink)
    @intlink_lexer_table["\r"] = method(:match_newline_in_intlink)
    @intlink_lexer_table["\n"] = method(:match_newline_in_intlink)
    @intlink_lexer_table[":"] = method(:match_colon_in_intlink)
    @intlink_lexer_table["|"] = method(:match_pipe_in_intlink)
    @intlink_lexer_table["C"] = method(:match_c_char_in_intlink)
    
    # Lexer table used inside the category name of the left half of an 
    # internal link
    @intlink_cat_lexer_table = {}
    @intlink_cat_lexer_table["]"] = method(:match_right_square_in_intlink)
    @intlink_cat_lexer_table["\r"] = method(:match_newline_in_intlink)
    @intlink_cat_lexer_table["\n"] = method(:match_newline_in_intlink)
    @intlink_cat_lexer_table["|"] = method(:match_pipe_in_intlink)    
    
    # Lexer table used inside the right half of an internal link
    @intlink_opt_lexer_table = @formatting_lexer_table.dup
    @intlink_opt_lexer_table["]"] = method(:match_right_square_in_intlink)
    @intlink_opt_lexer_table["\n"] = method(:match_newline_in_intlink)
    @intlink_opt_lexer_table["\r"] = method(:match_newline_in_intlink)
    
    # Lexer table used inside the right half of an internal resource link
    @resourcelink_opt_lexer_table = @inline_lexer_table.dup
    @resourcelink_opt_lexer_table["]"] = method(:match_right_square_in_intlink)
    @resourcelink_opt_lexer_table["\n"] = method(:match_newline_in_intlink)
    @resourcelink_opt_lexer_table["\r"] = method(:match_newline_in_intlink)
    @resourcelink_opt_lexer_table["|"] = method(:match_pipe_in_intlink)
    
    # Lexer table used to parse tables
    @table_lexer_table = @inline_lexer_table.dup
    @table_lexer_table["*"] = method(:match_star)
    @table_lexer_table["#"] = method(:match_hash)
    @table_lexer_table["|"] = method(:match_pipe_in_table)
    @table_lexer_table["!"] = method(:match_bang_in_table)
    @table_lexer_table["{"] = method(:match_left_curly)
    
    # Lexer table used to parse ordered and unordered list items (which may nest)
    @items_lexer_table = @inline_lexer_table.dup
    @items_lexer_table["\n"] = method(:match_newline_in_items)
    
    # Lexer table used to parse entries in a definition list (which may not nest)
    @entries_lexer_table = @inline_lexer_table.dup
    @entries_lexer_table["\n"] = method(:match_newline_in_entries)
    @entries_lexer_table[":"] = method(:match_colon_in_entries)
    
    # Lexer table used inside spans of indented text
    @indent_lexer_table = @inline_lexer_table.dup
    @indent_lexer_table["\n"] = method(:match_newline_in_indent)
    
    # Lexer table used inside spans of pre-formatted text
    @pre_lexer_table = @inline_lexer_table.dup
    @pre_lexer_table["<"] = method(:match_left_angle_in_pre)
        
    # Lexer table used when inside spans of wiki-escaped text
    @nowiki_lexer_table = {}
    @nowiki_lexer_table["<"] = method(:match_left_angle_in_nowiki)
        
    # Lexer table used when inside spans of math
    @math_lexer_table = {}
    @math_lexer_table["<"] = method(:match_left_angle_in_math)
        
    # Lexer table used when inside a wiki variable reference
    @variable_lexer_table = {}
    @variable_lexer_table["{"] = method(:match_left_curly_in_variable)
    @variable_lexer_table["}"] = method(:match_right_curly_in_variable)<commit_msg>PRE lexer table should be based on empty table.<commit_after>require 'strscan'
  
class String
  def is_empty_token?
    self.size == 0 or self == "\n" or self == "\r\n"
  end
end


class MediaWikiLexer
  
  INLINE_ELEMENTS = [:LINK, :INTLINK, :BOLD, :ITALIC]
  BLOCK_ELEMENTS = [:PARA, :PRE, :UL, :OL, :DL, :LI, :SECTION, :TABLE, :ROW, :CELL, :HEAD]
  PARA_BREAK_ELEMENTS = [:UL, :OL, :DL, :PRE, :SECTION, :TABLE, :HLINE, :KEYWORD]
  
  NAME_CHAR_TABLE = (0 .. 255).collect{|n| n.chr =~ /[a-zA-Z0-9_\-]/ ? true : false}
  TOKEN_CHAR_TABLE = (0 .. 255).collect{|n| n.chr =~ /[a-zA-Z0-9_\-.;?&~=#%\/]/ ? true : false}

  
  def initialize
    # Current position in token list
    @position = 0
    
    # Lexer table of methods that handle only formatting, e.g. bold or italicized
    # text; or spans of XHTML, or wiki-escape, markup
    @formatting_lexer_table = {}
    @formatting_lexer_table["'"] = method(:match_quote)
    @formatting_lexer_table["<"] = method(:match_left_angle)
    @formatting_lexer_table["&"] = method(:match_ampersand)
    @formatting_lexer_table["{"] = method(:match_left_curly)
    
    # Lexer table of methods that handle everything that may occur in-line in
    # addition to formatting, i.e. links and signatures
    @inline_lexer_table = @formatting_lexer_table.dup
    @inline_lexer_table["["] = method(:match_left_square)
    @inline_lexer_table["~"] = method(:match_tilde)
    @inline_lexer_table["h"] = method(:match_h_char)
        
    # Default lexer table, which includes all in-line formatting and links, plus
    # methods that handle constructs that begin on a newline
    @default_lexer_table = @inline_lexer_table.dup
    @default_lexer_table[" "] = method(:match_space)
    @default_lexer_table["="] = method(:match_equal)
    @default_lexer_table["*"] = method(:match_star)
    @default_lexer_table["#"] = method(:match_hash)
    @default_lexer_table[":"] = method(:match_colon)
    @default_lexer_table[";"] = method(:match_semicolon)
    @default_lexer_table["-"] = method(:match_dash)
    @default_lexer_table["_"] = method(:match_underscore)
    @default_lexer_table["\n"] = method(:match_newline)
    @default_lexer_table["\r"] = method(:match_newline)
    
    # Lexer table used inside spans of markup, wherein spans of newlines are not
    # automatically treated as paragraphs.
    @markup_lexer_table = @default_lexer_table.dup
    @markup_lexer_table["\n"] = nil
    @markup_lexer_table["\r"] = nil
    
    # Lexer table used inside of headings
    @heading_lexer_table = @inline_lexer_table.dup
    @heading_lexer_table["="] = method(:match_equal_in_heading)
    @heading_lexer_table["\n"] = method(:match_newline_in_heading)
    
    # Lexer table used inside the left half of an external link
    @link_lexer_table = {}
    @link_lexer_table["]"] = method(:match_right_square_in_link)
    @link_lexer_table["\n"] = method(:match_newline_in_link)
    @link_lexer_table["\r"] = method(:match_newline_in_link)
    @link_lexer_table[" "] = method(:match_space_in_link)
    
    # Lexer table used inside the right half of an external link, or the right
    # half of an internal link
    @link_opt_lexer_table = @inline_lexer_table.dup
    @link_opt_lexer_table["]"] = method(:match_right_square_in_link)
    @link_opt_lexer_table["\n"] = method(:match_newline_in_link)
    @link_opt_lexer_table["\r"] = method(:match_newline_in_link)
    
    # Lexer table used inside the left half of an internal link or internal
    # resource link
    @intlink_lexer_table = {}
    @intlink_lexer_table["]"] = method(:match_right_square_in_intlink)
    @intlink_lexer_table["\r"] = method(:match_newline_in_intlink)
    @intlink_lexer_table["\n"] = method(:match_newline_in_intlink)
    @intlink_lexer_table[":"] = method(:match_colon_in_intlink)
    @intlink_lexer_table["|"] = method(:match_pipe_in_intlink)
    @intlink_lexer_table["C"] = method(:match_c_char_in_intlink)
    
    # Lexer table used inside the category name of the left half of an 
    # internal link
    @intlink_cat_lexer_table = {}
    @intlink_cat_lexer_table["]"] = method(:match_right_square_in_intlink)
    @intlink_cat_lexer_table["\r"] = method(:match_newline_in_intlink)
    @intlink_cat_lexer_table["\n"] = method(:match_newline_in_intlink)
    @intlink_cat_lexer_table["|"] = method(:match_pipe_in_intlink)    
    
    # Lexer table used inside the right half of an internal link
    @intlink_opt_lexer_table = @formatting_lexer_table.dup
    @intlink_opt_lexer_table["]"] = method(:match_right_square_in_intlink)
    @intlink_opt_lexer_table["\n"] = method(:match_newline_in_intlink)
    @intlink_opt_lexer_table["\r"] = method(:match_newline_in_intlink)
    
    # Lexer table used inside the right half of an internal resource link
    @resourcelink_opt_lexer_table = @inline_lexer_table.dup
    @resourcelink_opt_lexer_table["]"] = method(:match_right_square_in_intlink)
    @resourcelink_opt_lexer_table["\n"] = method(:match_newline_in_intlink)
    @resourcelink_opt_lexer_table["\r"] = method(:match_newline_in_intlink)
    @resourcelink_opt_lexer_table["|"] = method(:match_pipe_in_intlink)
    
    # Lexer table used to parse tables
    @table_lexer_table = @inline_lexer_table.dup
    @table_lexer_table["*"] = method(:match_star)
    @table_lexer_table["#"] = method(:match_hash)
    @table_lexer_table["|"] = method(:match_pipe_in_table)
    @table_lexer_table["!"] = method(:match_bang_in_table)
    @table_lexer_table["{"] = method(:match_left_curly)
    
    # Lexer table used to parse ordered and unordered list items (which may nest)
    @items_lexer_table = @inline_lexer_table.dup
    @items_lexer_table["\n"] = method(:match_newline_in_items)
    
    # Lexer table used to parse entries in a definition list (which may not nest)
    @entries_lexer_table = @inline_lexer_table.dup
    @entries_lexer_table["\n"] = method(:match_newline_in_entries)
    @entries_lexer_table[":"] = method(:match_colon_in_entries)
    
    # Lexer table used inside spans of indented text
    @indent_lexer_table = @inline_lexer_table.dup
    @indent_lexer_table["\n"] = method(:match_newline_in_indent)
    
    # Lexer table used inside spans of pre-formatted text
    @pre_lexer_table = {}
    @pre_lexer_table["<"] = method(:match_left_angle_in_pre)
        
    # Lexer table used when inside spans of wiki-escaped text
    @nowiki_lexer_table = {}
    @nowiki_lexer_table["<"] = method(:match_left_angle_in_nowiki)
        
    # Lexer table used when inside spans of math
    @math_lexer_table = {}
    @math_lexer_table["<"] = method(:match_left_angle_in_math)
        
    # Lexer table used when inside a wiki variable reference
    @variable_lexer_table = {}
    @variable_lexer_table["{"] = method(:match_left_curly_in_variable)
    @variable_lexer_table["}"] = method(:match_right_curly_in_variable)
{'fraction_non_alphanumeric': 0.12246837756818016, 'fraction_numerical': 0.0020472325802442055, 'mean_word_length': 4.526464646464646, 'pattern_counts': {'":': 6, '<': 11, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1985880', 'n_tokens_mistral': 4973, 'n_tokens_neox': 4876, 'n_words': 918}