<commit_before>lexer grammar MrvLexer;

options {
   superClass = 'org.batfish.grammar.BatfishLexer';
}

@header {
package org.batfish.grammar.mrv;
}

tokens {
   QUOTED_TEXT
}

// Mrv Keywords

ASYNC
:
   'Async'
;

AUTOHANG
:
   'AutoHang'
;

BOOL
:
   'BOOL'
;

IFNAME
:
   'ifName'
;

INTERFACE
:
   'Interface'
;

SYSTEM
:
   'System'
;

SYSTEMNAME
:
   'SystemName'
;

TYPE
:
   'TYPE'
;

// Other tokens

DOUBLE_QUOTE
:
   '"' -> pushMode ( M_QuotedString )
;

PERIOD
:
   '.'
;

mode M_QuotedString;

M_QuotedString_QUOTED_TEXT
:
   ~'"'+ -> type ( QUOTED_TEXT )
;

M_QuotedString_DOUBLE_QUOTE
:
   '"' -> type ( DOUBLE_QUOTE ) , popMode
;
<commit_msg>added comment lexing for mrv<commit_after>lexer grammar MrvLexer;

options {
   superClass = 'org.batfish.grammar.BatfishLexer';
}

@header {
package org.batfish.grammar.mrv;
}

tokens {
   QUOTED_TEXT
}

// Mrv Keywords

ASYNC
:
   'Async'
;

AUTOHANG
:
   'AutoHang'
;

BOOL
:
   'BOOL'
;

IFNAME
:
   'ifName'
;

INTERFACE
:
   'Interface'
;

SYSTEM
:
   'System'
;

SYSTEMNAME
:
   'SystemName'
;

TYPE
:
   'TYPE'
;

// Other tokens

DOUBLE_QUOTE
:
   '"' -> pushMode ( M_QuotedString )
;

LINE_COMMENT
:
   '!' -> pushMode(M_LineComment), channel(HIDDEN)
;

PERIOD
:
   '.'
;

fragment
F_Newline
:
   [\r\n]+
;

fragment
F_NonNewline
:
   ~[\r\n]
;

mode M_LineComment;

M_LineComment_FILLER
:
   F_NonNewline+ F_Newline -> channel(HIDDEN), popMode
;

mode M_QuotedString;

M_QuotedString_QUOTED_TEXT
:
   ~'"'+ -> type ( QUOTED_TEXT )
;

M_QuotedString_DOUBLE_QUOTE
:
   '"' -> type ( DOUBLE_QUOTE ) , popMode
;

{'fraction_non_alphanumeric': 0.14122862571247624, 'fraction_numerical': 0.0, 'mean_word_length': 3.103896103896104, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4438100', 'n_tokens_mistral': 746, 'n_tokens_neox': 726, 'n_words': 108}