<commit_before>/* Todo:
 *  - Comments justifying and explaining every rule.
 */

lexer grammar CreoleTokens;

@members {
  public boolean inHeader = false;
  public boolean start = false;
  public boolean bold = false;
  public boolean italic = false;
  public boolean strike = false;
  public int olistLevel = 0;
  public int ulistLevel = 0;

  public void doHdr() {
    String prefix = getText().trim();
    boolean seekback = false;

    if(!prefix.substring(prefix.length() - 1).equals("=")) {
      prefix = prefix.substring(0, prefix.length() - 1);
      seekback = true;
    }

    if(prefix.length() <= 6) {
      if(seekback) {
        _input.seek(_input.index() - 1);
      }

      setText(prefix);
      inHeader = true;
    } else {
      setType(Any);
    }
  }

  public void setStart() {
    String next1 = _input.getText(new Interval(_input.index(), _input.index()));
    String next2 = _input.getText(new Interval(_input.index() + 1, _input.index() + 1));
    start = (next1.equals("*") && !next2.equals("*")) || (next1.equals("#") && !next2.equals("#"));
  }

  public void doUlist(int level) {
    ulistLevel = level;

    _input.seek(_input.index() - 1);
    setStart();
  }

  public void doOlist(int level) {
    olistLevel = level;

    _input.seek(_input.index() - 1);
    setStart();
  }

  public void resetFormatting() {
    bold   = false;
    italic = false;
    strike = false;
  }
}

/* ***** Headings ***** */

HSt  : LINE '='+ ~'=' {doHdr();} ;
HEnd : ' '* '='+ {inHeader}? {inHeader = false; resetFormatting();} ;

/* ***** Lists ***** */

U1 : START '*' ~'*'                        {doUlist(1); resetFormatting();} ;
U2 : START '**' ~'*'    {ulistLevel >= 1}? {doUlist(2); resetFormatting();} ;
U3 : START '***' ~'*'   {ulistLevel >= 2}? {doUlist(3); resetFormatting();} ;
U4 : START '****' ~'*'  {ulistLevel >= 3}? {doUlist(4); resetFormatting();} ;
U5 : START '*****' ~'*' {ulistLevel >= 4}? {doUlist(5); resetFormatting();} ;

O1 : START '#' ~'#'                        {doOlist(1); resetFormatting();} ;
O2 : START '##' ~'#'    {olistLevel >= 1}? {doOlist(2); resetFormatting();} ;
O3 : START '###' ~'#'   {olistLevel >= 2}? {doOlist(3); resetFormatting();} ;
O4 : START '####' ~'#'  {olistLevel >= 3}? {doOlist(4); resetFormatting();} ;
O5 : START '#####' ~'#' {olistLevel >= 4}? {doOlist(5); resetFormatting();} ;

/* ***** Horizontal Rules ***** */

Rule : LINE '---' '-'+? {resetFormatting();} ;

/* ***** Tables ***** */

CellSep : '|'  {resetFormatting();} ;
TdStart : '|'  {resetFormatting();} ;
ThStart : '|=' {resetFormatting();} ;

/* ***** Inline Formatting ***** */

BSt : '**' {!bold}?   {bold=true;} ;
ISt : '//' {!italic}? {italic=true;} ;
SSt : '--' {!strike}? {strike=true;} ;

BEnd : '**' {bold}?   {bold=false;} ;
IEnd : '//' {italic}? {italic=false;} ;
SEnd : '--' {strike}? {strike=false;} ;

NoWiki : '{{{' -> mode(PREFORMATTED_INLINE) ;

/* ***** Links ***** */

LiSt  : '[[' -> mode(LINK) ;
ImSt  : '{{' -> mode(LINK) ;

/* ***** Breaks ***** */

InlineBrk : '\\\\' ;

ParBreak  : LineBreak LineBreak+ {resetFormatting();} ;

LineBreak : '\r'? '\n'+? ;

/* ***** Links ***** */

RawUrl    : ('http' | 'ftp') '://' ~(' '|'\t'|'\r'|'\n')+;

WikiWords : (ALNUM+ ':')? (UPPER ((ALNUM|'.')* ALNUM)*) (UPPER ((ALNUM|'.')* ALNUM)*)+;

/* ***** Miscellaneous ***** */

Any : . ;
WS  : (' '|'\t'|'\r'|'\n')+ -> skip ;

fragment START : {start}? | LINE ;
fragment LINE  : ({getCharPositionInLine()==0}? WS? | LineBreak WS?);
fragment ALNUM : (ALPHA | DIGIT) ;
fragment ALPHA : (UPPER | LOWER) ;
fragment UPPER : ('A'..'Z') ;
fragment LOWER : ('a'..'z') ;
fragment DIGIT : ('0'..'9') ;

/* ***** Contextual stuff ***** */


mode LINK;

LiEnd : ']]' -> mode(DEFAULT_MODE) ;
ImEnd : '}}' -> mode(DEFAULT_MODE) ;

Sep : '|' ;

InLink : ~(']'|'}'|'|')+ ;

mode PREFORMATTED_INLINE;

AnyInlineText : ~('\r'|'\n') -> more;

AnyBlockText : AnyInlineText* LineBreak -> mode(PREFORMATTED_BLOCK), more ;

EndNoWikiInline : '}}}' (~'}' {_input.seek(_input.index()-1);} | EOF) -> mode(DEFAULT_MODE) ;

mode PREFORMATTED_BLOCK;

AnyText   : . -> more ;

EndNoWikiBlock : LINE '}}}' -> mode(DEFAULT_MODE) ;
<commit_msg>Correctly handle a URL followed by an italics<commit_after>/* Todo:
 *  - Comments justifying and explaining every rule.
 */

lexer grammar CreoleTokens;

@members {
  public boolean inHeader = false;
  public boolean start = false;
  public boolean bold = false;
  public boolean italic = false;
  public boolean strike = false;
  public int olistLevel = 0;
  public int ulistLevel = 0;

  public void doHdr() {
    String prefix = getText().trim();
    boolean seekback = false;

    if(!prefix.substring(prefix.length() - 1).equals("=")) {
      prefix = prefix.substring(0, prefix.length() - 1);
      seekback = true;
    }

    if(prefix.length() <= 6) {
      if(seekback) {
        _input.seek(_input.index() - 1);
      }

      setText(prefix);
      inHeader = true;
    } else {
      setType(Any);
    }
  }

  public void setStart() {
    String next1 = _input.getText(new Interval(_input.index(), _input.index()));
    String next2 = _input.getText(new Interval(_input.index() + 1, _input.index() + 1));
    start = (next1.equals("*") && !next2.equals("*")) || (next1.equals("#") && !next2.equals("#"));
  }

  public void doUlist(int level) {
    ulistLevel = level;

    _input.seek(_input.index() - 1);
    setStart();
  }

  public void doOlist(int level) {
    olistLevel = level;

    _input.seek(_input.index() - 1);
    setStart();
  }

  public void resetFormatting() {
    bold   = false;
    italic = false;
    strike = false;
  }

  public void doUrl() {
    String url = getText();
    String last = url.substring(url.length()-1);
    String next = _input.getText(new Interval(_input.index(), _input.index()));

    if((last + next).equals("//")) {
      _input.seek(_input.index() - 1);
      setText(url.substring(0, url.length() - 1));
    }
  }
}

/* ***** Headings ***** */

HSt  : LINE '='+ ~'=' {doHdr();} ;
HEnd : ' '* '='+ {inHeader}? {inHeader = false; resetFormatting();} ;

/* ***** Lists ***** */

U1 : START '*' ~'*'                        {doUlist(1); resetFormatting();} ;
U2 : START '**' ~'*'    {ulistLevel >= 1}? {doUlist(2); resetFormatting();} ;
U3 : START '***' ~'*'   {ulistLevel >= 2}? {doUlist(3); resetFormatting();} ;
U4 : START '****' ~'*'  {ulistLevel >= 3}? {doUlist(4); resetFormatting();} ;
U5 : START '*****' ~'*' {ulistLevel >= 4}? {doUlist(5); resetFormatting();} ;

O1 : START '#' ~'#'                        {doOlist(1); resetFormatting();} ;
O2 : START '##' ~'#'    {olistLevel >= 1}? {doOlist(2); resetFormatting();} ;
O3 : START '###' ~'#'   {olistLevel >= 2}? {doOlist(3); resetFormatting();} ;
O4 : START '####' ~'#'  {olistLevel >= 3}? {doOlist(4); resetFormatting();} ;
O5 : START '#####' ~'#' {olistLevel >= 4}? {doOlist(5); resetFormatting();} ;

/* ***** Horizontal Rules ***** */

Rule : LINE '---' '-'+? {resetFormatting();} ;

/* ***** Tables ***** */

CellSep : '|'  {resetFormatting();} ;
TdStart : '|'  {resetFormatting();} ;
ThStart : '|=' {resetFormatting();} ;

/* ***** Inline Formatting ***** */

BSt : '**' {!bold}?   {bold=true;} ;
ISt : '//' {!italic}? {italic=true;} ;
SSt : '--' {!strike}? {strike=true;} ;

BEnd : '**' {bold}?   {bold=false;} ;
IEnd : '//' {italic}? {italic=false;} ;
SEnd : '--' {strike}? {strike=false;} ;

NoWiki : '{{{' -> mode(PREFORMATTED_INLINE) ;

/* ***** Links ***** */

LiSt  : '[[' -> mode(LINK) ;
ImSt  : '{{' -> mode(LINK) ;

/* ***** Breaks ***** */

InlineBrk : '\\\\' ;

ParBreak  : LineBreak LineBreak+ {resetFormatting();} ;

LineBreak : '\r'? '\n'+? ;

/* ***** Links ***** */

RawUrl    : ('http' | 'ftp') '://' (~(' '|'\t'|'\r'|'\n'|'/')+ '/'?)+ {doUrl();};

WikiWords : (ALNUM+ ':')? (UPPER ((ALNUM|'.')* ALNUM)*) (UPPER ((ALNUM|'.')* ALNUM)*)+;

/* ***** Miscellaneous ***** */

Any : . ;
WS  : (' '|'\t'|'\r'|'\n')+ -> skip ;

fragment START : {start}? | LINE ;
fragment LINE  : ({getCharPositionInLine()==0}? WS? | LineBreak WS?);
fragment ALNUM : (ALPHA | DIGIT) ;
fragment ALPHA : (UPPER | LOWER) ;
fragment UPPER : ('A'..'Z') ;
fragment LOWER : ('a'..'z') ;
fragment DIGIT : ('0'..'9') ;

/* ***** Contextual stuff ***** */


mode LINK;

LiEnd : ']]' -> mode(DEFAULT_MODE) ;
ImEnd : '}}' -> mode(DEFAULT_MODE) ;

Sep : '|' ;

InLink : ~(']'|'}'|'|')+ ;

mode PREFORMATTED_INLINE;

AnyInlineText : ~('\r'|'\n') -> more;

AnyBlockText : AnyInlineText* LineBreak -> mode(PREFORMATTED_BLOCK), more ;

EndNoWikiInline : '}}}' (~'}' {_input.seek(_input.index()-1);} | EOF) -> mode(DEFAULT_MODE) ;

mode PREFORMATTED_BLOCK;

AnyText   : . -> more ;

EndNoWikiBlock : LINE '}}}' -> mode(DEFAULT_MODE) ;

{'fraction_non_alphanumeric': 0.2556079604279305, 'fraction_numerical': 0.011733578741516162, 'mean_word_length': 3.2997032640949553, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7250278', 'n_tokens_mistral': 3516, 'n_tokens_neox': 3309, 'n_words': 637}