<commit_before>{ 
  open Parser
  open Printf
}

let newline    = '\n' | "\r\n"
let whitespace = [' ' '\t']
let digit      = ['0'-'9']
let exp = 'e' ['-' '+']? digit+
let float = digit+ '.' digit* exp? | digit+ exp | '.' digit+ exp?

rule token = parse
  newline               { Lexing.new_line lexbuf; token lexbuf }
  | whitespace          { token lexbuf }
  
  (* Arithmetic operations *)
  | '+'                 { PLUS    }
  | '-'                 { MINUS   }
  | '*'                 { TIMES   }
  | '/'                 { DIVIDES }
  | '%'                 { MODE    }
  | '^'                 { CONVOLUTION }
  | '='                 { ASSIGN  }

 (* Logic operations *)
  | "!="                { NEQ     }
  | '<'                 { LT      }
  | "<="                { LEQ     }
  | '>'                 { GT      }
  | ">="                { GEQ     }
  | "=="                { EQ      }
  | "&&"                { AND     }
  | "||"                { OR      }
  | '!'                 { NOT     }
  | '?'                 { QUES    }

  (* Bit operations *)
   | '&'                { BITAND  }
   | '|'                { BITOR   }
   | '~'                { BITNOT  }

 (* Scoping, accessors, and sequences *)
  | '('                 { LPAREN    }
  | ')'                 { RPAREN    }
  | '['                 { LBRACKET  }
  | ']'                 { RBRACKET  }
  | '{'                 { LBRACE    }
  | '}'                 { RBRACE    }
  | ';'                 { SEMICOLON }
  | ':'                 { COLON     }
  | ','                 { COMMA     }
    
 (* Supported types *)
    | "bool"            { BOOL    }
    | "int"             { INT     }
    | "uint"            { UINT    }
    | "float"           { FLOAT   }
    | "histogram"       { HIST    }
    | "image"           { IMAGE   }
    
 (* Control flow and loop *)
  | "true"             { TRUE    }
  | "false"            { FALSE   }
  | "if"               { IF      }
  | "else"             { ELSE    }
  | "for"              { FOR     }
  | "in"               { IN      }
  | "while"            { WHILE   }
  | "return"           { RETURN  }

  | ['0'-'9']+ as lxm { LITERAL(int_of_string lxm) }
  | ['a'-'z' 'A'-'Z']['a'-'z' 'A'-'Z' '0'-'9' '_']* as lxm { ID(lxm) }
  | float   as flt     { FLOAT_LITERAL(float_of_string flt) }
  | "/*"                { comment lexbuf    }
  | "//"                { line_comment lexbuf   }
  | eof                { EOF }
  | _ as char { raise (Failure("illegal character " ^ Char.escaped char)) }

and comment = parse
     "*/" { token lexbuf }
    | _     { comment lexbuf }

and line_comment = parse
    newline { token lexbuf }
    | _     { line_comment lexbuf }<commit_msg>Fix single line comment.<commit_after>(* Scanner.mll for SIP language *)

{ 
  open Parser
  open Printf
}

let digit = ['0'-'9']
let exp   = 'e' ['-' '+']? digit+
let float = digit+ '.' digit* exp? | digit+ exp | '.' digit+ exp?

rule token = parse
  [' ' '\t' '\r' '\n']  { token lexbuf } (* Whitespace *)
  
  (* Read and write image files *)
  | "<<"                 { READIMG  }
  | ">>"                 { WRITEIMG }

  (* Arithmetic operations *)
  | '+'                 { PLUS    }
  | '-'                 { MINUS   }
  | '*'                 { TIMES   }
  | '/'                 { DIVIDES }
  | '%'                 { MODE    }
  | '^'                 { CONVOLUTION }
  | '='                 { ASSIGN  }

  (* Logic operations *)
  | "!="                { NEQ     }
  | '<'                 { LT      }
  | "<="                { LEQ     }
  | '>'                 { GT      }
  | ">="                { GEQ     }
  | "=="                { EQ      }
  | "&&"                { AND     }
  | "||"                { OR      }
  | '!'                 { NOT     }
  | '?'                 { QUES    }

  (* Bit operations *)
  | '&'                { BITAND  }
  | '|'                { BITOR   }
  | '~'                { BITNOT  }

  (* Scoping, accessors, and sequences *)
  | '('                 { LPAREN    }
  | ')'                 { RPAREN    }
  | '['                 { LBRACKET  }
  | ']'                 { RBRACKET  }
  | '{'                 { LBRACE    }
  | '}'                 { RBRACE    }
  | ';'                 { SEMICOLON }
  | ':'                 { COLON     }
  | ','                 { COMMA     }
    
  (* Supported types *)
  | "bool"            { BOOL    }
  | "int"             { INT     }
  | "uint"            { UINT    }
  | "float"           { FLOAT   }
  | "histogram"       { HIST    }
  | "image"           { IMAGE   }
    
  (* Control flow and loop *)
  | "true"             { TRUE    }
  | "false"            { FALSE   }
  | "if"               { IF      }
  | "else"             { ELSE    }
  | "for"              { FOR     }
  | "in"               { IN      }
  | "while"            { WHILE   }
  | "return"           { RETURN  }
  | "break"            { BREAK   }
  
  (* Identifier, types, comments and EOF. *)
  | digit+  as lxm     { ILITERAL(int_of_string lxm)   }
  | float   as flt     { FLITERAL(float_of_string flt) }  
  | ['a'-'z' 'A'-'Z']['a'-'z' 'A'-'Z' '0'-'9' '_']* as lxm { ID(lxm) }
  | "/*"               { comment lexbuf        }
  | "//"               { line_comment lexbuf   }
  | eof                { EOF }
  | _ as char { raise (Failure("illegal character " ^ Char.escaped char)) }

and comment = parse
     "*/"   { token lexbuf   }
    | _     { comment lexbuf }

and line_comment = parse
    '\n' | "\r\n" { token lexbuf        }
    | _           { line_comment lexbuf }
{'fraction_non_alphanumeric': 0.16612141039621955, 'fraction_numerical': 0.0018175209014903672, 'mean_word_length': 0.898895790200138, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6017962', 'n_tokens_mistral': 1931, 'n_tokens_neox': 1834, 'n_words': 327}