<commit_before>#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <assert.h>

/* debug */
#define TOKENV(name, v, len) \
  memset(sbuf, 0, 1024); \
  strncpy(sbuf, v, len); \
  printf("<%s:%s> ", name, sbuf)
#define TOKEN(name) \
  printf("<%s> ", name)

#define MAX_INDENT     30
#define INDENT_PUSH(i) (assert(pind < MAX_INDENT-1), inds[++pind] = i)
#define INDENT_POP()   inds[--pind]

%%{
  machine min;
  
  newline     = "\r"? "\n" %{ printf(" (line %d)\n", curline); curline++; };
  whitespace  = " " | "\f" | "\t" | "\v";

  indent      = newline whitespace+ newline?;
  term        = (newline | ".");
  id          = [a-z]+;
  int         = [0-9]+;
  string      = '"' (any - '"')* '"' | "'" (any - "'")* "'";
  comment     = "#"+ (any - newline)* newline;
  
  main := |*
    
    # indentation magic
    indent      => {
      if (ts[0] == '\r') ts++;
      if (ts[0] == '\n') ts++;
      
      if (te-ts > ind) {
        ind = te-ts;
        INDENT_PUSH(ind);
        TOKEN("term");
        TOKEN("indent");
      } else if (te-ts < ind) {
        ind = te-ts;
        INDENT_POP();
        TOKEN("dedent");
        TOKEN("term");
      } else {
        TOKEN("term");
      }
    };
    newline     => {
      while(pind > 0) {
        INDENT_POP();
        TOKEN("dedent");
      }
      ind = 0;
      TOKEN("term");
    };
    
    whitespace;
    comment;
    
    # literals
    id          => { TOKENV("id", ts, te-ts); };
    int         => { TOKENV("int", ts, te-ts); };
    string      => { TOKENV("string", ts+1, te-ts-2); };
    term        => { TOKEN("term"); };
    
    # ponctuation
    ","         => { TOKEN("comma"); };
    ":"         => { TOKEN("colon"); };
    "("         => { TOKEN("o_par"); };
    ")"         => { TOKEN("c_par"); };
    "{"         => { TOKEN("o_bra"); };
    "}"         => { TOKEN("c_bra"); };
    "["         => { TOKEN("o_sqbra"); };
    "]"         => { TOKEN("c_sqbra"); };
    
    # assign
    "="         => { TOKEN("assign"); };
    "+="        => { TOKEN("assign_plus"); };
    "-="        => { TOKEN("assign_minus"); };
    
    # operators
    "=="        => { TOKEN("eq"); };
    "!="        => { TOKEN("neq"); };
    "||"        => { TOKEN("or"); };
    "&&"        => { TOKEN("and"); };
    "<"         => { TOKEN("lt"); };
    "<="        => { TOKEN("lte"); };
    ">"         => { TOKEN("gt"); };
    ">="        => { TOKEN("gte"); };
    "<<"        => { TOKEN("lsh"); };
    ">>"        => { TOKEN("rsh"); };
    "*"         => { TOKEN("mult"); };
    "**"        => { TOKEN("pow"); };
    "/"         => { TOKEN("div"); };
    "%"         => { TOKEN("mod"); };
    "+"         => { TOKEN("plus"); };
    "-"         => { TOKEN("minus"); };
    "@"         => { TOKEN("at"); };
    "@@"         => { TOKEN("atat"); };
  *|;
  
  write data nofinal;
}%%

void min_parse(char *code) {
  int cs, act;
  char *p, *pe, *ts, *te, *eof = 0;
  int curline = 1;
  int ind = 0;
  int inds[MAX_INDENT]; /* max indent level */
  int pind = 0;
  char sbuf[1024]; /* debug */
  <commit_msg>More operators<commit_after>#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <assert.h>

/* debug */
#define TOKENV(name, v, len) \
  memset(sbuf, 0, 1024); \
  strncpy(sbuf, v, len); \
  printf("<%s:%s> ", name, sbuf)
#define TOKEN(name) \
  printf("<%s> ", name)

#define MAX_INDENT     30
#define INDENT_PUSH(i) (assert(pind < MAX_INDENT-1), inds[++pind] = i)
#define INDENT_POP()   inds[--pind]

%%{
  machine min;
  
  newline     = "\r"? "\n" %{ printf(" (line %d)\n", curline); curline++; };
  whitespace  = " " | "\f" | "\t" | "\v";

  indent      = newline whitespace+ newline?;
  term        = (newline | ".");
  id          = [a-z]+;
  int         = [0-9]+;
  string      = '"' (any - '"')* '"' | "'" (any - "'")* "'";
  comment     = "#"+ (any - newline)* newline;
  
  main := |*
    
    # indentation magic
    indent      => {
      if (ts[0] == '\r') ts++;
      if (ts[0] == '\n') ts++;
      
      if (te-ts > ind) {
        ind = te-ts;
        INDENT_PUSH(ind);
        TOKEN("term");
        TOKEN("indent");
      } else if (te-ts < ind) {
        ind = te-ts;
        INDENT_POP();
        TOKEN("dedent");
        TOKEN("term");
      } else {
        TOKEN("term");
      }
    };
    newline     => {
      while(pind > 0) {
        INDENT_POP();
        TOKEN("dedent");
      }
      ind = 0;
      TOKEN("term");
    };
    
    whitespace;
    comment;
    
    # literals
    id          => { TOKENV("id", ts, te-ts); };
    int         => { TOKENV("int", ts, te-ts); };
    string      => { TOKENV("string", ts+1, te-ts-2); };
    term        => { TOKEN("term"); };
    
    # ponctuation
    ","         => { TOKEN("comma"); };
    ":"         => { TOKEN("colon"); };
    "("         => { TOKEN("o_par"); };
    ")"         => { TOKEN("c_par"); };
    "{"         => { TOKEN("o_bra"); };
    "}"         => { TOKEN("c_bra"); };
    "["         => { TOKEN("o_sqbra"); };
    "]"         => { TOKEN("c_sqbra"); };
    
    # assign
    "="         => { TOKEN("assign"); };
    "+="        => { TOKEN("assign_plus"); };
    "-="        => { TOKEN("assign_minus"); };
    
    # operators
    "=="        => { TOKEN("eq"); };
    "!="        => { TOKEN("neq"); };
    "!"         => { TOKEN("not"); };
    "||"        => { TOKEN("or"); };
    "&&"        => { TOKEN("and"); };
    "<"         => { TOKEN("lt"); };
    "<="        => { TOKEN("lte"); };
    ">"         => { TOKEN("gt"); };
    ">="        => { TOKEN("gte"); };
    "<<"        => { TOKEN("lsh"); };
    ">>"        => { TOKEN("rsh"); };
    "*"         => { TOKEN("mult"); };
    "**"        => { TOKEN("pow"); };
    "/"         => { TOKEN("div"); };
    "%"         => { TOKEN("mod"); };
    "+"         => { TOKEN("plus"); };
    "-"         => { TOKEN("minus"); };
    "@"         => { TOKEN("at"); };
    "@@"        => { TOKEN("atat"); };
    "?"         => { TOKEN("ques"); };
    ".."        => { TOKEN("to"); };
  *|;
  
  write data nofinal;
}%%

void min_parse(char *code) {
  int cs, act;
  char *p, *pe, *ts, *te, *eof = 0;
  int curline = 1;
  int ind = 0;
  int inds[MAX_INDENT]; /* max indent level */
  int pind = 0;
  char sbuf[1024]; /* debug */
  
{'fraction_non_alphanumeric': 0.26071657069737686, 'fraction_numerical': 0.007677543186180422, 'mean_word_length': 1.613038027580443, 'pattern_counts': {'":': 2, '<': 27, '<?xml version=': 0, '>': 100, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7412843', 'n_tokens_mistral': 2501, 'n_tokens_neox': 2373, 'n_words': 384}