<commit_before>/*
 * Classic example grammar, which recognizes simple arithmetic expressions like
 * "2*(3+4)". The parser generated from this grammar then computes their value.
 */

/*
 * DCPU16 ASM Parser
 */

{
  var line = 1;
}

start
  = program:statements
    {
      return program;
    }


statements
  = stmt:stmt nl* statements:statements
    { return [stmt].concat(statements); }

  / stmt:stmt nl*
    { return [stmt]; }

  / nl*
    { return []; }


stmt
  = ws ":" label:label ws cmd:cmd ws
    {
      return {
        line: line,
        label: label,
        cmd: cmd,
        type: "op"
      };
    }

  / ws ":" label:label ws
    {
      return {
        line: line,
        label: label,
        type: "op"
      };
    }

  / "#macro" ws name:label ws "(" ws params:paramlist ws ")" ws "{" ws nl code:statements ws "}" ws
    {
      return {
        line: line,
        name: name,
        params: params,
        src: code,
        type: "macro"
      };
    }

  / ws name:label ws "(" ws params:paramlist ws ")"
    {
      return {
        line: line,
        name: name,
        params: params,
        type: "macrocall"
      };
    }

  / ws cmd:cmd ws
    {
      return {
        line: line,
        cmd: cmd,
        type: "op"
      };
    }


label "label"
  = label:([a-zA-Z0-9_\.]+)
    {
      return label.join('');
    }


cmd
  = ws op:op ws plist:paramlist
    {
      return {
        op: op,
        params: plist
      };
    }

op "operator"
  = op:([a-zA-Z][a-zA-Z][a-zA-Z]) { return op.join('').toUpperCase(); }

paramlist
  = v:param ws "," ws l:paramlist
    {
      return [v].concat(l);
    }
  / v:param
    {
      return [v];
    }

param "parameter"
  = "[" v:additive "]"
    {
      v.hasBrackets = true;
      return v;
    }
  / v:value
    {
      return v;
    }

additive
  = left:value "+" right:additive
    {
      return {
        action: "add",
        children: [left, right]
      };
    }
  / v:value
    {
      return v;
    }

value
  = v:number
    {
      return {
        value: v,
        isNumber: true
      };
    }
  / v:label
    {
      return {
        value: v,
        isString: true
      };
    }

number "number"
  /* oct */
  = v:("0"[0-9]+)
    { return parseInt(v[0] + v[1].join(''), 8); }
  /* hex */
  / v:("0x"[0-9a-fA-F]+)
    { return parseInt(v[0] + v[1].join(''), 16); }
  /* dec */
  / v:[0-9]+
    { return parseInt(v.join(''), 10); }


ws "whitespace"
  = [" ""\t"]*


nl "newline"
  = "\n" { line++; }
  / "\r\n" { line++; }
  / "\r" { line++; }
<commit_msg>fixed some issues with the grammar<commit_after>/*
 * Classic example grammar, which recognizes simple arithmetic expressions like
 * "2*(3+4)". The parser generated from this grammar then computes their value.
 */

/*
 * DCPU16 ASM Parser
 */

{
  var line = 1;
}

start
  = program:statements
    {
      return program;
    }


statements
  = stmt:stmt nl* statements:statements
    { return [stmt].concat(statements); }

  / stmt:stmt nl*
    { return [stmt]; }

  / nl*
    { return []; }


stmt
  = ws ":" label:label ws cmd:cmd ws
    {
      return {
        line: line,
        label: label,
        cmd: cmd,
        action: "op"
      };
    }

  / ws ":" label:label ws
    {
      return {
        line: line,
        label: label,
        action: "op"
      };
    }

  / ws "#macro"? ws name:label ws "(" ws params:paramlist ws ")" ws nl? "{" ws nl? code:statements ws "}" ws
    {
      return {
        name: name,
        params: params,
        src: code,
        action: "macro"
      };
    }

  / ws name:label ws "(" ws params:paramlist ws ")"
    {
      return {
        line: line,
        name: name,
        params: params,
        action: "macrocall"
      };
    }

  / ws cmd:cmd ws
    {
      return {
        line: line,
        cmd: cmd,
        action: "op"
      };
    }


label "label"
  = label:([a-zA-Z0-9_\.]+)
    {
      return label.join('');
    }


cmd
  = ws op:op ws plist:paramlist
    {
      return {
        op: op,
        params: plist
      };
    }

op "operator"
  = op:([a-zA-Z][a-zA-Z][a-zA-Z]) { return op.join('').toUpperCase(); }

paramlist
  = v:param ws "," ws l:paramlist
    {
      return [v].concat(l);
    }
  / v:param
    {
      return [v];
    }

param "parameter"
  = "[" v:additive "]"
    {
      v.hasBrackets = true;
      return v;
    }
  / v:value
    {
      return v;
    }

additive
  = left:value "+" right:additive
    {
      return {
        action: "add",
        children: [left, right]
      };
    }
  / v:value
    {
      return v;
    }

value
  = v:number
    {
      return {
        value: v,
        isNumber: true
      };
    }
  / v:label
    {
      return {
        value: v,
        isString: true
      };
    }

number "number"
  /* oct */
  = v:("0"[0-9]+)
    { return parseInt(v[0] + v[1].join(''), 8); }
  /* hex */
  / v:("0x"[0-9a-fA-F]+)
    { return parseInt(v[0] + v[1].join(''), 16); }
  /* dec */
  / v:[0-9]+
    { return parseInt(v.join(''), 10); }


ws "whitespace"
  = [" ""\t"]*


nl "newline"
  = "\n" { line++; }
  / "\r\n" { line++; }
  / "\r" { line++; }

{'fraction_non_alphanumeric': 0.1719782355227361, 'fraction_numerical': 0.00971628449280995, 'mean_word_length': 1.5379684418145956, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2517131', 'n_tokens_mistral': 2028, 'n_tokens_neox': 1908, 'n_words': 429}