<commit_before>
var lang = require('./lang');

var TOKENS = {
  IDENTIFIER: 'identifier',
  DEFINE: 'define',
  IF: 'if',
  ELSE: 'else',
  ELIF: 'elif',
  WHILE: 'while',
  DO: 'do',
  NUMBER: 'number',
  NEWLINE: '\n',
  INDENT: 'indent',
  DEDENT: 'dedent',
  COLON: ':',
  EOF: 'eof'
}


var ProgramParser = function(code, builtins) {
  this.code = code;
  this.builtins = builtins;
  this.charIndex = 0;
  this.curIndent = 0;
  this.indentStack = [0];
  this.identifier = null;
  this.number = null;
  this.currentLine = 0;
};

ProgramParser.prototype.getToken = function() {
  if (this.code[this.charIndex] == ' ') {
    if (this.code[this.charIndex-1] == '\n') {
      // pay attention to whitespace at the beginning of a line
      var indent = '';
      while (this.code[this.charIndex] == ' ' && this.charIndex < this.code.length) {
        indent += ' ';
        this.charIndex++;
      }
      if (this.code[this.charIndex] != '\n') {
        var curIndent = this.indentStack[this.indentStack.length-1];
        if (curIndent > indent.length) {
          this.indentStack.pop();
          this.charIndex -= indent.length;
          return TOKENS.DEDENT;
        } else if (indent.length > curIndent) {
          this.indentStack.push(indent.length);
          return TOKENS.INDENT;
        } else {
          // indentation is equal, so just skip
          return this.getToken();
        }
      }
    } else {
      // don't pay attention to whitespace not at the beginning of a line
      while (this.code[this.charIndex] == ' ') {
        this.charIndex++;
      }
    }
  } else if (this.code[this.charIndex-1] == '\n' && this.indentStack[this.indentStack.length-1] > 0) {
    var nextTokenStart = this.charIndex;
    var indent = 0
    while (this.code[nextTokenStart] && (this.code[nextTokenStart] == '\n' || this.code[nextTokenStart] == ' ')) {
      if (this.code[nextTokenStart] == '\n') {
        indent = 0;
      } else {
        indent++;
      }
      // skip over blank new lines and go until you find the beginning of the next token
      nextTokenStart++;
    }
    if (this.indentStack[this.indentStack.length-1] > indent) {
      this.indentStack.pop();
      return TOKENS.DEDENT;
    }
  }

  var isAlpha = function(c) {
    return (c >= 'A' && c <= 'z') || c == '_';
  }

  var isAlphaNum = function(c) {
    return isAlpha(c) || (c >= '0' && c <= '9');
  }

  this.identifier = '';
  if (isAlpha(this.code[this.charIndex])) {
    //identifier, eat up the entire identifier
    while (isAlphaNum(this.code[this.charIndex])) {
      this.identifier += this.code[this.charIndex++];
    }
    switch (this.identifier) {
      case 'define':
      return TOKENS.DEFINE;
      case 'while':
      return TOKENS.WHILE;
      case 'if':
      return TOKENS.IF;
      case 'else':
      return TOKENS.ELSE;
      case 'elif':
      return TOKENS.ELIF;
      case 'do':
      return TOKENS.DO;
      default:
      return TOKENS.IDENTIFIER;
    }
  }

  if (this.code[this.charIndex] >= '0' && this.code[this.charIndex] <= '9') {
    this.number = '';
    while (this.code[this.charIndex] >= '0' && this.code[this.charIndex] <= '9') {
      this.number += this.code[this.charIndex++];
    }
    this.number = parseInt(this.number);
    return TOKENS.NUMBER;
  }

  if (this.code[this.charIndex] == '#') {
    while (this.code[this.charIndex] != '\n' && this.charIndex < this.code.length) {
      this.charIndex++;
    }
    return this.getToken();
  }

  if (this.code[this.charIndex] == ':') {
    return this.code[this.charIndex++];
  }

  if (this.code[this.charIndex] == '\n') {
    this.currentLine++;
    return this.code[this.charIndex++];
  }

  if (this.code[this.charIndex] == undefined) {
    return TOKENS.EOF;
  }

  return this.code[++this.charIndex];
};

ProgramParser.prototype.parseNewBlock = function() {
  if (this.getToken() != TOKENS.COLON) {
    throw new Error("Expected a colon on line "+this.currentLine);
  }
  if (this.getToken() != TOKENS.NEWLINE) {
    throw new Error("Expected a newline on line "+this.currentLine);
  }
  if (this.getToken() != TOKENS.INDENT) {
    throw new Error("Expected an indented block on line"+this.currentLine);
  }
};

ProgramParser.prototype.parseDefine = function() {
  var identifier = this.getToken();
  if (identifier != TOKENS.IDENTIFIER) {
    throw new Error("Expected a function name after define");
  }
  var funcName = this.identifier;
  var funcLine = this.currentLine;
  this.parseNewBlock();
  console.log("> Parsed function prototype with name", funcName);
  var expressions = this.parseBlock();
  console.log("< done parsing define", funcName);
  return new lang.Define(funcLine, funcName, expressions);
};

ProgramParser.prototype.parseDo = function() {
  var numberToken = this.getToken();
  if (numberToken != TOKENS.NUMBER) {
    throw new Error("Expected number after do on line "+this.currentLine);
  }<commit_msg>fix bug with indented comment handling and multiple new lines at start of block<commit_after>
var lang = require('./lang');

var TOKENS = {
  IDENTIFIER: 'identifier',
  DEFINE: 'define',
  IF: 'if',
  ELSE: 'else',
  ELIF: 'elif',
  WHILE: 'while',
  DO: 'do',
  NUMBER: 'number',
  NEWLINE: '\n',
  INDENT: 'indent',
  DEDENT: 'dedent',
  COLON: ':',
  EOF: 'eof'
}


var ProgramParser = function(code, builtins) {
  this.code = code;
  this.builtins = builtins;
  this.charIndex = 0;
  this.curIndent = 0;
  this.indentStack = [0];
  this.identifier = null;
  this.number = null;
  this.currentLine = 0;
};

ProgramParser.prototype.getToken = function() {
  if (this.code[this.charIndex] == ' ') {
    if (this.code[this.charIndex-1] == '\n') {
      // pay attention to whitespace at the beginning of a line
      var indent = '';
      while (this.code[this.charIndex] == ' ' && this.charIndex < this.code.length) {
        indent += ' ';
        this.charIndex++;
      }
      if (this.code[this.charIndex] != '\n' && this.code[this.charIndex] != '#') {
        var curIndent = this.indentStack[this.indentStack.length-1];
        if (curIndent > indent.length) {
          this.indentStack.pop();
          this.charIndex -= indent.length;
          return TOKENS.DEDENT;
        } else if (indent.length > curIndent) {
          this.indentStack.push(indent.length);
          return TOKENS.INDENT;
        } else {
          // indentation is equal, so just skip
          return this.getToken();
        }
      }
    } else {
      // don't pay attention to whitespace not at the beginning of a line
      while (this.code[this.charIndex] == ' ') {
        this.charIndex++;
      }
    }
  } else if (this.code[this.charIndex-1] == '\n' && this.indentStack[this.indentStack.length-1] > 0) {
    var nextTokenStart = this.charIndex;
    var indent = 0
    while (this.code[nextTokenStart] && (this.code[nextTokenStart] == '\n' || this.code[nextTokenStart] == ' ')) {
      if (this.code[nextTokenStart] == '\n') {
        indent = 0;
      } else {
        indent++;
      }
      // skip over blank new lines and go until you find the beginning of the next token
      nextTokenStart++;
    }
    if (this.indentStack[this.indentStack.length-1] > indent) {
      this.indentStack.pop();
      return TOKENS.DEDENT;
    }
  }

  var isAlpha = function(c) {
    return (c >= 'A' && c <= 'z') || c == '_';
  }

  var isAlphaNum = function(c) {
    return isAlpha(c) || (c >= '0' && c <= '9');
  }

  this.identifier = '';
  if (isAlpha(this.code[this.charIndex])) {
    //identifier, eat up the entire identifier
    while (isAlphaNum(this.code[this.charIndex])) {
      this.identifier += this.code[this.charIndex++];
    }
    switch (this.identifier) {
      case 'define':
      return TOKENS.DEFINE;
      case 'while':
      return TOKENS.WHILE;
      case 'if':
      return TOKENS.IF;
      case 'else':
      return TOKENS.ELSE;
      case 'elif':
      return TOKENS.ELIF;
      case 'do':
      return TOKENS.DO;
      default:
      return TOKENS.IDENTIFIER;
    }
  }

  if (this.code[this.charIndex] >= '0' && this.code[this.charIndex] <= '9') {
    this.number = '';
    while (this.code[this.charIndex] >= '0' && this.code[this.charIndex] <= '9') {
      this.number += this.code[this.charIndex++];
    }
    this.number = parseInt(this.number);
    return TOKENS.NUMBER;
  }

  if (this.code[this.charIndex] == '#') {
    while (this.code[this.charIndex] != '\n' && this.charIndex < this.code.length) {
      this.charIndex++;
    }
    return this.getToken();
  }

  if (this.code[this.charIndex] == ':') {
    return this.code[this.charIndex++];
  }

  if (this.code[this.charIndex] == '\n') {
    this.currentLine++;
    return this.code[this.charIndex++];
  }

  if (this.code[this.charIndex] == undefined) {
    return TOKENS.EOF;
  }

  return this.code[++this.charIndex];
};

ProgramParser.prototype.parseNewBlock = function() {
  if (this.getToken() != TOKENS.COLON) {
    throw new Error("Expected a colon on line "+this.currentLine);
  }
  if (this.getToken() != TOKENS.NEWLINE) {
    throw new Error("Expected a newline on line "+this.currentLine);
  }
  var token;
  while ((token = this.getToken()) == TOKENS.NEWLINE) {
    continue
  }
  if (token != TOKENS.INDENT) {
    throw new Error("Expected an indented block on line "+this.currentLine);
  }
};

ProgramParser.prototype.parseDefine = function() {
  var identifier = this.getToken();
  if (identifier != TOKENS.IDENTIFIER) {
    throw new Error("Expected a function name after define");
  }
  var funcName = this.identifier;
  var funcLine = this.currentLine;
  this.parseNewBlock();
  console.log("> Parsed function prototype with name", funcName);
  var expressions = this.parseBlock();
  console.log("< done parsing define", funcName);
  return new lang.Define(funcLine, funcName, expressions);
};

ProgramParser.prototype.parseDo = function() {
  var numberToken = this.getToken();
  if (numberToken != TOKENS.NUMBER) {
    throw new Error("Expected number after do on line "+this.currentLine);
  }
{'fraction_non_alphanumeric': 0.158898093622118, 'fraction_numerical': 0.0035931729713544265, 'mean_word_length': 2.7318435754189943, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1277207', 'n_tokens_mistral': 3391, 'n_tokens_neox': 3337, 'n_words': 758}