<commit_before>var Stream = require("./stream.js").Stream;
var Lexer  = require("./lexer.js").Lexer;
var Node   = require("./node.js").Node;
var assert = require("assert");

function Parser() {
}

Parser.prototype = {
  initStatus: function (stream) {
    if (typeof stream === "string")
      stream = new Stream(stream);
    this.lexer = new Lexer(stream);
    this.document = [];
  },

  parse: function (stream) {
    this.initStatus(stream);
    this.parseDocument();
    return this.document;
  },

  // ------------------------------------------------------------
  // <Document> ::= <Element>*
  // ------------------------------------------------------------

  parseDocument: function () {
    while (this.lexer.hasNext())
      this.document.push(this.parseElement());
  },

  // ------------------------------------------------------------
  // <Element> ::= (<Header> | <List>
  //              | <Preformatted> | <Paragraph>
  //              | <Table>)*
  // ------------------------------------------------------------

  parseElement: function () {
    switch (this.lexer.peekNextToken().type) {
    case Lexer.tokens.header:
      return this.parseHeader();
    case Lexer.tokens.preformatted:
      return this.parsePreformatted();
    case Lexer.tokens.orderedListElement:
    case Lexer.tokens.unorderedListElement:
      return this.parseList();
    case Lexer.tokens.paragraph:
      return this.parseParagraph();
    case Lexer.tokens.blank:
      this.lexer.getNextToken();
      return this.parseElement(); // loop
    }

    throw new Error("SyntaxError: Unknown Line");
  },

  // ------------------------------------------------------------
  // <Header>
  //
  // : preformatted
  // : block
  // ------------------------------------------------------------

  parseHeader: function () {
    var headerToken = this.lexer.getNextToken();
    assert.ok(headerToken.indentation === 0);

    var header = Node.createHeader([
      this.createTextNode(headerToken.content) // TODO: Parse inline markups
    ], { level: headerToken.level });

    return header;
  },

  // ------------------------------------------------------------
  // <Preformatted>
  //
  // : preformatted
  // : block
  // ------------------------------------------------------------

  parsePreformatted: function () {
    var preformattedFirstToken = this.lexer.peekNextToken();
    assert.ok(preformattedFirstToken.type === Lexer.tokens.preformatted);
    var preformatted = Node.createPreformatted([]);

    while (this.lexer.hasNext()) {
      var token = this.lexer.peekNextToken();
      if (token.type !== Lexer.tokens.preformatted ||
          token.indentation < preformattedFirstToken.indentation)
        break;
      this.lexer.getNextToken();
      preformatted.children.push(this.createTextNode(token.content));
    }

    return preformatted;
  },

  // ------------------------------------------------------------
  // <List>
  //
  //  - foo
  //    1. bar
  //    2. baz
  // ------------------------------------------------------------

  parseList: function () {
    var rootToken = this.lexer.peekNextToken();
    var list = rootToken.type === Lexer.tokens.unorderedListElement ?
      Node.createUnorderedList([]) : Node.createOrderedList([]);

    while (this.lexer.hasNext()) {
      var nextToken = this.lexer.peekNextToken();
      if (!nextToken.isListElement() || nextToken.indentation !== rootToken.indentation)
        break;
      list.children.push(this.parseListElement(rootToken.indentation));
    }

    return list;
  },

  parseListElement: function (rootIndentation) {
    var listElementToken = this.lexer.getNextToken();
    assert.ok(listElementToken.isListElement());
    var listElement = Node.createListElement([
      this.createTextNode(listElementToken.content)
    ]);

    while (this.lexer.hasNext()) {
      var nextToken = this.lexer.peekNextToken();
      if (nextToken.indentation <= rootIndentation && nextToken.type !== Lexer.tokens.blank)
        break;
      listElement.children.push(this.parseElement()); // recursive
    }

    return listElement;
  },

  // ------------------------------------------------------------
  // <Paragraph>
  // ------------------------------------------------------------

  parseParagraph: function () {
    var paragraphFisrtToken = this.lexer.peekNextToken();
    assert.ok(paragraphFisrtToken.type === Lexer.tokens.paragraph);
    var paragraph = Node.createParagraph([]);

    while (this.lexer.hasNext()) {
      var nextToken = this.lexer.peekNextToken();
      if (nextToken.type !== Lexer.tokens.paragraph
          || nextToken.indentation < paragraphFisrtToken.indentation)
        break;
      this.lexer.getNextToken();
      paragraph.children.push(this.createTextNode(nextToken.content));
    }
<commit_msg>parseElement may returns null<commit_after>var Stream = require("./stream.js").Stream;
var Lexer  = require("./lexer.js").Lexer;
var Node   = require("./node.js").Node;
var assert = require("assert");

function Parser() {
}

Parser.prototype = {
  initStatus: function (stream) {
    if (typeof stream === "string")
      stream = new Stream(stream);
    this.lexer = new Lexer(stream);
    this.document = [];
  },

  parse: function (stream) {
    this.initStatus(stream);
    this.parseDocument();
    return this.document;
  },

  // ------------------------------------------------------------
  // <Document> ::= <Element>*
  // ------------------------------------------------------------

  parseDocument: function () {
    while (this.lexer.hasNext()) {
      var element = this.parseElement();
      if (element) this.document.push(element);
    }
  },

  // ------------------------------------------------------------
  // <Element> ::= (<Header> | <List>
  //              | <Preformatted> | <Paragraph>
  //              | <Table>)*
  // ------------------------------------------------------------

  parseElement: function () {
    switch (this.lexer.peekNextToken().type) {
    case Lexer.tokens.header:
      return this.parseHeader();
    case Lexer.tokens.preformatted:
      return this.parsePreformatted();
    case Lexer.tokens.orderedListElement:
    case Lexer.tokens.unorderedListElement:
      return this.parseList();
    case Lexer.tokens.paragraph:
      return this.parseParagraph();
    case Lexer.tokens.blank:
      this.lexer.getNextToken();
      return this.lexer.hasNext() ? this.parseElement() : null; // loop
    }

    throw new Error("SyntaxError: Unknown Line");
  },

  // ------------------------------------------------------------
  // <Header>
  //
  // : preformatted
  // : block
  // ------------------------------------------------------------

  parseHeader: function () {
    var headerToken = this.lexer.getNextToken();
    assert.ok(headerToken.indentation === 0);

    var header = Node.createHeader([
      this.createTextNode(headerToken.content) // TODO: Parse inline markups
    ], { level: headerToken.level });

    return header;
  },

  // ------------------------------------------------------------
  // <Preformatted>
  //
  // : preformatted
  // : block
  // ------------------------------------------------------------

  parsePreformatted: function () {
    var preformattedFirstToken = this.lexer.peekNextToken();
    assert.ok(preformattedFirstToken.type === Lexer.tokens.preformatted);
    var preformatted = Node.createPreformatted([]);

    while (this.lexer.hasNext()) {
      var token = this.lexer.peekNextToken();
      if (token.type !== Lexer.tokens.preformatted ||
          token.indentation < preformattedFirstToken.indentation)
        break;
      this.lexer.getNextToken();
      preformatted.children.push(this.createTextNode(token.content));
    }

    return preformatted;
  },

  // ------------------------------------------------------------
  // <List>
  //
  //  - foo
  //    1. bar
  //    2. baz
  // ------------------------------------------------------------

  parseList: function () {
    var rootToken = this.lexer.peekNextToken();
    var list = rootToken.type === Lexer.tokens.unorderedListElement ?
      Node.createUnorderedList([]) : Node.createOrderedList([]);

    while (this.lexer.hasNext()) {
      var nextToken = this.lexer.peekNextToken();
      if (!nextToken.isListElement() || nextToken.indentation !== rootToken.indentation)
        break;
      list.children.push(this.parseListElement(rootToken.indentation));
    }

    return list;
  },

  parseListElement: function (rootIndentation) {
    var listElementToken = this.lexer.getNextToken();
    assert.ok(listElementToken.isListElement());
    var listElement = Node.createListElement([
      this.createTextNode(listElementToken.content)
    ]);

    while (this.lexer.hasNext()) {
      var nextToken = this.lexer.peekNextToken();
      if (nextToken.indentation <= rootIndentation && nextToken.type !== Lexer.tokens.blank)
        break;
      var element = this.parseElement();
      if (element) listElement.children.push(element); // recursive
    }

    return listElement;
  },

  // ------------------------------------------------------------
  // <Paragraph>
  // ------------------------------------------------------------

  parseParagraph: function () {
    var paragraphFisrtToken = this.lexer.peekNextToken();
    assert.ok(paragraphFisrtToken.type === Lexer.tokens.paragraph);
    var paragraph = Node.createParagraph([]);

    while (this.lexer.hasNext()) {
      var nextToken = this.lexer.peekNextToken();
      if (nextToken.type !== Lexer.tokens.paragraph
          || nextToken.indentation < paragraphFisrtToken.indentation)
        break;
      this.lexer.getNextToken();
      paragraph.children.push(this.createTextNode(nextToken.content));
    }

{'fraction_non_alphanumeric': 0.2720196640721016, 'fraction_numerical': 0.0006145022531749283, 'mean_word_length': 1.6753424657534246, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4513332', 'n_tokens_mistral': 2659, 'n_tokens_neox': 2600, 'n_words': 439}