<commit_before>/**
 * nadesiko v3 parser (demo version)
 */
const {opPriority, keizokuJosi} = require('./nako_parser_const')
const {NakoParserBase, NakoSyntaxError} = require('./nako_parser_base')
const operatorList = []
for (const key in opPriority) {operatorList.push(key)}

class NakoParser extends NakoParserBase {
  /**
   * @param tokens 字句解析済みのトークンの配列
   * @return {{type, block, line}} AST(構文木)
   */
  parse (tokens) {
    this.reset()
    this.tokens = tokens
    // 解析開始
    return this.startParser()
  }

  startParser () {
    const b = this.ySentenceList()
    const c = this.get()
    if (c && c.type !== 'eof') {
      const name = this.nodeToStr(c)
      throw new NakoSyntaxError(
        `構文解析でエラー。${name}の使い方が間違っています。`, c.line, this.filename)
    }
    return b
  }

  ySentenceList () {
    const blocks = []
    let line = -1
    while (!this.isEOF()) {
      const n = this.ySentence()
      if (!n) {break}
      blocks.push(n)
      if (line < 0) {line = n.line}
    }
    if (blocks.length === 0) {
      throw new NakoSyntaxError('構文解析に失敗:' + this.nodeToStr(this.peek()),
        line, this.filename)
    }

    return {type: 'block', block: blocks, line}
  }

  ySentence () {
    // 最初の語句が決まっている構文
    if (this.check('eol')) {return this.get()}
    if (this.check('embed_code')) {return this.get()}
    if (this.check('もし')) {return this.yIF()}
    if (this.check('エラー監視')) {return this.yTryExcept()}
    if (this.check('逐次実行')) {return this.yPromise()}
    if (this.accept(['抜ける'])) {return {type: 'break', line: this.y[0].line, josi: ''}}
    if (this.accept(['続ける'])) {return {type: 'continue', line: this.y[0].line, josi: ''}}
    if (this.accept(['require', 'string', '取込']))
      {return {
        type: 'require',
        value: this.y[1].value,
        line: this.y[0].line,
        josi: ''
      }}

    // 先読みして初めて確定する構文
    if (this.accept([this.yLet])) {return this.y[0]}
    if (this.accept([this.yDefFunc])) {return this.y[0]}
    if (this.accept([this.yCall])) { // 関数呼び出しの他、各種構文の実装
      const c1 = this.y[0]
      if (c1.josi === 'して') { // 連文をblockとして接続する(もし構文、逐次実行構文などのため)
        const c2 = this.ySentence()
        if (c2 !== null) {
          return {
            type: 'block',
            block: [c1, c2],
            josi: c2.josi
          }
        }
      }
      return c1
    }
    return null
  }

  yBlock () {
    const blocks = []
    let line = -1
    if (this.check('ここから')) {this.get()}
    while (!this.isEOF()) {
      if (this.checkTypes(['違えば', 'ここまで', 'エラー'])) {break}
      if (!this.accept([this.ySentence])) {break}
      blocks.push(this.y[0])
      if (line < 0) {line = this.y[0].line}
    }<commit_msg>パーサーを実装<commit_after>/**
 * nadesiko v3 parser (demo version)
 */
const {opPriority, keizokuJosi} = require('./nako_parser_const')
const {NakoParserBase, NakoSyntaxError} = require('./nako_parser_base')
const operatorList = []
for (const key in opPriority) {operatorList.push(key)}

class NakoParser extends NakoParserBase {
  /**
   * @param tokens 字句解析済みのトークンの配列
   * @return {{type, block, line}} AST(構文木)
   */
  parse (tokens) {
    this.reset()
    this.tokens = tokens
    // 解析開始
    return this.startParser()
  }

  startParser () {
    const b = this.ySentenceList()
    const c = this.get()
    if (c && c.type !== 'eof') {
      const name = this.nodeToStr(c)
      throw new NakoSyntaxError(
        `構文解析でエラー。${name}の使い方が間違っています。`, c.line, this.filename)
    }
    return b
  }

  ySentenceList () {
    const blocks = []
    let line = -1
    while (!this.isEOF()) {
      const n = this.ySentence()
      if (!n) {break}
      blocks.push(n)
      if (line < 0) {line = n.line}
    }
    if (blocks.length === 0) {
      throw new NakoSyntaxError('構文解析に失敗:' + this.nodeToStr(this.peek()),
        line, this.filename)
    }

    return {type: 'block', block: blocks, line}
  }

  ySentence () {
    // 最初の語句が決まっている構文
    if (this.check('eol')) {return this.get()}
    if (this.check('embed_code')) {return this.get()}
    if (this.check('もし')) {return this.yIF()}
    if (this.check('エラー監視')) {return this.yTryExcept()}
    if (this.check('逐次実行')) {return this.yPromise()}
    if (this.accept(['抜ける'])) {return {type: 'break', line: this.y[0].line, josi: ''}}
    if (this.accept(['続ける'])) {return {type: 'continue', line: this.y[0].line, josi: ''}}
    if (this.accept(['require', 'string', '取込']))
      {return {
        type: 'require',
        value: this.y[1].value,
        line: this.y[0].line,
        josi: ''
      }}

    // 先読みして初めて確定する構文
    if (this.accept([this.yLet])) {return this.y[0]}
    if (this.accept([this.yDefTest])) {return this.y[0]}
    if (this.accept([this.yDefFunc])) {return this.y[0]}
    if (this.accept([this.yCall])) { // 関数呼び出しの他、各種構文の実装
      const c1 = this.y[0]
      if (c1.josi === 'して') { // 連文をblockとして接続する(もし構文、逐次実行構文などのため)
        const c2 = this.ySentence()
        if (c2 !== null) {
          return {
            type: 'block',
            block: [c1, c2],
            josi: c2.josi
          }
        }
      }
      return c1
    }
    return null
  }

  yBlock () {
    const blocks = []
    let line = -1
    if (this.check('ここから')) {this.get()}
    while (!this.isEOF()) {
      if (this.checkTypes(['違えば', 'ここまで', 'エラー'])) {break}
      if (!this.accept([this.ySentence])) {break}
      blocks.push(this.y[0])
      if (line < 0) {line = this.y[0].line}
    }
{'fraction_non_alphanumeric': 0.18448753462603878, 'fraction_numerical': 0.008679593721144968, 'mean_word_length': 2.4629156010230178, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2315477', 'n_tokens_mistral': 2179, 'n_tokens_neox': 2184, 'n_words': 426}