<commit_before>import { Parser, Grammar } from 'nearley'
import * as grammar from './grammar'
import { ruleNodesMiddleware } from './nearleyMiddleware'
import { Token, toCst } from './cst'
import { toAst } from './ast'
import { Script } from './astTypes'
import { formatLexError } from './lexer'
import { getBoundaries, TextSpanError } from '../tokenHelpers'

export * from './astTypes'
export { Token, CstNode, CstNodeChild } from './cst'

const wrappedGrammar = { ...grammar, ParserRules: ruleNodesMiddleware(grammar.ParserRules) }
// TODO: Fix nearley.Grammar.fromCompiled() TypeScript definition.
const compiledGrammar: Grammar = (Grammar.fromCompiled as any)(wrappedGrammar)

export function parse (input: string): { errors: TextSpanError[], ast?: Script } {
  const parser = new Parser(compiledGrammar)
  try {
    parser.feed(input)
    const parsings = parser.results.map(toCst).map(toAst)
    if (parsings.length > 1) {
      throw new Error('Ambiguous grammar! This is likely a problem with Mangudai itself, ' +
        'not your script. Please report this issue along with the script that caused it.')
    }
    if (parsings.length === 0) {
      const { line, col } = parser.lexer as any
      throw {
        name: 'ParseError',
        message: 'Unexpected end of input.',
        boundaries: {
          start: { line: line, col: col - 1 },
          end: { line: line, col: col }
        }
      }
    }
    return {
      ast: parsings[0],
      errors: []
    }
  } catch (error) {
    let errorWithTextSpan: TextSpanError
    if (error && error.boundaries) {
      errorWithTextSpan = error
    } else if (error && error.token && error.token.type === 'invalid') {
      errorWithTextSpan = formatLexError(error)
    } else if (error && error.token) {
      errorWithTextSpan = formatParseError(error)
    } else {
      errorWithTextSpan = {
        name: 'ParseError',
        message: error && error.message,
        boundaries: {
          start: { line: 1, col: 0 },
          end: { line: 1, col: 1 }
        }
      }
    }
    return {
      errors: [errorWithTextSpan]
    }
  }
}

export function formatParseError (err: Error & { token: Token }): TextSpanError {
  return {
    name: 'ParseError',
    message: `Unexpected token ${err.token.type}: '${err.token.value}'.`,
    boundaries: getBoundaries(err.token)
  }
}
<commit_msg>fix(parse): export `TextSpanError` for linters<commit_after>import { Parser, Grammar } from 'nearley'
import * as grammar from './grammar'
import { ruleNodesMiddleware } from './nearleyMiddleware'
import { Token, toCst } from './cst'
import { toAst } from './ast'
import { Script } from './astTypes'
import { formatLexError } from './lexer'
import { getBoundaries, TextSpanError } from '../tokenHelpers'

export * from './astTypes'
export { Token, CstNode, CstNodeChild } from './cst'
export { TextSpanError } from '../tokenHelpers'

const wrappedGrammar = { ...grammar, ParserRules: ruleNodesMiddleware(grammar.ParserRules) }
// TODO: Fix nearley.Grammar.fromCompiled() TypeScript definition.
const compiledGrammar: Grammar = (Grammar.fromCompiled as any)(wrappedGrammar)

export function parse (input: string): { errors: TextSpanError[], ast?: Script } {
  const parser = new Parser(compiledGrammar)
  try {
    parser.feed(input)
    const parsings = parser.results.map(toCst).map(toAst)
    if (parsings.length > 1) {
      throw new Error('Ambiguous grammar! This is likely a problem with Mangudai itself, ' +
        'not your script. Please report this issue along with the script that caused it.')
    }
    if (parsings.length === 0) {
      const { line, col } = parser.lexer as any
      throw {
        name: 'ParseError',
        message: 'Unexpected end of input.',
        boundaries: {
          start: { line: line, col: col - 1 },
          end: { line: line, col: col }
        }
      }
    }
    return {
      ast: parsings[0],
      errors: []
    }
  } catch (error) {
    let errorWithTextSpan: TextSpanError
    if (error && error.boundaries) {
      errorWithTextSpan = error
    } else if (error && error.token && error.token.type === 'invalid') {
      errorWithTextSpan = formatLexError(error)
    } else if (error && error.token) {
      errorWithTextSpan = formatParseError(error)
    } else {
      errorWithTextSpan = {
        name: 'ParseError',
        message: error && error.message,
        boundaries: {
          start: { line: 1, col: 0 },
          end: { line: 1, col: 1 }
        }
      }
    }
    return {
      errors: [errorWithTextSpan]
    }
  }
}

export function formatParseError (err: Error & { token: Token }): TextSpanError {
  return {
    name: 'ParseError',
    message: `Unexpected token ${err.token.type}: '${err.token.value}'.`,
    boundaries: getBoundaries(err.token)
  }
}

{'fraction_non_alphanumeric': 0.12877682850593875, 'fraction_numerical': 0.003334027922483851, 'mean_word_length': 2.8095238095238093, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7177004', 'n_tokens_mistral': 1501, 'n_tokens_neox': 1477, 'n_words': 416}