<commit_before>import { last } from '../../CollectionHelpers'
import { NormalizedSettings } from '../../NormalizedSettings'
import { NON_BLANK_PATTERN } from '../../Patterns'
import { OutlineSyntaxNode } from '../../SyntaxNodes/OutlineSyntaxNode'
import { ThematicBreak } from '../../SyntaxNodes/ThematicBreak'
import { HeadingLeveler } from './HeadingLeveler'
import { LineConsumer } from './LineConsumer'
import { parseParagraphOrLineBlock } from './parseParagraphOrLineBlock'
import { tryToParseBlankLineSeparation } from './tryToParseBlankLineSeparation'
import { tryToParseBlockquote } from './tryToParseBlockquote'
import { tryToParseBulletedList } from './tryToParseBulletedList'
import { tryToParseCodeBlock } from './tryToParseCodeBlock'
import { tryToParseDescriptionList } from './tryToParseDescriptionList'
import { tryToParseHeading } from './tryToParseHeading'
import { tryToParseNumberedList } from './tryToParseNumberedList'
import { tryToParseRevealableBlock } from './tryToParseRevealableBlock'
import { tryToParseTable } from './tryToParseTable'
import { tryToParseThematicBreakStreak } from './tryToParseThematicBreakStreak'



// This includes every outline convention.
const OUTLINE_CONVENTION_PARSERS = [
  tryToParseBlankLineSeparation,
  tryToParseBulletedList,
  tryToParseNumberedList,
  tryToParseHeading,
  tryToParseThematicBreakStreak,
  tryToParseCodeBlock,
  tryToParseBlockquote,
  tryToParseTable,
  tryToParseRevealableBlock,
  tryToParseDescriptionList,
  parseParagraphOrLineBlock
]


export function getOutlineSyntaxNodes(
  args: {
    markupLines: string[],
    sourceLineNumber: number,
    headingLeveler: HeadingLeveler,
    settings: NormalizedSettings.Parsing
  }
): OutlineSyntaxNode[] {
  const { markupLines, headingLeveler, settings } = args

  // Alright! Our first task is to strip away any leading and trailing lines that
  // are entirely blank. These outer blank lines carry no semantic significance,
  // and we don't want them to be inadvertently parsed as thematic breaks.
  //
  // For what it's worth, we *do* allow thematic breaks at the beginning and end of
  // the document, but they must deliberately be produced from streaks rather than
  // from blank lines. For example:
  //
  // #~#~#~#~#~#~#~#~#~#~#~#~#~#~#~#~#~#~#~#~#~#~#

  let firstIndexOfNonBlankLine = 0
  for (; firstIndexOfNonBlankLine < markupLines.length; firstIndexOfNonBlankLine++) {
    if (NON_BLANK_PATTERN.test(markupLines[firstIndexOfNonBlankLine])) {
      break
    }
  }

  const initialSourceLineNumber =
    args.sourceLineNumber + firstIndexOfNonBlankLine

  let lastIndexOfNonBlankLine = markupLines.length - 1
  for (; lastIndexOfNonBlankLine >= 0; lastIndexOfNonBlankLine--) {
    if (NON_BLANK_PATTERN.test(markupLines[lastIndexOfNonBlankLine])) {
      break
    }
  }

  const markupWithoutOuterBlankLines =
    markupLines.slice(firstIndexOfNonBlankLine, lastIndexOfNonBlankLine + 1)

  const markupLineConsumer = new LineConsumer(markupWithoutOuterBlankLines)
  const nodes: OutlineSyntaxNode[] = []

  // We're all set up. Here's our main parser loop!
  while (!markupLineConsumer.done()) {
    const sourceLineNumber =
      initialSourceLineNumber + markupLineConsumer.countLinesConsumed()

    const outlineParserArgs = {
      markupLines: markupLineConsumer.remaining(),
      sourceLineNumber,<commit_msg>Use existing type for signatures<commit_after>import { last } from '../../CollectionHelpers'
import { NON_BLANK_PATTERN } from '../../Patterns'
import { OutlineSyntaxNode } from '../../SyntaxNodes/OutlineSyntaxNode'
import { ThematicBreak } from '../../SyntaxNodes/ThematicBreak'
import { LineConsumer } from './LineConsumer'
import { parseParagraphOrLineBlock } from './parseParagraphOrLineBlock'
import { tryToParseBlankLineSeparation } from './tryToParseBlankLineSeparation'
import { tryToParseBlockquote } from './tryToParseBlockquote'
import { tryToParseBulletedList } from './tryToParseBulletedList'
import { tryToParseCodeBlock } from './tryToParseCodeBlock'
import { tryToParseDescriptionList } from './tryToParseDescriptionList'
import { tryToParseHeading } from './tryToParseHeading'
import { tryToParseNumberedList } from './tryToParseNumberedList'
import { tryToParseRevealableBlock } from './tryToParseRevealableBlock'
import { tryToParseTable } from './tryToParseTable'
import { tryToParseThematicBreakStreak } from './tryToParseThematicBreakStreak'
import { OutlineParser } from './OutlineParser'


// This includes every outline convention.
const OUTLINE_CONVENTION_PARSERS = [
  tryToParseBlankLineSeparation,
  tryToParseBulletedList,
  tryToParseNumberedList,
  tryToParseHeading,
  tryToParseThematicBreakStreak,
  tryToParseCodeBlock,
  tryToParseBlockquote,
  tryToParseTable,
  tryToParseRevealableBlock,
  tryToParseDescriptionList,
  parseParagraphOrLineBlock
]


export function getOutlineSyntaxNodes(args: OutlineParser.Args): OutlineSyntaxNode[] {
  const { markupLines, headingLeveler, settings } = args

  // Alright! Our first task is to strip away any leading and trailing lines that
  // are entirely blank. These outer blank lines carry no semantic significance,
  // and we don't want them to be inadvertently parsed as thematic breaks.
  //
  // For what it's worth, we *do* allow thematic breaks at the beginning and end of
  // the document, but they must deliberately be produced from streaks rather than
  // from blank lines. For example:
  //
  // #~#~#~#~#~#~#~#~#~#~#~#~#~#~#~#~#~#~#~#~#~#~#

  let firstIndexOfNonBlankLine = 0
  for (; firstIndexOfNonBlankLine < markupLines.length; firstIndexOfNonBlankLine++) {
    if (NON_BLANK_PATTERN.test(markupLines[firstIndexOfNonBlankLine])) {
      break
    }
  }

  const initialSourceLineNumber =
    args.sourceLineNumber + firstIndexOfNonBlankLine

  let lastIndexOfNonBlankLine = markupLines.length - 1
  for (; lastIndexOfNonBlankLine >= 0; lastIndexOfNonBlankLine--) {
    if (NON_BLANK_PATTERN.test(markupLines[lastIndexOfNonBlankLine])) {
      break
    }
  }

  const markupWithoutOuterBlankLines =
    markupLines.slice(firstIndexOfNonBlankLine, lastIndexOfNonBlankLine + 1)

  const markupLineConsumer = new LineConsumer(markupWithoutOuterBlankLines)
  const nodes: OutlineSyntaxNode[] = []

  // We're all set up. Here's our main parser loop!
  while (!markupLineConsumer.done()) {
    const sourceLineNumber =
      initialSourceLineNumber + markupLineConsumer.countLinesConsumed()

    const outlineParserArgs: OutlineParser.Args = {
      markupLines: markupLineConsumer.remaining(),
      sourceLineNumber,
{'fraction_non_alphanumeric': 0.0986772084537023, 'fraction_numerical': 0.0012163600425726016, 'mean_word_length': 4.888988361683079, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2401809', 'n_tokens_mistral': 2017, 'n_tokens_neox': 1962, 'n_words': 460}