<commit_before>import { TextConsumer } from '../TextConsumer'
import { HeadingNode } from '../../SyntaxNodes/HeadingNode'
import { OutlineParser, OutlineParserArgs, } from './OutlineParser'
import { either, NON_BLANK, STREAK } from './Patterns'
import { parseInlineConventions } from '../Inline/ParseInlineConventions'
import { HeadingLeveler, isUnderlineConsistentWithOverline} from './HeadingLeveler'

const NON_BLANK_PATTERN = new RegExp(
  NON_BLANK
)

const STREAK_PATTERN = new RegExp(
  STREAK
)

// Underlined text is treated as a heading. Headings can have an optional overline, too.
export function getHeadingParser(headingLeveler: HeadingLeveler): OutlineParser {
  
  return function parseHeading(args: OutlineParserArgs): boolean {
    const consumer = new TextConsumer(args.text)

    // First, let's parse the optional overline.
    let optionalOverline: string

    consumer.consumeLineIfMatches({
      pattern: STREAK_PATTERN,
      then: (line) => optionalOverline = line
    })
  
    // Next, let's parse the content and underline.
    //
    // The content must not be a streak! Why not? Take a look:
    //
    // =============================================
    // #~#~#~#~#~#~#~#~#~#~#~#~#~#~#~#~#~#~#~#~#~#~#
    // =============================================
    //
    // The author almost certainly included those lines to separate sections, not as a heading.
    //
    // We wouldn't need to care about this if we could parse separator streaks before headings,
    // but that would introduce its own complications: The separator streak parser would always
    // want to consume every heading's overline.
    let content: string
    let underline: string

    const hasContentAndUnderline = (
      // Parse the content
      consumer.consumeLine({
        if: (line) => NON_BLANK_PATTERN.test(line) && !STREAK_PATTERN.test(line),
        then: (line) => content = line
      })
      
      // Parse the underline
      && consumer.consumeLine({
        if: (line) => (
          STREAK_PATTERN.test(line)
          && isUnderlineConsistentWithOverline(optionalOverline, line)),
        then: (line) => underline = line
      })
    )

    if (!hasContentAndUnderline) {
      return false
    }

    const headingLevel = headingLeveler.registerUnderlineAndGetLevel(underline)

    parseInlineConventions(content, { parentNode: new HeadingNode(null, headingLevel) },
      (inlineNodes, countCharsParsed, headingNode) => {
        headingNode.addChildren(inlineNodes)
        args.then([headingNode], consumer.lengthConsumed())
      })
<commit_msg>Use getInlineNodes in heading parser<commit_after>import { TextConsumer } from '../TextConsumer'
import { HeadingNode } from '../../SyntaxNodes/HeadingNode'
import { OutlineParser, OutlineParserArgs, } from './OutlineParser'
import { either, NON_BLANK, STREAK } from './Patterns'
import { getInlineNodes } from '../Inline/GetInlineNodes'
import { HeadingLeveler, isUnderlineConsistentWithOverline} from './HeadingLeveler'

const NON_BLANK_PATTERN = new RegExp(
  NON_BLANK
)

const STREAK_PATTERN = new RegExp(
  STREAK
)

// Underlined text is treated as a heading. Headings can have an optional overline, too.
export function getHeadingParser(headingLeveler: HeadingLeveler): OutlineParser {
  
  return function parseHeading(args: OutlineParserArgs): boolean {
    const consumer = new TextConsumer(args.text)

    // First, let's parse the optional overline.
    let optionalOverline: string

    consumer.consumeLineIfMatches({
      pattern: STREAK_PATTERN,
      then: (line) => optionalOverline = line
    })
  
    // Next, let's parse the content and underline.
    //
    // The content must not be a streak! Why not? Take a look:
    //
    // =============================================
    // #~#~#~#~#~#~#~#~#~#~#~#~#~#~#~#~#~#~#~#~#~#~#
    // =============================================
    //
    // The author almost certainly included those lines to separate sections, not as a heading.
    //
    // We wouldn't need to care about this if we could parse separator streaks before headings,
    // but that would introduce its own complications: The separator streak parser would always
    // want to consume every heading's overline.
    let content: string
    let underline: string

    const hasContentAndUnderline = (
      // Parse the content
      consumer.consumeLine({
        if: (line) => NON_BLANK_PATTERN.test(line) && !STREAK_PATTERN.test(line),
        then: (line) => content = line
      })
      
      // Parse the underline
      && consumer.consumeLine({
        if: (line) => (
          STREAK_PATTERN.test(line)
          && isUnderlineConsistentWithOverline(optionalOverline, line)),
        then: (line) => underline = line
      })
    )

    if (!hasContentAndUnderline) {
      return false
    }

    const headingLevel = headingLeveler.registerUnderlineAndGetLevel(underline)

    args.then(
      [new HeadingNode(getInlineNodes(content), headingLevel)],
      consumer.lengthConsumed())
    
{'fraction_non_alphanumeric': 0.14501390544298767, 'fraction_numerical': 0.0, 'mean_word_length': 3.2561284868977176, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2352103', 'n_tokens_mistral': 1490, 'n_tokens_neox': 1448, 'n_words': 398}