<commit_before>import { LINK_CONVENTION, STRESS_CONVENTION, EMPHASIS_CONVENTION, REVISION_DELETION_CONVENTION, REVISION_INSERTION_CONVENTION, SPOILER_CONVENTION, NSFW_CONVENTION, NSFL_CONVENTION, FOOTNOTE_CONVENTION, PARENTHESIZED_CONVENTION, SQUARE_BRACKETED_CONVENTION, ACTION_CONVENTION } from '../RichConventions'
import { RichConvention } from '../RichConvention'
import { Token } from './Token'


// Conventions can overlap, which makes it painful to produce an abstract syntax tree. This function rearranges
// and adds tokens to make that process simpler.
//
// Overlapping conventions are split into multiple pieces to ensure each piece has just a single parent.
export function nestOverlappingConventions(tokens: Token[]): Token[] {
  return new ConventionNester(tokens).tokens
}

// We're always okay with splitting these conventions.
const FREELY_SPLITTABLE_CONVENTIONS: RichConvention[] = [
  REVISION_DELETION_CONVENTION,
  REVISION_INSERTION_CONVENTION,
  STRESS_CONVENTION,
  EMPHASIS_CONVENTION,
  PARENTHESIZED_CONVENTION,
  SQUARE_BRACKETED_CONVENTION
]

// We avoid splitting these conventions.
//
// The order is important: We'd rather split a link than a spoiler, and we'll never split a footnote.
const CONVENTIONS_TO_AVOID_SPLITTING_FROM_LEAST_TO_MOST_IMPORTANT = [
  LINK_CONVENTION,
  ACTION_CONVENTION,
  SPOILER_CONVENTION,
  NSFW_CONVENTION,
  NSFL_CONVENTION,
  FOOTNOTE_CONVENTION
]


class ConventionNester {
  constructor(public tokens: Token[]) {
    this.tokens = tokens.slice()

    const splittableConventions =
      FREELY_SPLITTABLE_CONVENTIONS.slice()

    this.splitConventionsThatStartInsideAnotherConventionAndEndAfter(
      splittableConventions)

    for (const conventionNotToSplit of CONVENTIONS_TO_AVOID_SPLITTING_FROM_LEAST_TO_MOST_IMPORTANT) {
      this.resolveOverlapping(splittableConventions, conventionNotToSplit)

      // We'd rather split the current convention than the ones that follow. 
      splittableConventions.push(conventionNotToSplit)
    }
  }

  // This method assumes no tokens are missing.
  splitConventionsThatStartInsideAnotherConventionAndEndAfter(conventions: RichConvention[]): void {
    const unclosedStartTokens: Token[] = []

    // Here's our overall strategy:
    //
    // We'll to traverse our token array, keeping track of which conventions are still unclosed as of the current
    // position. Once we reach a convention end token, we check whether any still-unclosed conventions were opened
    // inside the current convention. If so, we essentially chop those conventions in half: We add end tokens
    // immediately before the end of the current convention, and we add corresponding start tokens immediately
    // after the end of the current convention.
    for (let tokenIndex = 0; tokenIndex < this.tokens.length; tokenIndex++) {
      const token = this.tokens[tokenIndex]

      if (doesTokenStartConvention(token, conventions)) {
        unclosedStartTokens.push(token)
        continue
      }

      if (!doesTokenEndConvention(token, conventions)) {
        continue
      }

      // Alright, we've found a token that closes one of our unclosed start tokens. If any conventions were opened
      // between this end token and its corresponding start token, those conventions (overlapping this one) will
      // be chopped in half.
      const endToken = token
      
      // Why do we store a collection of end tokens rather than a collection of conventions?
      //
      // Link end tokens have a URL that needs to be copied when links are split in half. Right now, links aren't 
      // split using this method (and none of the conventions split using this method have any values in their end
      // tokens), but this method uses the same helper method used to split links.
      let endTokensOfOverlappingConventions: Token[] = []

      // We'll check the unclosed start tokens from most recently opened to least recently opened.
      for (let i = unclosedStartTokens.length - 1; i >= 0; i--) {
        const unclosedStartToken = unclosedStartTokens[i]

        if (unclosedStartToken.correspondsToToken === endToken) {
          // Hooray! We've reached the start token that is closed by the current token.
          unclosedStartTokens.splice(i, 1)

          // Any conventions opened before this one don't overlap with the current convention, so we can bail.
          break
        }

        endTokensOfOverlappingConventions.push(unclosedStartToken.correspondsToToken)
      }

      // Okay, now we know which conventions overlap the one that's about to close. To preserve overlapping
      // while making it easier to produce an abstract syntax tree, we make the following changes:
      //
      // 1. Just before the end token of the current convention, we add a closing token for each unclosed
      //    convention. To preserve proper nesting, we close the conventions in order of most to least recent.
      //  
      // 2. Just after the end token of the current convention, we add a start token for each unclosed convention.
      //    To avoid producing a surprising syntax tree, we re-open the conventions in their original order.
      this.closeAndReopenConventionsAroundTokenAtIndex(tokenIndex, endTokensOfOverlappingConventions)

      const countOverlapping = endTokensOfOverlappingConventions.length

      // Advance index to reflect the fact that we just added tokens 
      tokenIndex += (2 * countOverlapping)
    }
  }

  // This method assumes that any `conventionsToSplit` tokens are already properly nested within each other.
  private resolveOverlapping(splittableConventions: RichConvention[], conventionNotToSplit: RichConvention): void {

    // To keep local variable names shorter, we'll refer to `conventionNotToSplit` as the hero convention.
<commit_msg>Clarify advancement of token index<commit_after>import { LINK_CONVENTION, STRESS_CONVENTION, EMPHASIS_CONVENTION, REVISION_DELETION_CONVENTION, REVISION_INSERTION_CONVENTION, SPOILER_CONVENTION, NSFW_CONVENTION, NSFL_CONVENTION, FOOTNOTE_CONVENTION, PARENTHESIZED_CONVENTION, SQUARE_BRACKETED_CONVENTION, ACTION_CONVENTION } from '../RichConventions'
import { RichConvention } from '../RichConvention'
import { Token } from './Token'


// Conventions can overlap, which makes it painful to produce an abstract syntax tree. This function rearranges
// and adds tokens to make that process simpler.
//
// Overlapping conventions are split into multiple pieces to ensure each piece has just a single parent.
export function nestOverlappingConventions(tokens: Token[]): Token[] {
  return new ConventionNester(tokens).tokens
}

// We're always okay with splitting these conventions.
const FREELY_SPLITTABLE_CONVENTIONS: RichConvention[] = [
  REVISION_DELETION_CONVENTION,
  REVISION_INSERTION_CONVENTION,
  STRESS_CONVENTION,
  EMPHASIS_CONVENTION,
  PARENTHESIZED_CONVENTION,
  SQUARE_BRACKETED_CONVENTION
]

// We avoid splitting these conventions.
//
// The order is important: We'd rather split a link than a spoiler, and we'll never split a footnote.
const CONVENTIONS_TO_AVOID_SPLITTING_FROM_LEAST_TO_MOST_IMPORTANT = [
  LINK_CONVENTION,
  ACTION_CONVENTION,
  SPOILER_CONVENTION,
  NSFW_CONVENTION,
  NSFL_CONVENTION,
  FOOTNOTE_CONVENTION
]


class ConventionNester {
  constructor(public tokens: Token[]) {
    this.tokens = tokens.slice()

    const splittableConventions =
      FREELY_SPLITTABLE_CONVENTIONS.slice()

    this.splitConventionsThatStartInsideAnotherConventionAndEndAfter(
      splittableConventions)

    for (const conventionNotToSplit of CONVENTIONS_TO_AVOID_SPLITTING_FROM_LEAST_TO_MOST_IMPORTANT) {
      this.resolveOverlapping(splittableConventions, conventionNotToSplit)

      // We'd rather split the current convention than the ones that follow. 
      splittableConventions.push(conventionNotToSplit)
    }
  }

  // This method assumes no tokens are missing.
  splitConventionsThatStartInsideAnotherConventionAndEndAfter(conventions: RichConvention[]): void {
    const unclosedStartTokens: Token[] = []

    // Here's our overall strategy:
    //
    // We'll to traverse our token array, keeping track of which conventions are still unclosed as of the current
    // position. Once we reach a convention end token, we check whether any still-unclosed conventions were opened
    // inside the current convention. If so, we essentially chop those conventions in half: We add end tokens
    // immediately before the end of the current convention, and we add corresponding start tokens immediately
    // after the end of the current convention.
    for (let tokenIndex = 0; tokenIndex < this.tokens.length; tokenIndex++) {
      const token = this.tokens[tokenIndex]

      if (doesTokenStartConvention(token, conventions)) {
        unclosedStartTokens.push(token)
        continue
      }

      if (!doesTokenEndConvention(token, conventions)) {
        continue
      }

      // Alright, we've found a token that closes one of our unclosed start tokens. If any conventions were opened
      // between this end token and its corresponding start token, those conventions (overlapping this one) will
      // be chopped in half.
      const endToken = token
      
      // Why do we store a collection of end tokens rather than a collection of conventions?
      //
      // Link end tokens have a URL that needs to be copied when links are split in half. Right now, links aren't 
      // split using this method (and none of the conventions split using this method have any values in their end
      // tokens), but this method uses the same helper method used to split links.
      let endTokensOfOverlappingConventions: Token[] = []

      // We'll check the unclosed start tokens from most recently opened to least recently opened.
      for (let i = unclosedStartTokens.length - 1; i >= 0; i--) {
        const unclosedStartToken = unclosedStartTokens[i]

        if (unclosedStartToken.correspondsToToken === endToken) {
          // Hooray! We've reached the start token that is closed by the current token.
          unclosedStartTokens.splice(i, 1)

          // Any conventions opened before this one don't overlap with the current convention, so we can bail.
          break
        }

        endTokensOfOverlappingConventions.push(unclosedStartToken.correspondsToToken)
      }

      // Okay, now we know which conventions overlap the one that's about to close. To preserve overlapping
      // while making it easier to produce an abstract syntax tree, we make the following changes:
      //
      // 1. Just before the end token of the current convention, we add a closing token for each unclosed
      //    convention. To preserve proper nesting, we close the conventions in order of most to least recent.
      //  
      // 2. Just after the end token of the current convention, we add a start token for each unclosed convention.
      //    To avoid producing a surprising syntax tree, we re-open the conventions in their original order.
      this.closeAndReopenConventionsAroundTokenAtIndex(tokenIndex, endTokensOfOverlappingConventions)

      const countOverlapping = endTokensOfOverlappingConventions.length

      // Advance index to reflect the fact that we just added tokens. We aded `countOverlapping` before `endToken`,
      // and `countOverlapping` start tokens before `endToken`.
      tokenIndex += (2 * countOverlapping)
    }
  }

  // This method assumes that any `conventionsToSplit` tokens are already properly nested within each other.
  private resolveOverlapping(splittableConventions: RichConvention[], conventionNotToSplit: RichConvention): void {

    // To keep local variable names shorter, we'll refer to `conventionNotToSplit` as the hero convention.

{'fraction_non_alphanumeric': 0.061633936920853526, 'fraction_numerical': 0.0011901725750233785, 'mean_word_length': 3.6242138364779874, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '416092', 'n_tokens_mistral': 3319, 'n_tokens_neox': 3122, 'n_words': 1179}