<commit_before>import { InlineSyntaxNode } from '../../SyntaxNodes/InlineSyntaxNode'
import { EmphasisNode } from '../../SyntaxNodes/EmphasisNode'
import { PlainTextNode } from '../../SyntaxNodes/PlainTextNode'
import { Convention } from './Convention'
import { Sandwich } from './Sandwich'
import { TextConsumer } from '../TextConsumer'
import { last, lastChar, swap } from '../CollectionHelpers'
import { Token, TokenMeaning } from './Token'
import { FailureTracker } from './FailureTracker'
import { applyBackslashEscaping } from '../TextHelpers'
import { STRESS, EMPHASIS, REVISION_DELETION, REVISION_INSERTION, SPOILER, INLINE_ASIDE } from './Sandwiches'

const POTENTIAL_RAISED_VOICE_TOKEN_MEANINGS = [
    TokenMeaning.PotentialRaisedVoiceStart,
    TokenMeaning.PotentialRaisedVoiceEnd,
    TokenMeaning.PotentialRaisedVoiceStartOrEnd
  ]
  
export function applyRaisedVoices(tokens: Token[]): Token[] {
  const intentions = getEmptyIntentions(tokens)
  return applyIntentions(intentions, tokens)
}

function getEmptyIntentions(tokens: Token[]): RaisedVoiceTokenIntention[] {
    const intents: RaisedVoiceTokenIntention[] = []
  
  for (let tokenIndex = 0; tokenIndex < tokens.length; tokenIndex++) {
    const token = tokens[tokenIndex]
    const tokenMeaning = tokens[tokenIndex].meaning
    
    if (-1 !== POTENTIAL_RAISED_VOICE_TOKEN_MEANINGS.indexOf(token.meaning)) {
      intents.push(new RaisedVoiceTokenIntention(tokenIndex, token))
    }
  }
  
  return intents
}

function applyIntentions(intentions: RaisedVoiceTokenIntention[], tokens: Token[]): Token[] {
  // We could probably be naughty and modify the `tokens` collection directly.
  const resultTokens = tokens.slice()
  
  for (const intention of intentions.sort(compareIntentionsDecending)) {
    tokens.splice(intention.originalTokenIndex, 1, ...intention.finalTokens())
  }
  
  return resultTokens
}

class RaisedVoiceTokenIntention {
  constructor(public originalTokenIndex: number, private originalToken: Token) { }
  
  finalTokens(): Token[] {
    return [new Token(TokenMeaning.PlainText, this.originalToken.value)]
  }
}

function compareIntentionsDecending(a: RaisedVoiceTokenIntention, b: RaisedVoiceTokenIntention): number {
  return b.originalTokenIndex - a.originalTokenIndex
}<commit_msg>Flesh out RaisedVoiceTokenIntention<commit_after>import { InlineSyntaxNode } from '../../SyntaxNodes/InlineSyntaxNode'
import { EmphasisNode } from '../../SyntaxNodes/EmphasisNode'
import { PlainTextNode } from '../../SyntaxNodes/PlainTextNode'
import { Convention } from './Convention'
import { Sandwich } from './Sandwich'
import { TextConsumer } from '../TextConsumer'
import { last, lastChar, swap } from '../CollectionHelpers'
import { Token, TokenMeaning } from './Token'
import { FailureTracker } from './FailureTracker'
import { applyBackslashEscaping } from '../TextHelpers'
import { STRESS, EMPHASIS, REVISION_DELETION, REVISION_INSERTION, SPOILER, INLINE_ASIDE } from './Sandwiches'


export function applyRaisedVoices(tokens: Token[]): Token[] {
  const intentions = getEmptyIntentions(tokens)
  return applyIntentions(intentions, tokens)
}

const POTENTIAL_RAISED_VOICE_TOKEN_MEANINGS = [
    TokenMeaning.PotentialRaisedVoiceStart,
    TokenMeaning.PotentialRaisedVoiceEnd,
    TokenMeaning.PotentialRaisedVoiceStartOrEnd
  ]

function getEmptyIntentions(tokens: Token[]): RaisedVoiceTokenIntention[] {
    const intents: RaisedVoiceTokenIntention[] = []
  
  for (let tokenIndex = 0; tokenIndex < tokens.length; tokenIndex++) {
    const token = tokens[tokenIndex]
    const tokenMeaning = tokens[tokenIndex].meaning
    
    if (-1 !== POTENTIAL_RAISED_VOICE_TOKEN_MEANINGS.indexOf(token.meaning)) {
      intents.push(new RaisedVoiceTokenIntention(tokenIndex, token))
    }
  }
  
  return intents
}


function applyIntentions(intentions: RaisedVoiceTokenIntention[], tokens: Token[]): Token[] {
  // We could probably be naughty and modify the `tokens` collection directly.
  const resultTokens = tokens.slice()
  
  for (const intention of intentions.sort(compareIntentionsDecending)) {
    tokens.splice(intention.originalTokenIndex, 1, ...intention.finalTokens())
  }
  
  return resultTokens
}


class RaisedVoiceTokenIntention {
  private startTokenMeanings: TokenMeaning[] = []
  private endTokenMeanings: TokenMeaning[] = []
  
  constructor(public originalTokenIndex: number, private originalToken: Token) { }
  
  finalTokens(): Token[] {
    if (this.startTokenMeanings.length && this.endTokenMeanings.length) {
      throw new Error(`Delimiter serving multiple roles at index ${this.originalTokenIndex}`)
    }
    
    if (this.startTokenMeanings.length) {
      // We indicate the intent to end emphasis/stress conventions in order, which means we're implicitly
      // indicating the intent to start emphasis/stress in reverse order 
      return this.startTokenMeanings.reverse().map(toToken)
    }
    
    if (this.endTokenMeanings.length) { 
      return this.endTokenMeanings.map(toToken)
    }
    
    
    return [new Token(TokenMeaning.PlainText, this.originalToken.value)]
  }
  
  endEmphasis(startDelimiterIntention: RaisedVoiceTokenIntention): void {
    this.endTokenMeanings.push(TokenMeaning.EmphasisEnd)
    
    startDelimiterIntention.startEmphasis()
  }
  
  endStress(startDelimiterIntention: RaisedVoiceTokenIntention): void {
    this.endTokenMeanings.push(TokenMeaning.StressEnd)
    
    startDelimiterIntention.startStress()
  }
  
  private startEmphasis(): void {
    this.startTokenMeanings.push(TokenMeaning.EmphasisStart)
  }
  
  private startStress() {
    this.startTokenMeanings.push(TokenMeaning.StressStart)
  }
}


function compareIntentionsDecending(a: RaisedVoiceTokenIntention, b: RaisedVoiceTokenIntention): number {
  return b.originalTokenIndex - a.originalTokenIndex
}


function toToken(meaning: TokenMeaning): Token {
  return new Token(meaning)
}
{'fraction_non_alphanumeric': 0.10357082416652563, 'fraction_numerical': 0.001015400236926722, 'mean_word_length': 4.726744186046512, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '397890', 'n_tokens_mistral': 1834, 'n_tokens_neox': 1747, 'n_words': 373}