<commit_before>import type { MarkupNode } from '../typings/markup'

import parse5 = require('parse5/lib/parser')
const htmlParser = new parse5({ sourceCodeLocationInfo: true })

import { titleCased } from '../title-case'

import charCategories = require('xregexp/tools/output/categories')

const re = {
  Nl: charCategories.find(cat => cat.alias === 'Letter_Number').bmp,
  Nd: charCategories.find(cat => cat.alias === 'Decimal_Number').bmp,
  Mn: charCategories.find(cat => cat.alias === 'Nonspacing_Mark').bmp,
  Mc: charCategories.find(cat => cat.alias === 'Spacing_Mark').bmp,
  Lu: charCategories.find(cat => cat.alias === 'Uppercase_Letter').bmp,
  Lt: charCategories.find(cat => cat.alias === 'Titlecase_Letter').bmp,
  Ll: charCategories.find(cat => cat.alias === 'Lowercase_Letter').bmp,
  Lm: charCategories.find(cat => cat.alias === 'Modifier_Letter').bmp,
  Lo: charCategories.find(cat => cat.alias === 'Other_Letter').bmp,

  // punctuation
  P: /\.\u002D\u2000-\u206F\u2E00-\u2E7F\\'!"#\$%&\(\)\*\+,\/:;<=>\?@\[\]^_`{\|}~/.source,
  // P: charCategories.find(cat => cat.alias === 'Punctuation').bmp,

  Whitespace: / \t\n\r\u00A0/.source,

  // calculated below
  lcChar: null,
  char: null,
  L: null,
  protectedWord: null,

  leadingUnprotectedWord: null,
  protectedWords: null,
  unprotectedWord: null,
  url: null,
  whitespace: null,
  sentenceEnd: /^[:?]/,
}

// eslint-disable-next-line @typescript-eslint/restrict-plus-operands
re.lcChar = re.Ll + re.Lt + re.Lm + re.Lo + re.Mn + re.Mc + re.Nd + re.Nl
// eslint-disable-next-line @typescript-eslint/restrict-plus-operands
re.char = re.Lu + re.lcChar
re.L = `${re.Lu}${re.Ll}${re.Lt}${re.Lm}${re.Lo}`
re.protectedWord = `[${re.lcChar}]*[${re.Lu}][${re.char}]*`

/* actual regexps */

/* TODO: add punctuation */
re.leadingUnprotectedWord = new RegExp(`^([${re.Lu}][${re.lcChar}]*)[${re.Whitespace}${re.P}]`)
re.protectedWords = new RegExp(`^(${re.protectedWord})(([${re.Whitespace}])(${re.protectedWord}))*`)
re.unprotectedWord = new RegExp(`^[${re.char}]+`)
re.url = /^(https?|mailto):\/\/[^\s]+/
re.whitespace = new RegExp(`^[${re.Whitespace}]+`)

/* eslint-disable quote-props */
const ligatures = {
  '\u01F1': 'DZ',
  '\u01F2': 'Dz',
  '\u01F3': 'dz',
  '\u01C4': 'D\u017D',
  '\u01C5': 'D\u017E',
  '\u01C6': 'd\u017E',
  '\uFB00': 'ff',
  '\uFB01': 'fi',
  '\uFB02': 'fl',
  '\uFB03': 'ffi',
  '\uFB04': 'ffl',
  '\uFB05': '\u017Ft',
  '\uFB06': 'st',
  '\u0132': 'IJ',
  '\u0133': 'ij',
  '\u01C7': 'LJ',
  '\u01C8': 'Lj',
  '\u01C9': 'lj',
  '\u01CA': 'NJ',
  '\u01CB': 'Nj',
  '\u01CC': 'nj',
}
/* eslint-enable */


const titleCaseKeep = new RegExp(`(?:(?:(?:[:?]? +)|^)[${re.L}][${re.P}]?(?:[${re.Whitespace}]|$))|(?:(?:<span class="nocase">.*?</span>)|(?:<nc>.*?</nc>))`, 'gi')
const singleLetter = /^([:?])? +(.)/

export function titleCase(text: string): string {
  // let titlecased = CSL.Output.Formatters.title(new State, text)
  let titlecased: string = titleCased(text)

  let m
  // restore single-letter "words". Shame firefox doesn't do lookbehind, but this will work
  text.replace(titleCaseKeep, (match: string, offset: number) => {
    if (match[0] !== '<' && (m = match.match(singleLetter)) && (m[1] || m[2] !== 'a')) match = match.toUpperCase()
    titlecased = titlecased.substr(0, offset) + match + titlecased.substr(offset + match.length)
    return match
  })<commit_msg>caser<commit_after>import type { MarkupNode } from '../typings/markup'

import parse5 = require('parse5/lib/parser')
const htmlParser = new parse5({ sourceCodeLocationInfo: true })

import { titleCased } from '../title-case'

import charCategories = require('xregexp/tools/output/categories')

const re = {
  Nl: charCategories.find(cat => cat.alias === 'Letter_Number').bmp,
  Nd: charCategories.find(cat => cat.alias === 'Decimal_Number').bmp,
  Mn: charCategories.find(cat => cat.alias === 'Nonspacing_Mark').bmp,
  Mc: charCategories.find(cat => cat.alias === 'Spacing_Mark').bmp,
  Lu: charCategories.find(cat => cat.alias === 'Uppercase_Letter').bmp,
  Lt: charCategories.find(cat => cat.alias === 'Titlecase_Letter').bmp,
  Ll: charCategories.find(cat => cat.alias === 'Lowercase_Letter').bmp,
  Lm: charCategories.find(cat => cat.alias === 'Modifier_Letter').bmp,
  Lo: charCategories.find(cat => cat.alias === 'Other_Letter').bmp,

  // punctuation
  P: /\.\u002D\u2000-\u206F\u2E00-\u2E7F\\'!"#\$%&\(\)\*\+,\/:;<=>\?@\[\]^_`{\|}~/.source,
  // P: charCategories.find(cat => cat.alias === 'Punctuation').bmp,

  Whitespace: / \t\n\r\u00A0/.source,

  // calculated below
  lcChar: null,
  char: null,
  L: null,
  protectedWord: null,

  leadingUnprotectedWord: null,
  protectedWords: null,
  unprotectedWord: null,
  url: null,
  whitespace: null,
  sentenceEnd: /^[:?]/,
}

// eslint-disable-next-line @typescript-eslint/restrict-plus-operands
re.lcChar = re.Ll + re.Lt + re.Lm + re.Lo + re.Mn + re.Mc + re.Nd + re.Nl
// eslint-disable-next-line @typescript-eslint/restrict-plus-operands
re.char = re.Lu + re.lcChar
re.L = `${re.Lu}${re.Ll}${re.Lt}${re.Lm}${re.Lo}`
re.protectedWord = `[${re.lcChar}]*[${re.Lu}][${re.char}]*`

/* actual regexps */

/* TODO: add punctuation */
re.leadingUnprotectedWord = new RegExp(`^([${re.Lu}][${re.lcChar}]*)[${re.Whitespace}${re.P}]`)
re.protectedWords = new RegExp(`^(${re.protectedWord})(([${re.Whitespace}])(${re.protectedWord}))*`)
re.unprotectedWord = new RegExp(`^[${re.char}]+`)
re.url = /^(https?|mailto):\/\/[^\s]+/
re.whitespace = new RegExp(`^[${re.Whitespace}]+`)

/* eslint-disable quote-props */
const ligatures = {
  '\u01F1': 'DZ',
  '\u01F2': 'Dz',
  '\u01F3': 'dz',
  '\u01C4': 'D\u017D',
  '\u01C5': 'D\u017E',
  '\u01C6': 'd\u017E',
  '\uFB00': 'ff',
  '\uFB01': 'fi',
  '\uFB02': 'fl',
  '\uFB03': 'ffi',
  '\uFB04': 'ffl',
  '\uFB05': '\u017Ft',
  '\uFB06': 'st',
  '\u0132': 'IJ',
  '\u0133': 'ij',
  '\u01C7': 'LJ',
  '\u01C8': 'Lj',
  '\u01C9': 'lj',
  '\u01CA': 'NJ',
  '\u01CB': 'Nj',
  '\u01CC': 'nj',
}
/* eslint-enable */


const titleCaseKeep = new RegExp(`(?:(?:[:?]?[${re.Whitespace}]+)[${re.L}][${re.P}]?(?:[${re.Whitespace}]|$))|(?:(?:<span class="nocase">.*?</span>)|(?:<nc>.*?</nc>))`, 'gi')
const singleLetter = new RegExp(`^([:?])?[${re.Whitespace}]+(.)`)

export function titleCase(text: string): string {
  // let titlecased = CSL.Output.Formatters.title(new State, text)
  let titlecased: string = titleCased(text)

  // restore single-letter "words". Shame firefox doesn't do lookbehind, but this will work
  text.replace(titleCaseKeep, (match: string, offset: number) => {
    if (match[0] !== '<') {
      const [ , punc, l ] = match.match(singleLetter)
      if (punc && (l === 'a' || l === 'A')) {
        match = match.toUpperCase()
      }
    }
    titlecased = titlecased.substr(0, offset) + match + titlecased.substr(offset + match.length)
    return match
  })
{'fraction_non_alphanumeric': 0.24813569235268315, 'fraction_numerical': 0.02661207778915046, 'mean_word_length': 4.051698670605613, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2919462', 'n_tokens_mistral': 2918, 'n_tokens_neox': 2754, 'n_words': 472}