<commit_before>package org.allenai.common

import scala.language.implicitConversions

class AI2String(string: String) {
  def replaceWeirdChars(): String =
    AI2String.unicodeCharMap.foldLeft(string) {
      case (s, (unicodeChar, replacement)) => s.replace(unicodeChar.toString, replacement)
    }
}

object AI2String {
  /** Maps weird unicode characters to ASCII equivalents */
  val unicodeCharMap = Map(
    '\u00AB' -> "\"",
    '\u00AD' -> "-",<commit_msg>Added source attribution<commit_after>package org.allenai.common

import scala.language.implicitConversions

class AI2String(string: String) {
  def replaceWeirdChars(): String =
    AI2String.unicodeCharMap.foldLeft(string) {
      case (s, (unicodeChar, replacement)) => s.replace(unicodeChar.toString, replacement)
    }
}

object AI2String {
  /** Maps weird unicode characters to ASCII equivalents
    * This list comes from http://lexsrv3.nlm.nih.gov/LexSysGroup/Projects/lvg/current/docs/designDoc/UDF/unicode/DefaultTables/symbolTable.html */
  val unicodeCharMap = Map(
    '\u00AB' -> "\"",
    '\u00AD' -> "-",
{'fraction_non_alphanumeric': 0.13075506445672191, 'fraction_numerical': 0.013812154696132596, 'mean_word_length': 4.151658767772512, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3829939', 'n_tokens_mistral': 364, 'n_tokens_neox': 338, 'n_words': 69}