<commit_before>
import string

contentModelFlags = {
    "PCDATA":0,
    "RCDATA":1,
    "CDATA":2,
    "PLAINTEXT":3
}

spaceCharacters = (
    u"\t",
    u"\n",
    u"\u000B",
    u"\u000C",
    u" "
)

entitiesWindows1252 = {
    128: 8364,
    129: 65533,
    130: 8218,
    131: 402,
    132: 8222,
    133: 8230,
    134: 8224,
    135: 8225,
    136: 710,
    137: 8240,
    138: 352,
    139: 8249,
    140: 338,
    141: 65533,
    142: 381,
    143: 65533,
    144: 65533,
    145: 8216,
    146: 8217,
    147: 8220,
    148: 8221,
    149: 8226,
    150: 8211,
    151: 8212,
    152: 732,
    153: 8482,
    154: 353,
    155: 8250,
    156: 339,
    157: 65533,
    158: 382,
    159: 376
}

entities = {
    "AElig": u"\u00C6",
    "Aacute": u"\u00C1",
    "Acirc": u"\u00C2",
    "Agrave": u"\u00C0",
    "Alpha": u"\u0391",
    "Aring": u"\u00C5",
    "Atilde": u"\u00C3",
    "Auml": u"\u00C4",
    "Beta": u"\u0392",
    "Ccedil": u"\u00C7",
    "Chi": u"\u03A7",
    "Dagger": u"\u2021",
    "Delta": u"\u0394",
    "ETH": u"\u00D0",
    "Eacute": u"\u00C9",
    "Ecirc": u"\u00CA",
    "Egrave": u"\u00C8",
    "Epsilon": u"\u0395",
    "Eta": u"\u0397",
    "Euml": u"\u00CB",
    "Gamma": u"\u0393",
    "Iacute": u"\u00CD",
    "Icirc": u"\u00CE",
    "Igrave": u"\u00CC",
    "Iota": u"\u0399",<commit_msg>Added comments to describe each of the CP1252 to Unicode mappings<commit_after>
import string

contentModelFlags = {
    "PCDATA":0,
    "RCDATA":1,
    "CDATA":2,
    "PLAINTEXT":3
}

spaceCharacters = (
    u"\t",
    u"\n",
    u"\u000B",
    u"\u000C",
    u" "
)

entitiesWindows1252 = {
    128: 8364,  # 0x80  0x20AC  EURO SIGN
    129: 65533, # 0x81          UNDEFINED
    130: 8218,  # 0x82  0x201A  SINGLE LOW-9 QUOTATION MARK
    131: 402,   # 0x83  0x0192  LATIN SMALL LETTER F WITH HOOK
    132: 8222,  # 0x84  0x201E  DOUBLE LOW-9 QUOTATION MARK
    133: 8230,  # 0x85  0x2026  HORIZONTAL ELLIPSIS
    134: 8224,  # 0x86  0x2020  DAGGER
    135: 8225,  # 0x87  0x2021  DOUBLE DAGGER
    136: 710,   # 0x88  0x02C6  MODIFIER LETTER CIRCUMFLEX ACCENT
    137: 8240,  # 0x89  0x2030  PER MILLE SIGN
    138: 352,   # 0x8A  0x0160  LATIN CAPITAL LETTER S WITH CARON
    139: 8249,  # 0x8B  0x2039  SINGLE LEFT-POINTING ANGLE QUOTATION MARK
    140: 338,   # 0x8C  0x0152  LATIN CAPITAL LIGATURE OE
    141: 65533, # 0x8D          UNDEFINED
    142: 381,   # 0x8E  0x017D  LATIN CAPITAL LETTER Z WITH CARON
    143: 65533, # 0x8F          UNDEFINED
    144: 65533, # 0x90          UNDEFINED
    145: 8216,  # 0x91  0x2018  LEFT SINGLE QUOTATION MARK
    146: 8217,  # 0x92  0x2019  RIGHT SINGLE QUOTATION MARK
    147: 8220,  # 0x93  0x201C  LEFT DOUBLE QUOTATION MARK
    148: 8221,  # 0x94  0x201D  RIGHT DOUBLE QUOTATION MARK
    149: 8226,  # 0x95  0x2022  BULLET
    150: 8211,  # 0x96  0x2013  EN DASH
    151: 8212,  # 0x97  0x2014  EM DASH
    152: 732,   # 0x98  0x02DC  SMALL TILDE
    153: 8482,  # 0x99  0x2122  TRADE MARK SIGN
    154: 353,   # 0x9A  0x0161  LATIN SMALL LETTER S WITH CARON
    155: 8250,  # 0x9B  0x203A  SINGLE RIGHT-POINTING ANGLE QUOTATION MARK
    156: 339,   # 0x9C  0x0153  LATIN SMALL LIGATURE OE
    157: 65533, # 0x9D          UNDEFINED
    158: 382,   # 0x9E  0x017E  LATIN SMALL LETTER Z WITH CARON
    159: 376    # 0x9F  0x0178  LATIN CAPITAL LETTER Y WITH DIAERESIS
}

entities = {
    "AElig": u"\u00C6",
    "Aacute": u"\u00C1",
    "Acirc": u"\u00C2",
    "Agrave": u"\u00C0",
    "Alpha": u"\u0391",
    "Aring": u"\u00C5",
    "Atilde": u"\u00C3",
    "Auml": u"\u00C4",
    "Beta": u"\u0392",
    "Ccedil": u"\u00C7",
    "Chi": u"\u03A7",
    "Dagger": u"\u2021",
    "Delta": u"\u0394",
    "ETH": u"\u00D0",
    "Eacute": u"\u00C9",
    "Ecirc": u"\u00CA",
    "Egrave": u"\u00C8",
    "Epsilon": u"\u0395",
    "Eta": u"\u0397",
    "Euml": u"\u00CB",
    "Gamma": u"\u0393",
    "Iacute": u"\u00CD",
    "Icirc": u"\u00CE",
    "Igrave": u"\u00CC",
    "Iota": u"\u0399",
{'fraction_non_alphanumeric': 0.15276027224602975, 'fraction_numerical': 0.21048651373834132, 'mean_word_length': 2.236541598694943, 'pattern_counts': {'":': 58, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1246080', 'n_tokens_mistral': 2435, 'n_tokens_neox': 1837, 'n_words': 435}