<commit_before># -*- coding: utf8 -*-

#: Dictionary of tuples mapping locations of files to fields
UNIHAN_MANIFEST = {
    'Unihan_DictionaryIndices.txt': (
        'kCheungBauerIndex',
        'kCowles',
        'kDaeJaweon',
        'kFennIndex',
        'kGSR',
        'kHanYu',
        'kIRGDaeJaweon',
        'kIRGDaiKanwaZiten',
        'kIRGHanyuDaZidian',
        'kIRGKangXi',
        'kKangXi',
        'kKarlgren',
        'kLau',
        'kMatthews',
        'kMeyerWempe',
        'kMorohashi',
        'kNelson',
        'kSBGY',
    ),
    'Unihan_DictionaryLikeData.txt': (
        'kCangjie',
        'kCheungBauer',
        'kCihaiT',
        'kFenn',
        'kFourCornerCode',
        'kFrequency',
        'kGradeLevel',
        'kHDZRadBreak',
        'kHKGlyph',
        'kPhonetic',
        'kTotalStrokes',
    ),
    'Unihan_IRGSources.txt': (
        'kCompatibilityVariant',
        'kIICore',
        'kIRG_GSource',
        'kIRG_HSource',
        'kIRG_JSource',
        'kIRG_KPSource',
        'kIRG_KSource',
        'kIRG_MSource',
        'kIRG_TSource',
        'kIRG_USource',
        'kIRG_VSource',
    ),
    'Unihan_NumericValues.txt': (
        'kAccountingNumeric',
        'kOtherNumeric',
        'kPrimaryNumeric',
    ),
    'Unihan_OtherMappings.txt': (
        'kBigFive',
        'kCCCII',
        'kCNS1986',
        'kCNS1992',
        'kEACC',
        'kGB0',
        'kGB1',
        'kGB3',
        'kGB5',
        'kGB7',
        'kGB8',
        'kHKSCS',
        'kIBMJapan',
        'kJa',
        'kJis0',
        'kJis1',
        'kJIS0213',
        'kKPS0',
        'kKPS1',
        'kKSC0',
        'kKSC1',
        'kMainlandTelegraph',
        'kPseudoGB1',
        'kTaiwanTelegraph',
        'kXerox',
    ),
    'Unihan_RadicalStrokeCounts.txt': (
        'kRSAdobe_Japan1_6',
        'kRSJapanese',
        'kRSKangXi',
        'kRSKanWa',
        'kRSKorean',
        'kRSUnicode',
    ),
    'Unihan_Readings.txt': (
        'kCantonese',
        'kDefinition',
        'kHangul',
        'kHanyuPinlu',
        'kHanyuPinyin',
        'kJapaneseKun',
        'kJapaneseOn',
        'kKorean',
        'kMandarin',
        'kTang',
        'kVietnamese',
        'kXHC1983',
    ),
    'Unihan_Variants.txt': (
        'kSemanticVariant',
        'kSimplifiedVariant',
        'kSpecializedSemanticVariant',
        'kTraditionalVariant',
        'kZVariant',
    )
}

#: FIELDS with multiple values via custom delimiters
CUSTOM_DELIMITED_FIELDS = (
    'kDefinition',
    'kDaeJaweon',
    'kHDZRadBreak',
    'kIRG_GSource',
    'kIRG_HSource',
)

#: Fields with multiple values UNIHAN delimits by spaces -> dict
SPACE_DELIMITED_DICT_FIELDS = (
    'kHanYu',
    'kXHC1983',
    'kMandarin',
    'kTotalStrokes',
)<commit_msg>add more kIRG sources for custom delimited fields<commit_after># -*- coding: utf8 -*-

#: Dictionary of tuples mapping locations of files to fields
UNIHAN_MANIFEST = {
    'Unihan_DictionaryIndices.txt': (
        'kCheungBauerIndex',
        'kCowles',
        'kDaeJaweon',
        'kFennIndex',
        'kGSR',
        'kHanYu',
        'kIRGDaeJaweon',
        'kIRGDaiKanwaZiten',
        'kIRGHanyuDaZidian',
        'kIRGKangXi',
        'kKangXi',
        'kKarlgren',
        'kLau',
        'kMatthews',
        'kMeyerWempe',
        'kMorohashi',
        'kNelson',
        'kSBGY',
    ),
    'Unihan_DictionaryLikeData.txt': (
        'kCangjie',
        'kCheungBauer',
        'kCihaiT',
        'kFenn',
        'kFourCornerCode',
        'kFrequency',
        'kGradeLevel',
        'kHDZRadBreak',
        'kHKGlyph',
        'kPhonetic',
        'kTotalStrokes',
    ),
    'Unihan_IRGSources.txt': (
        'kCompatibilityVariant',
        'kIICore',
        'kIRG_GSource',
        'kIRG_HSource',
        'kIRG_JSource',
        'kIRG_KPSource',
        'kIRG_KSource',
        'kIRG_MSource',
        'kIRG_TSource',
        'kIRG_USource',
        'kIRG_VSource',
    ),
    'Unihan_NumericValues.txt': (
        'kAccountingNumeric',
        'kOtherNumeric',
        'kPrimaryNumeric',
    ),
    'Unihan_OtherMappings.txt': (
        'kBigFive',
        'kCCCII',
        'kCNS1986',
        'kCNS1992',
        'kEACC',
        'kGB0',
        'kGB1',
        'kGB3',
        'kGB5',
        'kGB7',
        'kGB8',
        'kHKSCS',
        'kIBMJapan',
        'kJa',
        'kJis0',
        'kJis1',
        'kJIS0213',
        'kKPS0',
        'kKPS1',
        'kKSC0',
        'kKSC1',
        'kMainlandTelegraph',
        'kPseudoGB1',
        'kTaiwanTelegraph',
        'kXerox',
    ),
    'Unihan_RadicalStrokeCounts.txt': (
        'kRSAdobe_Japan1_6',
        'kRSJapanese',
        'kRSKangXi',
        'kRSKanWa',
        'kRSKorean',
        'kRSUnicode',
    ),
    'Unihan_Readings.txt': (
        'kCantonese',
        'kDefinition',
        'kHangul',
        'kHanyuPinlu',
        'kHanyuPinyin',
        'kJapaneseKun',
        'kJapaneseOn',
        'kKorean',
        'kMandarin',
        'kTang',
        'kVietnamese',
        'kXHC1983',
    ),
    'Unihan_Variants.txt': (
        'kSemanticVariant',
        'kSimplifiedVariant',
        'kSpecializedSemanticVariant',
        'kTraditionalVariant',
        'kZVariant',
    )
}

#: FIELDS with multiple values via custom delimiters
CUSTOM_DELIMITED_FIELDS = (
    'kDefinition',
    'kDaeJaweon',
    'kHDZRadBreak',
    'kIRG_GSource',
    'kIRG_HSource',
    'kIRG_JSource',
    'kIRG_KPSource',
    'kIRG_KSource',
    'kIRG_MSource',
    'kIRG_TSource',
    'kIRG_USource',
    'kIRG_VSource',
)

#: Fields with multiple values UNIHAN delimits by spaces -> dict
SPACE_DELIMITED_DICT_FIELDS = (
    'kHanYu',
    'kXHC1983',
    'kMandarin',
    'kTotalStrokes',
)
{'fraction_non_alphanumeric': 0.13618273057622426, 'fraction_numerical': 0.012458902924381381, 'mean_word_length': 1.793620106331561, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1333515', 'n_tokens_mistral': 2365, 'n_tokens_neox': 2238, 'n_words': 292}