<commit_before># coding: utf8
from __future__ import unicode_literals

from clld.util import slug
from clld.lib.dsv import reader


_LANGUAGES = [
    ('psc', 'Proto-Siouan-Catawba', None, 'siou1252', 'D2B48C', None),
        ('psi', 'Proto-Siouan', 'sio', 'core1249', 'FFE4C4', 'psc'),
            ('pch', 'Proto-Crow-Hidatsa', None, 'miss1252', '87CEEB', 'psi'),
                ('cr', 'Crow', 'cro', 'crow1244', 'ADD8E6', 'pch'),
                ('hi', 'Hidatsa', 'hid', 'hida1246', 'ADD8E6', 'pch'),
            ('pma', 'Pre-Mandan', None, None, '2E8B57', 'psi'),
                ('ma', 'Mandan', 'mhq', 'mand1446', '3CB371', 'pma'),
            ('pmv', 'Proto-Mississipi-Valley', None, 'miss1254', 'BA55D3', 'psi'),
                ('pda', 'Proto-Dakota', None, 'dako1257', 'DA70D6', 'pmv'),
                    ('la', 'Lakota', 'lkt', 'lako1247', 'EE82EE', 'pda'),
                    ('da', 'Dakota', 'dak', 'dako1258', 'EE82EE', 'pda'),
                    ('as', 'Assiniboine', 'asb', 'assi1247', 'EE82EE', 'pda'),
                    ('ya', 'Yanktonai', 'dak', 'dako1258', 'EE82EE', 'pda'),
                    ('st', 'Stoney', 'sto', 'ston1242', 'EE82EE', 'pda'),
                    ('sv', 'Sioux Valley', None, None, 'EE82EE', 'pda'),
                ('pwc', 'Proto-Hoocąk-Chiwere', None, 'winn1245', 'FF4500', 'pmv'),
                    ('ch', 'Chiwere', 'iow', 'iowa1245', 'FF7F50', 'pwc'),
                    #('io', 'Ioway', 'iow', 'iowa1245', 'FF7F50', 'pwc'),
                    ('mo', 'Missouria', None, 'miss1249', 'FF7F50', 'pwc'),
                    ('ot', 'Otoe', 'iow', 'iowa1245', 'FF7F50', 'pwc'),
                    ('wi', 'Hoocąk', 'win', 'hoch1243', 'FF7F50', 'pwc'),
                ('pdh', 'Proto-Dhegiha', None, 'dheg1241', 'FFA500', 'pmv'),
                    ('om', 'Omaha', 'oma', 'omah1247', 'FFD700', 'pdh'),
                    ('op', 'Omaha-Ponca', 'oma', 'omah1247', 'FFD700', 'pdh'),
                    ('pn', 'Ponca', 'oma', 'omah1247', 'FFD700', 'pdh'),
                    ('ks', 'Kanza/Kaw', 'ksk', 'kans1243', 'FFD700', 'pdh'),
                    ('os', 'Osage', 'osa', 'osag1243', 'FFD700', 'pdh'),
                    ('qu', 'Quapaw', 'qua', 'quap1242', 'FFD700', 'pdh'),
            ('pse', 'Proto-Southeastern', None, 'bilo1247', '008080', 'psi'),
                ('pbo', 'Proto-Biloxi-Ofo', None, 'bilo1247', '008B8B', 'pse'),
                    ('bi', 'Biloxi', 'bll', 'bilo1248', '00FFFF', 'pbo'),
                    ('of', 'Ofo', 'ofo', 'ofoo1242', '00FFFF', 'pbo'),
                ('pts', 'Proto-Tutelo-Saponi', None, 'bilo1247', '008B8B', 'pse'),
                    ('tu', 'Tutelo', 'tta', 'tute1247', '00FFFF', 'pts'),
                    ('sp', 'Saponi', 'tta', 'tute1247', '00FFFF', 'pts'),
        ('pca', 'Proto-Catawba', None, 'cata1285', '778899', 'psc'),
            ('ca', 'Catawba', 'chc', 'cata1286', 'B0C4DE', 'pca'),
            ('wo', 'Woccon', 'xwc', 'wocc1242', 'B0C4DE', 'pca'),
    ]

SOURCES = {
    'N': 'Nikonha (Tutelo informant)',
    'AG': '<NAME>',
    'ASG': '<NAME>',
    'AWJ': '<NAME>',
    'B': 'Eugene Buechel dictionary',
    'C': '<NAME>',
    'DEC': 'DICTIONARY OF EVERYDAY CROW (GORDON/GRACZYK)',
    'DS': 'DORSEY/SWANTON OFO DICTIONARY',
    'EB': '<NAME>',
    'EJ': '<NAME> (Lakota speaker)',
    'F + LF': 'Fletcher and Laflesche',
    'FLF': 'Fletcher and Laflesche',
    'Fontaine': 'Fontaine (Saponi word list)',
    'FS': '<NAME>',
    'GG': 'GORDON AND GRACZYK',
    'GK': 'GEOFFREY KIMBALL',
    'G': 'GORDON? GILLMORE? GRACZYK?',
    'GMsf': 'GORDON MARSH SOMETHING',
    'H': 'RO<NAME>',
    'H&Vf': 'HOLLOW AND ?? (ONLY ONE INSTANCE)',
    'H(N)': '<NAME>',
    'H.': '<NAME>',
    'HWM': 'WASHINGTON MATTHEWS 1877 HIDATSA GRAMMAR',
    'HH': 'H<NAME>',
    'J': '<NAME>',
    'JEK': '<NAME>',
    'JGT': 'Jimm Good Tracks',
    'JOD': '<NAME>',
    'JWE': 'YES - WHITE EAGLE',
    'KM': '<NAME>',
    'KS': '<NAME> or the Kansa language',
    'LB': '<NAME>',
    'LF': 'Laflesche',
    'Lawson': '<NAME> WOCCON VOCABULARY',
    'Lipkind': 'WILLIAM LIPKIND WINNEBAGO GRAMMAR',
    'Lk': 'Lipkind',
    'LWR': '<NAME>',
    'Marsh': '<NAME>',
    'M': 'GH MATTHEWS',
    'mr': "SEEMS TO BE ONE OF BOB'S KANSA SPEAKERS - CK HIS DICTIONARY INTRO",
    'MS': 'MAURICE SWADESH OR "MANUSCRIPT?"',
    'PAS': '<NAME>',
    'per': "ANOTHER KANSA SPEAKER -- SEE RANKIN'S DICTONARY INTRO??",
    'R': 'STEPHEN R. RIGGS DICTIONARY',
    'RG': '<NAME>',
    'RLR': '<NAME>',
    'RR': '<NAME>',
    'RTC': '<NAME>',
    'RTG': '<NAME>',
    'Sp.': '<NAME>',
    'Speck': '<NAME>',
    'Taylor': '<NAME>',
    'Ssf': 'SOMETHING WITH OFO DATA',
    'SW': '?MARK SWETLAND DICTIONARY?',
    'Wm': 'WILLIAMSON ENGLISH-DAKOT DICTIONARY',
    'W': "CAN''T FIND IT -- WOLFF? WHITMAN?  NEED TO KNOW LANGUAGE",
    'Miller + Davis': '<NAME> and ? Davis',<commit_msg>fixes #34<commit_after># coding: utf8
from __future__ import unicode_literals

from clld.util import slug
from clld.lib.dsv import reader


_LANGUAGES = [
    ('psc', 'Proto-Siouan-Catawba', None, 'siou1252', 'D2B48C', None),
        ('psi', 'Proto-Siouan', 'sio', 'core1249', 'FFE4C4', 'psc'),
            ('pch', 'Proto-Crow-Hidatsa', None, 'miss1252', '87CEEB', 'psi'),
                ('cr', 'Crow', 'cro', 'crow1244', 'ADD8E6', 'pch'),
                ('hi', 'Hidatsa', 'hid', 'hida1246', 'ADD8E6', 'pch'),
            ('pma', 'Pre-Mandan', None, None, '2E8B57', 'psi'),
                ('ma', 'Mandan', 'mhq', 'mand1446', '3CB371', 'pma'),
            ('pmv', 'Proto-Mississipi-Valley', None, 'miss1254', 'BA55D3', 'psi'),
                ('pda', 'Proto-Dakota', None, 'dako1257', 'DA70D6', 'pmv'),
                    ('la', 'Lakota', 'lkt', 'lako1247', 'EE82EE', 'pda'),
                    ('da', 'Dakota', 'dak', 'dako1258', 'EE82EE', 'pda'),
                    ('as', 'Assiniboine', 'asb', 'assi1247', 'EE82EE', 'pda'),
                    ('ya', 'Yanktonai', 'dak', 'dako1258', 'EE82EE', 'pda'),
                    ('st', 'Stoney', 'sto', 'ston1242', 'EE82EE', 'pda'),
                    ('sv', 'Sioux Valley', None, None, 'EE82EE', 'pda'),
                ('pwc', 'Proto-Hoocąk-Chiwere', None, 'winn1245', 'FF4500', 'pmv'),
                    ('ch', 'Chiwere', 'iow', 'iowa1245', 'FF7F50', 'pwc'),
                    #('io', 'Ioway', 'iow', 'iowa1245', 'FF7F50', 'pwc'),
                    ('mo', 'Missouria', None, 'miss1249', 'FF7F50', 'pwc'),
                    ('ot', 'Otoe', 'iow', 'iowa1245', 'FF7F50', 'pwc'),
                    ('wi', 'Hoocąk', 'win', 'hoch1243', 'FF7F50', 'pwc'),
                ('pdh', 'Proto-Dhegiha', None, 'dheg1241', 'FFA500', 'pmv'),
                    ('om', 'Omaha', 'oma', 'omah1247', 'FFD700', 'pdh'),
                    ('op', 'Omaha-Ponca', 'oma', 'omah1247', 'FFD700', 'pdh'),
                    ('pn', 'Ponca', 'oma', 'omah1247', 'FFD700', 'pdh'),
                    ('ks', 'Kanza/Kaw', 'ksk', 'kans1243', 'FFD700', 'pdh'),
                    ('os', 'Osage', 'osa', 'osag1243', 'FFD700', 'pdh'),
                    ('qu', 'Quapaw', 'qua', 'quap1242', 'FFD700', 'pdh'),
            ('pse', 'Proto-Southeastern', None, 'bilo1247', '008080', 'psi'),
                ('pbo', 'Proto-Biloxi-Ofo', None, 'bilo1247', '008B8B', 'pse'),
                    ('bi', 'Biloxi', 'bll', 'bilo1248', '00FFFF', 'pbo'),
                    ('of', 'Ofo', 'ofo', 'ofoo1242', '00FFFF', 'pbo'),
                ('pts', 'Proto-Tutelo-Saponi', None, 'bilo1247', '008B8B', 'pse'),
                    ('tu', 'Tutelo', 'tta', 'tute1247', '00FFFF', 'pts'),
                    ('sp', 'Saponi', 'tta', 'tute1247', '00FFFF', 'pts'),
        ('pca', 'Proto-Catawba', None, 'cata1285', '778899', 'psc'),
            ('ca', 'Catawba', 'chc', 'cata1286', 'B0C4DE', 'pca'),
            ('wo', 'Woccon', 'xwc', 'wocc1242', 'B0C4DE', 'pca'),
    ]

SOURCES = {
    'N': 'Nikonha (Tutelo informant)',
    'AG': '<NAME>',
    'ASG': '<NAME>',
    'AWJ': '<NAME>',
    'B': 'Eugene Buechel dictionary',
    'C': '<NAME>',
    'DEC': 'DICTIONARY OF EVERYDAY CROW (GORDON/GRACZYK)',
    'DS': 'DORSEY/SWANTON OFO DICTIONARY',
    'EB': '<NAME>',
    'EJ': '<NAME> (Lakota speaker)',
    'Fontaine': 'Fontaine (Saponi word list)',
    'FS': '<NAME>',
    'GG': 'GORDON AND GRACZYK',
    'GK': 'GEOFFREY KIMBALL',
    'G': 'GORDON? GILLMORE? GRACZYK?',
    'GMsf': 'GORDON MARSH SOMETHING',
    'H': 'RO<NAME>',
    'H&Vf': 'HOLLOW AND ?? (ONLY ONE INSTANCE)',
    'H(N)': '<NAME>',
    'H.': '<NAME>',
    'HWM': 'WASHINGTON MATTHEWS 1877 HIDATSA GRAMMAR',
    'HH': 'HORATIO HALE',
    'J': '<NAME>',
    'JEK': '<NAME>',
    'JGT': 'Jimm Good Tracks',
    'JOD': '<NAME>',
    'JWE': 'YES - WHITE EAGLE',
    'KM': '<NAME>',
    'KS': '<NAME> or the Kansa language',
    'LB': '<NAME>',
    'LF': 'Laflesche',
    'Lawson': '<NAME> WOCCON VOCABULARY',
    'Lipkind': 'WILLIAM LIPKIND WINNEBAGO GRAMMAR',
    'Lk': 'Lipkind',
    'LWR': '<NAME>',
    'Marsh': '<NAME>',
    'M': 'GH MATTHEWS',
    'mr': "SEEMS TO BE ONE OF BOB'S KANSA SPEAKERS - CK HIS DICTIONARY INTRO",
    'MS': 'MAURICE SWADESH OR "MANUSCRIPT?"',
    'PAS': '<NAME>',
    'R': 'STEPHEN R. RIGGS DICTIONARY',
    'RG': '<NAME>',
    'RLR': '<NAME>',
    'RR': '<NAME>',
    'RTC': '<NAME>',
    'RTG': '<NAME>',
    'Sp.': '<NAME>',
    'Speck': '<NAME>',
    'Taylor': '<NAME>',
    'Ssf': 'SOMETHING WITH OFO DATA',
    'SW': '?MARK SWETLAND DICTIONARY?',
    'Wm': 'WILLIAMSON ENGLISH-DAKOT DICTIONARY',
    'W': "CAN''T FIND IT -- WOLFF? WHITMAN?  NEED TO KNOW LANGUAGE",
    'Miller + Davis': '<NAME> and ? Davis',
{'fraction_non_alphanumeric': 0.2503652682112294, 'fraction_numerical': 0.05405969526194949, 'mean_word_length': 2.5257542310522445, 'pattern_counts': {'":': 0, '<': 62, '<?xml version=': 0, '>': 62, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6844204', 'n_tokens_mistral': 4307, 'n_tokens_neox': 3950, 'n_words': 884}