<commit_before># -*- coding: utf8 - *-
"""Cihai dataset for `Unihan`_, Han Unification from Unicode, Inc.

cihai.datasets.unihan
~~~~~~~~~~~~~~~~~~~~~

"""

from __future__ import absolute_import, division, print_function, \
    with_statement, unicode_literals

import os
import sys
import glob
import hashlib
import fileinput
try:
    from urllib import urlretrieve
except:
    from urllib.request import urlretrieve
import logging

from sqlalchemy import Table, String, Column, Integer, Index, select, or_, and_

from .. import conversion, CihaiDataset
from ..util import get_datafile, UnicodeReader
from .._compat import PY2, text_type, configparser

__copyright__ = 'Copyright 2013 <NAME>.'
__license__ = 'BSD, see LICENSE for details.'

log = logging.getLogger(__name__)

# This is the Unihan_DictionaryLikeData as of 6.3.0
# The format is { 'FILENAME': ['fields'] }
UNIHAN_DATASETS = {
    'Unihan_DictionaryIndices.txt': [
        'kCheungBauerIndex',
        'kCowles',
        'kDaeJaweon',
        'kFennIndex',<commit_msg>py3k, unihan test, StringIO from compat.<commit_after># -*- coding: utf8 - *-
"""Cihai dataset for `Unihan`_, Han Unification from Unicode, Inc.

cihai.datasets.unihan
~~~~~~~~~~~~~~~~~~~~~

"""

from __future__ import absolute_import, division, print_function, \
    with_statement, unicode_literals

import os
import sys
import glob
import hashlib
import fileinput
try:
    from urllib import urlretrieve
except:
    from urllib.request import urlretrieve
import logging

from sqlalchemy import Table, String, Column, Integer, Index, select, or_, and_

from .. import conversion, CihaiDataset
from ..util import get_datafile, UnicodeReader
from .._compat import StringIO

__copyright__ = 'Copyright 2013 <NAME>.'
__license__ = 'BSD, see LICENSE for details.'

log = logging.getLogger(__name__)

# This is the Unihan_DictionaryLikeData as of 6.3.0
# The format is { 'FILENAME': ['fields'] }
UNIHAN_DATASETS = {
    'Unihan_DictionaryIndices.txt': [
        'kCheungBauerIndex',
        'kCowles',
        'kDaeJaweon',
        'kFennIndex',
{'fraction_non_alphanumeric': 0.11325301204819277, 'fraction_numerical': 0.008674698795180723, 'mean_word_length': 4.309462915601023, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6925391', 'n_tokens_mistral': 685, 'n_tokens_neox': 653, 'n_words': 200}