<commit_before># -*- coding: utf-8 -*-

from __future__ import (absolute_import, division, print_function,
                        unicode_literals, with_statement)

from unihan_tabular import process


def test_expands_spaces(expanded_data):

    for item in expanded_data:
        for field in item.keys():
            if field in process.SPACE_DELIMITED_FIELDS and item[field]:
                assert isinstance(item[field], list)


def test_expand_kCantonese(expanded_data):
    # test kCantonese
    item = [i for i in expanded_data if i['ucn'] == 'U+342B'][0]
    if item['ucn'] == 'U+342B':
        assert set(item['kCantonese']) == set(['gun3', 'hung1', 'zung1'])<commit_msg>add description to test_expansion section<commit_after># -*- coding: utf-8 -*-
"""Test expansion of multi-value fields in UNIHAN."""

from __future__ import (absolute_import, division, print_function,
                        unicode_literals, with_statement)

from unihan_tabular import process


def test_expands_spaces(expanded_data):

    for item in expanded_data:
        for field in item.keys():
            if field in process.SPACE_DELIMITED_FIELDS and item[field]:
                assert isinstance(item[field], list)


def test_expand_kCantonese(expanded_data):
    # test kCantonese
    item = [i for i in expanded_data if i['ucn'] == 'U+342B'][0]
    if item['ucn'] == 'U+342B':
        assert set(item['kCantonese']) == set(['gun3', 'hung1', 'zung1'])
{'fraction_non_alphanumeric': 0.12016574585635359, 'fraction_numerical': 0.015193370165745856, 'mean_word_length': 3.3644578313253013, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5182079', 'n_tokens_mistral': 477, 'n_tokens_neox': 459, 'n_words': 120}