<commit_before>#!/usr/bin/env python
# -*- coding: utf8 - *-

from __future__ import print_function, unicode_literals

from cihai.bootstrap import bootstrap_unihan
from cihai.core import Cihai


def variant_list(unihan, field):
    for char in unihan.with_fields(field):
        print("Character: {}".format(char.char))
        for var in char.untagged_vars(field):
            print(var)


def script(unihan_options={}):
    """Wrapped so we can test in tests/test_examples.py"""
    print("This example prints variant character data.")

    c = Cihai()
    c.add_dataset('cihai.unihan.Unihan', namespace='unihan')
    if not c.sql.is_bootstrapped:  # download and install Unihan to db
        bootstrap_unihan(c.sql.metadata, options=unihan_options)
        c.sql.reflect_db()  # automap new table created during bootstrap

    print("## ZVariants")
    variant_list(c.unihan, "kZVariant")

    print("## kSemanticVariant")
    variant_list(c.unihan, "kSemanticVariant")
<commit_msg>Add variant extension in example script<commit_after>#!/usr/bin/env python
# -*- coding: utf8 - *-

from __future__ import print_function, unicode_literals

from cihai.bootstrap import bootstrap_unihan
from cihai.core import Cihai


def variant_list(unihan, field):
    for char in unihan.with_fields(field):
        print("Character: {}".format(char.char))
        for var in char.untagged_vars(field):
            print(var)


def script(unihan_options={}):
    """Wrapped so we can test in tests/test_examples.py"""
    print("This example prints variant character data.")

    c = Cihai()
    c.add_dataset('cihai.unihan.Unihan', namespace='unihan')
    if not c.sql.is_bootstrapped:  # download and install Unihan to db
        bootstrap_unihan(c.sql.metadata, options=unihan_options)
        c.sql.reflect_db()  # automap new table created during bootstrap

    c.unihan.add_extension('cihai.unihan.UnihanVariants', namespace='variants')

    print("## ZVariants")
    variant_list(c.unihan, "kZVariant")

    print("## kSemanticVariant")
    variant_list(c.unihan, "kSemanticVariant")

{'fraction_non_alphanumeric': 0.11747713047664901, 'fraction_numerical': 0.0009629272989889263, 'mean_word_length': 3.8665105386416863, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4078698', 'n_tokens_mistral': 710, 'n_tokens_neox': 657, 'n_words': 159}