<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import unittest

import Mykytea

def showTags(t):
    for word in t:
        out = word.surface + "\t"
        for t1 in word.tag:
            for t2 in t1:
                for t3 in t2:
                    out = out + "/" + str(t3)
                out += "\t"
            out += "\t"
        print out

def list_tags(t):
    def convert(t2):<commit_msg>Add debug print<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import unittest
import sys
print sys.path

import Mykytea

def showTags(t):
    for word in t:
        out = word.surface + "\t"
        for t1 in word.tag:
            for t2 in t1:
                for t3 in t2:
                    out = out + "/" + str(t3)
                out += "\t"
            out += "\t"
        print out

def list_tags(t):
    def convert(t2):
{'fraction_non_alphanumeric': 0.12906976744186047, 'fraction_numerical': 0.018604651162790697, 'mean_word_length': 1.3653846153846154, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5252576', 'n_tokens_mistral': 299, 'n_tokens_neox': 290, 'n_words': 89}