<jupyter_start><jupyter_text>Compose an ekorpkit config<jupyter_code># %%capture
# %pip install ekorpkit[transformers,tokenize]
from ekorpkit import eKonf<jupyter_output><empty_output><jupyter_text>Instantiating a mecab class<jupyter_code>config_group='preprocessor/tokenizer=mecab'
cfg = eKonf.compose(config_group=config_group)
eKonf.pprint(cfg)
mecab = eKonf.instantiate(cfg)
text = 'IMF가 推定한 우리나라의 GDP갭률은 今年에도 소폭의 마이너스(−)를 持續하고 있다.'
tokens = mecab.tokenize(text)
print(tokens)
print(mecab(text))
config_group='preprocessor/normalizer=formal_ko'
cfg_norm = eKonf.compose(config_group=config_group)
eKonf.pprint(cfg_norm)
norm = eKonf.instantiate(cfg_norm)
norm(text)
config_group='preprocessor/tokenizer=mecab'
cfg = eKonf.compose(config_group=config_group)
mecab = eKonf.instantiate(cfg, normalize=norm)
tokens = mecab.tokenize(text)
print(tokens)
config_group='preprocessor/tokenizer=mecab'
cfg = eKonf.compose(config_group=config_group)
cfg['normalize'] = cfg_norm
eKonf.pprint(cfg)
mecab = eKonf.instantiate(cfg)
tokens = mecab.tokenize(text)
print(tokens)
print(mecab.nouns(text))
print(mecab.morphs(text))
text = '금통위는 통화신용정책과 한국은행의 운영에 관한 의결권을 행사한다.'
print(mecab.tokenize(text))
config_group='preprocessor/tokenizer=mecab_econ'
cfg = eKonf.compose(config_group=config_group)
eKonf.pprint(cfg)
mecab = eKonf.instantiate(cfg)
tokens = mecab.tokenize(text)
print(tokens)<jupyter_output>{'_target_': 'ekorpkit.preprocessors.tokenizer.MecabTokenizer',
 'extract': {'no_space_for_non_nouns': False,
             'noun_postags': ['NNG', 'NNP', 'XSN', 'SL', 'XR', 'NNB', 'NR'],
             'postags': None,
             'stop_postags': ['SP'],
             'stopwords': None,
             'stopwords_path': None},
 'mecab': {'backend': 'mecab-python3',
           'userdic_path': '/workspace/projects/ekorpkit/ekorpkit/resources/dictionaries/mecab/ekon_v1.dic',
           'verbose': False},
 'normalize': None,
 'tokenize': {'concat_surface_and_pos': True,
              'flatten': True,
              'include_whitespace_token': True,
              'lowercase': False,
              'punct_postags': ['SF', 'SP', 'SSO', 'SSC', 'SY'],
              'tokenize_each_word': False,
              'userdic_path': None,
              'wordpieces_prefix': '##'},
 'tokenize_article': {'return_type': 'str', 'sentence_separator': '\\n'},
 'verbose': False}
{'fraction_non_alphanumeric': 0.1483843537414966, 'fraction_numerical': 0.0008503401360544217, 'mean_word_length': 5.033333333333333, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '251164', 'license': 'CC-BY-4.0', 'n_tokens_mistral': 966, 'n_tokens_neox': 980, 'n_words': 137, 'path': 'notebooks/preprocessors/mecab.ipynb', 'repo_name': 'entelecheia/ekorpkit-config', 'size': '2352'}