<commit_before># -*- coding: utf-8 -*-
import os
#
import pytest
import numpy as np
# pylint: disable=W0611
import conftest
# pylint: enable=W0611

from serializer import Serializer
from naivebayes import NaiveBayes, CorpusTokenizer

# pylint: disable=C0103
class TestClass(object):
    def test_loadSettingFile(self):
        basepath = '../resource/'
        config = Serializer.load_json(os.path.join(basepath, 'setting.json'))
        Serializer.load_ini(config['AUTH']['TWITTER'])
        Serializer.load_json(os.path.join(basepath, 'ocr.json'))
        Serializer.load_csv(os.path.join(basepath, 'corpus.tsv'))
    def test_naivebayes_compare(self):
        basepath = '../resource/'
        naivebayes = NaiveBayes()
        json_data = Serializer.load_json(os.path.join(basepath, 'ocr.json'))
        x_list = ['ネツァワル王国','カセドリア連合王国','ゲブランド帝国','ホルデイン王国','エルソード王国']
        out = naivebayes.predict_all(x_list, json_data['translate']['country'])
        for i, y in enumerate(out):
            if x_list[i] != y:
                raise Exception('compare x:{0},predict:{1}'.format(x_list[i], y))
    def test_naivebayes_labeling(self):
        naivebayes = NaiveBayes()
        corpus = Serializer.load_csv('../resource/corpus.tsv')
        data = []
        target = []
        for row in corpus:
            data.append(str(row[0]))
            t = int(row[1])
            if t > 5:
                raise Exception(t)<commit_msg>remove CorpusTokenizer<commit_after># -*- coding: utf-8 -*-
import os
#
import pytest
import numpy as np
# pylint: disable=W0611
import conftest
# pylint: enable=W0611

from serializer import Serializer
from naivebayes import NaiveBayes

# pylint: disable=C0103
class TestClass(object):
    def test_loadSettingFile(self):
        basepath = '../resource/'
        config = Serializer.load_json(os.path.join(basepath, 'setting.json'))
        Serializer.load_ini(config['AUTH']['TWITTER'])
        Serializer.load_json(os.path.join(basepath, 'ocr.json'))
        Serializer.load_csv(os.path.join(basepath, 'corpus.tsv'))
    def test_naivebayes_compare(self):
        basepath = '../resource/'
        naivebayes = NaiveBayes()
        json_data = Serializer.load_json(os.path.join(basepath, 'ocr.json'))
        x_list = ['ネツァワル王国','カセドリア連合王国','ゲブランド帝国','ホルデイン王国','エルソード王国']
        
        print(json_data['translate']['country'])
        out = naivebayes.predict_all(x_list, json_data['translate']['country'])
        for i, y in enumerate(out):
            if x_list[i] != y:
                raise Exception('compare x:{0},predict:{1}'.format(x_list[i], y))
    def test_naivebayes_labeling(self):
        naivebayes = NaiveBayes()
        corpus = Serializer.load_csv('../resource/corpus.tsv')
        data = []
        target = []
        for row in corpus:
            data.append(str(row[0]))
            t = int(row[1])
            if t > 5:
                raise Exception(t)
{'fraction_non_alphanumeric': 0.14134154688569472, 'fraction_numerical': 0.012320328542094456, 'mean_word_length': 2.8511198945981553, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1638984', 'n_tokens_mistral': 1036, 'n_tokens_neox': 1006, 'n_words': 176}