<commit_before>"""Test the CLTK.
TODO: Write test for copy_dir_contents
"""

__author__ = '<NAME> <<EMAIL>>'
__license__ = 'MIT License. See LICENSE.'

import os
import unittest

from cltk.corpus.greek.beta_to_unicode import Replacer

from cltk.corpus.greek.tlgu import TLGU
from cltk.utils.file_operations import open_pickle
from cltk.utils.build_contribs_index import build_contribs_file
from cltk.corpus.utils.formatter import remove_non_ascii
from cltk.corpus.utils.formatter import assemble_phi5_author_filepaths
from cltk.corpus.utils.formatter import assemble_phi5_works_filepaths
from cltk.corpus.utils.formatter import assemble_tlg_author_filepaths
from cltk.corpus.utils.formatter import assemble_tlg_works_filepaths
from cltk.corpus.utils.formatter import phi5_plaintext_cleanup
from cltk.corpus.utils.formatter import tlg_plaintext_cleanup
from cltk.corpus.utils.importer import CorpusImporter
from cltk.stem.latin.j_v import JVReplacer
from cltk.stem.lemma import LemmaReplacer
from cltk.stem.latin.stem import Stemmer
from cltk.stop.greek.stops import STOPS_LIST as greek_stops
from cltk.stem.latin.syllabifier import Syllabifier
from cltk.stop.latin.stops import STOPS_LIST as latin_stops
from cltk.tag.pos import POSTag
from cltk.tokenize.sentence import TokenizeSentence
from nltk.tokenize.punkt import PunktLanguageVars
import stat


class TestSequenceFunctions(unittest.TestCase):  # pylint: disable=R0904
    """Class for unittest"""

    def setUp(self):
        pass

    def test_import_perseus_latin_text(self):
        """Test downloading the Perseus Latin text corpus."""
        c = CorpusImporter('latin')
        c.import_corpus('latin_text_perseus')
        dir_rel = os.path.join('~/cltk_data/latin/text/latin_text_perseus/latin_corpus_perseus/Ammianus')
        dir = os.path.expanduser(dir_rel)
        dir_exists = os.path.isdir(dir)
        self.assertTrue(dir_exists)

    def test_import_proper_names_latin(self):
        """Test downloading the Latin proper names corpus."""
        c = CorpusImporter('latin')
        c.import_corpus('latin_proper_names')
        file_rel = os.path.join('~/cltk_data/latin/dictionary/latin_proper_names/proper_names.txt')
        file = os.path.expanduser(file_rel)
        file_exists = os.path.isfile(file)
        self.assertTrue(file_exists)

    def test_import_tlgu(self):
        """Test downloading TLGU."""
        c = CorpusImporter('greek')
        c.import_corpus('greek_proper_names')
        file_rel = os.path.join('~/cltk_data/greek/software/tlgu/tlgu.h')
        file = os.path.expanduser(file_rel)
        file_exists = os.path.isfile(file)
        self.assertTrue(file_exists)

    def test_import_proper_names_greek(self):
        """Test downloading the Greek proper names corpus."""
        c = CorpusImporter('greek')
        c.import_corpus('greek_proper_names')
        file_rel = os.path.join('~/cltk_data/greek/dictionary/greek_proper_names/proper_names.txt')
        file = os.path.expanduser(file_rel)
        file_exists = os.path.isfile(file)
        self.assertTrue(file_exists)

    def test_import_greek_trbnk_perseus(self):
        """Test downloading the Perseus greek treebank corpus."""
        c = CorpusImporter('greek')
        c.import_corpus('greek_treebank_perseus')
        file_rel = os.path.join('~/cltk_data/greek/treebank/greek_treebank_perseus/unigram.pickle')
        file = os.path.expanduser(file_rel)
        file_exists = os.path.isfile(file)
        self.assertTrue(file_exists)

    def test_import_latin_trbnk_perseus(self):
        """Test downloading the Perseus Latin treebank corpus."""
        c = CorpusImporter('latin')
        c.import_corpus('latin_treebank_perseus')
        file_rel = os.path.join('~/cltk_data/latin/treebank/latin_treebank_perseus/unigram.pickle')
        file = os.path.expanduser(file_rel)
        file_exists = os.path.isfile(file)
        self.assertTrue(file_exists)
    
    def test_import_lacus_curtius(self):
        """Test downloading the Lacus_Curtius Latin text corpus."""
        c = CorpusImporter('latin')
        c.import_corpus('latin_text_lacus_curtius')
        dir_rel = os.path.join('~/cltk_data/latin/text/latin_text_lacus_curtius/lacus_curtius/Aelian')
        dir = os.path.expanduser(dir_rel)
        dir_exists = os.path.isdir(dir)
        self.assertTrue(dir_exists)

    def test_import_perseus_greek_text(self):
        """Test downloading the Perseus Greek text corpus."""
        c = CorpusImporter('greek')
        c.import_corpus('greek_text_perseus')
        dir_rel = os.path.join('~/cltk_data/greek/text/greek_text_perseus/greek_corpus_perseus/Aeschines')
        dir = os.path.expanduser(dir_rel)
        dir_exists = os.path.isdir(dir)
        self.assertTrue(dir_exists)
    
    def test_import_latin_library(self):
        """Test downloading the Latin Library text corpus."""
        c = CorpusImporter('latin')
        c.import_corpus('latin_text_latin_library')
        dir_rel = os.path.join('~/cltk_data/latin/text/latin_text_latin_library/latin_library/abelard')
        dir = os.path.expanduser(dir_rel)
        dir_exists = os.path.isdir(dir)
        self.assertTrue(dir_exists)

    def test_latin_ling_import(self):
        c = CorpusImporter('latin')
        c.import_corpus('cltk_linguistic_data')
        dir_rel = os.path.join('~/cltk_data/latin/trained_model/cltk_linguistic_data')
        dir = os.path.expanduser(dir_rel)
        dir_exists = os.path.isdir(dir)
        self.assertTrue(dir_exists)

    def test_greek_ling_import(self):<commit_msg>comment out ll test<commit_after>"""Test the CLTK.
TODO: Write test for copy_dir_contents
"""

__author__ = '<NAME> <<EMAIL>>'
__license__ = 'MIT License. See LICENSE.'

import os
import unittest

from cltk.corpus.greek.beta_to_unicode import Replacer

from cltk.corpus.greek.tlgu import TLGU
from cltk.utils.file_operations import open_pickle
from cltk.utils.build_contribs_index import build_contribs_file
from cltk.corpus.utils.formatter import remove_non_ascii
from cltk.corpus.utils.formatter import assemble_phi5_author_filepaths
from cltk.corpus.utils.formatter import assemble_phi5_works_filepaths
from cltk.corpus.utils.formatter import assemble_tlg_author_filepaths
from cltk.corpus.utils.formatter import assemble_tlg_works_filepaths
from cltk.corpus.utils.formatter import phi5_plaintext_cleanup
from cltk.corpus.utils.formatter import tlg_plaintext_cleanup
from cltk.corpus.utils.importer import CorpusImporter
from cltk.stem.latin.j_v import JVReplacer
from cltk.stem.lemma import LemmaReplacer
from cltk.stem.latin.stem import Stemmer
from cltk.stop.greek.stops import STOPS_LIST as greek_stops
from cltk.stem.latin.syllabifier import Syllabifier
from cltk.stop.latin.stops import STOPS_LIST as latin_stops
from cltk.tag.pos import POSTag
from cltk.tokenize.sentence import TokenizeSentence
from nltk.tokenize.punkt import PunktLanguageVars
import stat


class TestSequenceFunctions(unittest.TestCase):  # pylint: disable=R0904
    """Class for unittest"""

    def setUp(self):
        pass

    def test_import_perseus_latin_text(self):
        """Test downloading the Perseus Latin text corpus."""
        c = CorpusImporter('latin')
        c.import_corpus('latin_text_perseus')
        dir_rel = os.path.join('~/cltk_data/latin/text/latin_text_perseus/latin_corpus_perseus/Ammianus')
        dir = os.path.expanduser(dir_rel)
        dir_exists = os.path.isdir(dir)
        self.assertTrue(dir_exists)

    def test_import_proper_names_latin(self):
        """Test downloading the Latin proper names corpus."""
        c = CorpusImporter('latin')
        c.import_corpus('latin_proper_names')
        file_rel = os.path.join('~/cltk_data/latin/dictionary/latin_proper_names/proper_names.txt')
        file = os.path.expanduser(file_rel)
        file_exists = os.path.isfile(file)
        self.assertTrue(file_exists)

    def test_import_tlgu(self):
        """Test downloading TLGU."""
        c = CorpusImporter('greek')
        c.import_corpus('greek_proper_names')
        file_rel = os.path.join('~/cltk_data/greek/software/tlgu/tlgu.h')
        file = os.path.expanduser(file_rel)
        file_exists = os.path.isfile(file)
        self.assertTrue(file_exists)

    def test_import_proper_names_greek(self):
        """Test downloading the Greek proper names corpus."""
        c = CorpusImporter('greek')
        c.import_corpus('greek_proper_names')
        file_rel = os.path.join('~/cltk_data/greek/dictionary/greek_proper_names/proper_names.txt')
        file = os.path.expanduser(file_rel)
        file_exists = os.path.isfile(file)
        self.assertTrue(file_exists)

    def test_import_greek_trbnk_perseus(self):
        """Test downloading the Perseus greek treebank corpus."""
        c = CorpusImporter('greek')
        c.import_corpus('greek_treebank_perseus')
        file_rel = os.path.join('~/cltk_data/greek/treebank/greek_treebank_perseus/unigram.pickle')
        file = os.path.expanduser(file_rel)
        file_exists = os.path.isfile(file)
        self.assertTrue(file_exists)

    def test_import_latin_trbnk_perseus(self):
        """Test downloading the Perseus Latin treebank corpus."""
        c = CorpusImporter('latin')
        c.import_corpus('latin_treebank_perseus')
        file_rel = os.path.join('~/cltk_data/latin/treebank/latin_treebank_perseus/unigram.pickle')
        file = os.path.expanduser(file_rel)
        file_exists = os.path.isfile(file)
        self.assertTrue(file_exists)
    
    def test_import_lacus_curtius(self):
        """Test downloading the Lacus_Curtius Latin text corpus."""
        c = CorpusImporter('latin')
        c.import_corpus('latin_text_lacus_curtius')
        dir_rel = os.path.join('~/cltk_data/latin/text/latin_text_lacus_curtius/lacus_curtius/Aelian')
        dir = os.path.expanduser(dir_rel)
        dir_exists = os.path.isdir(dir)
        self.assertTrue(dir_exists)

    def test_import_perseus_greek_text(self):
        """Test downloading the Perseus Greek text corpus."""
        c = CorpusImporter('greek')
        c.import_corpus('greek_text_perseus')
        dir_rel = os.path.join('~/cltk_data/greek/text/greek_text_perseus/greek_corpus_perseus/Aeschines')
        dir = os.path.expanduser(dir_rel)
        dir_exists = os.path.isdir(dir)
        self.assertTrue(dir_exists)

    '''
    TODO: fix the dl of this
    def test_import_latin_library(self):
        """Test downloading the Latin Library text corpus."""
        c = CorpusImporter('latin')
        c.import_corpus('latin_text_latin_library')
        dir_rel = os.path.join('~/cltk_data/latin/text/latin_text_latin_library/latin_library/abelard')
        dir = os.path.expanduser(dir_rel)
        dir_exists = os.path.isdir(dir)
        self.assertTrue(dir_exists)
    '''

    def test_latin_ling_import(self):
        c = CorpusImporter('latin')
        c.import_corpus('cltk_linguistic_data')
        dir_rel = os.path.join('~/cltk_data/latin/trained_model/cltk_linguistic_data')
        dir = os.path.expanduser(dir_rel)
        dir_exists = os.path.isdir(dir)
        self.assertTrue(dir_exists)

    def test_greek_ling_import(self):
{'fraction_non_alphanumeric': 0.10115374295680171, 'fraction_numerical': 0.0012521241391646543, 'mean_word_length': 3.655287260616153, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3674771', 'n_tokens_mistral': 3942, 'n_tokens_neox': 3729, 'n_words': 610}