# Path: shorttext/utils/textpreprocessing.py
# def preprocess_text(text, pipeline):
# def text_preprocessor(pipeline):
# def oldschool_standard_text_preprocessor(stopwordsfile):
# def standard_text_preprocessor_1():
# def standard_text_preprocessor_2():


import unittest
import re
import pandas as pd
import shorttext
from shorttext.utils import stemword, tokenize




class TestDTM(unittest.TestCase):
    def test_inaugural(self):
        # preparing data
        usprez = shorttext.data.inaugural()
        docids = sorted(usprez.keys())
        usprez = [' '.join(usprez[docid]) for docid in docids]

# Path: shorttext/utils/textpreprocessing.py
# def preprocess_text(text, pipeline):
# def text_preprocessor(pipeline):
# def oldschool_standard_text_preprocessor(stopwordsfile):
# def standard_text_preprocessor_1():
# def standard_text_preprocessor_2():


# Path: test/test_dtm.py
import unittest
import re
import pandas as pd
import shorttext
from shorttext.utils import stemword, tokenize





class TestDTM(unittest.TestCase):
    def test_inaugural(self):
        # preparing data
        usprez = shorttext.data.inaugural()
        docids = sorted(usprez.keys())
        usprez = [' '.join(usprez[docid]) for docid in docids]

        usprezdf = pd.DataFrame({'yrprez': docids, 'speech': usprez})