# Path: mlscripts/text/word_desc.py
# class WordDesc:
#     def __init__(self, id, stem):
#         self.__id = id
#         self.__stem = stem
#         self.__n_doc = 0
#         self.__n_upper = 0
#         self.__n_lower = 0
#         
#     def is_noun(self):
#         return self.__n_upper >= 1#/ (self.__n_upper + self.__n_lower) >= 0.5
#     
#     def add_variant(self, variant):
#         if variant.istitle():
#             self.__n_upper += 1
#         else:
#             self.__n_lower += 1
#     
#     @property
#     def stem(self):
#         return self.__stem
#     
#     @property
#     def total_count(self):
#         return (self.__n_upper + self.__n_lower)
#      
#     @property
#     def document_count(self):
#         return self.__n_doc
#      
#     @document_count.setter
#     def document_count(self, new_count):
#         self.__n_doc = new_count
#     
#     @property
#     def id(self):
#         return self.__id


from mlscripts.text.utilities import *
from mlscripts.text.word_desc import WordDesc
from numpy import argsort



class WordDict:
    def __init__(self):
        self.__current_id = 0
        self.__words = {}

    def vectorize(self, word_list):
        """Word_list must be in a sorted order"""
        #cleaned_list.sort()
        id_list = []
        for stem in word_list:
            if stem in self.__words:
                item = self.__words[stem]
                if item.document_count >= 2:
                    id_list.append(item.id)
            else:
                raise NameError('unknown word %s' % stem)
        return id_list

    def fit(self, word_list):
        """Word_list must be in a sorted order"""
        #cleaned_list.sort()
        previous_word = None
        for stem in word_list:
            if stem in self.__words:
                item = self.__words[stem]
                if stem != previous_word:
                    item.document_count += 1
            else:

# Path: mlscripts/text/word_desc.py
# class WordDesc:
#     def __init__(self, id, stem):
#         self.__id = id
#         self.__stem = stem
#         self.__n_doc = 0
#         self.__n_upper = 0
#         self.__n_lower = 0
#         
#     def is_noun(self):
#         return self.__n_upper >= 1#/ (self.__n_upper + self.__n_lower) >= 0.5
#     
#     def add_variant(self, variant):
#         if variant.istitle():
#             self.__n_upper += 1
#         else:
#             self.__n_lower += 1
#     
#     @property
#     def stem(self):
#         return self.__stem
#     
#     @property
#     def total_count(self):
#         return (self.__n_upper + self.__n_lower)
#      
#     @property
#     def document_count(self):
#         return self.__n_doc
#      
#     @document_count.setter
#     def document_count(self, new_count):
#         self.__n_doc = new_count
#     
#     @property
#     def id(self):
#         return self.__id


# Path: mlscripts/text/word_dict.py
from mlscripts.text.utilities import *
from mlscripts.text.word_desc import WordDesc
from numpy import argsort



class WordDict:
    def __init__(self):
        self.__current_id = 0
        self.__words = {}

    def vectorize(self, word_list):
        """Word_list must be in a sorted order"""
        #cleaned_list.sort()
        id_list = []
        for stem in word_list:
            if stem in self.__words:
                item = self.__words[stem]
                if item.document_count >= 2:
                    id_list.append(item.id)
            else:
                raise NameError('unknown word %s' % stem)
        return id_list

    def fit(self, word_list):
        """Word_list must be in a sorted order"""
        #cleaned_list.sort()
        previous_word = None
        for stem in word_list:
            if stem in self.__words:
                item = self.__words[stem]
                if stem != previous_word:
                    item.document_count += 1
            else:

                item = WordDesc(self.__current_id, stem)
