def make_Tfidf_array(df_content):

    def tokenize(text):
        '\n        Function splits text into separate words and gets a word lowercased and removes whitespaces at the ends of a word. \n        The funtions also cleans irrelevant stopwords.\n        Input:\n        1. text: text message\n        Output:\n        1. Clean_tokens : list of tokenized clean words\n        '
        text = re.sub('[^a-zA-Z0-9]', ' ', text)
        tokens = word_tokenize(text)
        lemmatizer = WordNetLemmatizer()
        clean_tokens = []
        for tok in tokens:
            clean_tok = lemmatizer.lemmatize(tok, pos='v').lower().strip()
            clean_tokens.append(clean_tok)
        stopwords = nltk.corpus.stopwords.words('english')
        clean_tokens = [token for token in clean_tokens if (token not in stopwords)]
        return clean_tokens
    corpus = df_content['doc_description']
    df_content['doc_description'].fillna(df_content['doc_full_name'], inplace=True)
    stop_words = stopwords.words('english')
    lemmatizer = WordNetLemmatizer()
    vect = TfidfVectorizer(tokenizer=tokenize)
    X = vect.fit_transform(corpus)
    X = X.toarray()
    return (vect, X)
def make_Tfidf_array(df_content):

    def tokenize(text):
        '\n        Function splits text into separate words and gets a word lowercased and removes whitespaces at the ends of a word. \n        The funtions also cleans irrelevant stopwords.\n        Input:\n        1. text: text message\n        Output:\n        1. Clean_tokens : list of tokenized clean words\n        '
        text = re.sub('[^a-zA-Z0-9]', ' ', text)
        tokens = word_tokenize(text)
        lemmatizer = WordNetLemmatizer()
        clean_tokens = []
        for tok in tokens:
            clean_tok = lemmatizer.lemmatize(tok, pos='v').lower().strip()
            clean_tokens.append(clean_tok)
        stopwords = nltk.corpus.stopwords.words('english')
        clean_tokens = [token for token in clean_tokens if (token not in stopwords)]
        return clean_tokens
    corpus = df_content['doc_description']
    df_content['doc_description'].fillna(df_content['doc_full_name'], inplace=True)
    stop_words = stopwords.words('english')
    lemmatizer = WordNetLemmatizer()
    vect = TfidfVectorizer(tokenizer=tokenize)
    X = vect.fit_transform(corpus)
    X = X.toarray()
    return (vect, X)