<jupyter_start><jupyter_text>Text Vectorization<jupyter_code># Import python libs

import sqlite3 as sqlite       # work with sqlite databases
import os                      # used to set working directory
import pandas as pd            # process data with pandas dataframe
import numpy as np

# Setup pandas display options

pd.options.display.max_colwidth = 500

# Constants

small_sqlite = "example_db.sqlite"

# Set working directory

os.chdir('../Data/')


# Read sqlite query results into a pandas DataFrame
con = sqlite.connect(small_sqlite)

df = pd.read_sql_query("SELECT * from Documents", con)

con.close()

df.head()
from sklearn.feature_extraction.text import CountVectorizer
vectorizer = CountVectorizer()
X = vectorizer.fit_transform(df['NOTE_TEXT'].tolist())

X
X.toarray()
vectorizer.get_feature_names()<jupyter_output><empty_output><jupyter_text>Let's remove stop words and include bigrams...<jupyter_code>vectorizer2 = CountVectorizer(stop_words='english', ngram_range=(1, 2))
X2 = vectorizer2.fit_transform(df['NOTE_TEXT'].tolist())
vectorizer2.get_feature_names()<jupyter_output><empty_output><jupyter_text>tf-idf weighting* tf - Term Frequency* idf - Inverse Document Frequency * The tf-idf weight of a term is the product of its tf weight and its idf weight.$W_{t,d} = log(1+tf_{t,d}) \cdot log_{10}(\frac{N}{df_{t}})$* Best known weighting scheme in information retrieval* Note: the “-” in tf-idf is a hyphen, not a minus sign!* Increases with the number of occurrences within a document* Increases with the rarity of the term in the collection<jupyter_code>from sklearn.feature_extraction.text import TfidfTransformer

transformer = TfidfTransformer(use_idf=True)
tfidf_result = transformer.fit_transform(X2)
def display_scores(vectorizer, tfidf_result):
    scores = zip(vectorizer.get_feature_names(),
                 np.asarray(tfidf_result.sum(axis=0)).ravel())
    sorted_scores = sorted(scores, key=lambda x: x[1], reverse=True)
    for item in sorted_scores:
        print "{0:20} Score: {1}".format(item[0], item[1])

display_scores(vectorizer2, tfidf_result)

from nltk.stem.porter import *
from nltk.tokenize import word_tokenize
import string

stemmer = PorterStemmer()
def stem_tokens(tokens, stemmer):
    stemmed = []
    for item in tokens:
        stemmed.append(stemmer.stem(item))
    return stemmed

def tokenize(text):
    tokens = word_tokenize(text)
    tokens = [i for i in tokens if i not in string.punctuation]
    stems = stem_tokens(tokens, stemmer)
    return stems

vectorizer3 = CountVectorizer(tokenizer=tokenize, stop_words='english', ngram_range=(1, 2))
X3 = vectorizer3.fit_transform(df['NOTE_TEXT'].tolist())
vectorizer3.get_feature_names()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1047235444891981, 'fraction_numerical': 0.010618820944708897, 'mean_word_length': 5.125560538116592, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '75714', 'license': 'MIT', 'n_tokens_mistral': 928, 'n_tokens_neox': 882, 'n_words': 231, 'path': 'TextVectorization.ipynb', 'repo_name': 'alandschaft/DataScienceMeetup', 'size': '2731'}