<jupyter_start><jupyter_text>Dummy example on short texts  Bag of word model<jupyter_code># list of text documents
text_test = ["This is a test document","this is a second text","and here is a third text", "this is a dum text!"]
# create the transform
vectorizer_test = CountVectorizer()
# tokenize and build vocab
vectorizer_test.fit(text_test)
# summarize
print(vectorizer_test.vocabulary_)
# encode document
vector_test = vectorizer_test.transform(text_test)
# summarize encoded vector
print(vector_test.toarray())<jupyter_output>[[0 1 0 0 1 0 1 0 0 1]
 [0 0 0 0 1 1 0 1 0 1]
 [1 0 0 1 1 0 0 1 1 0]
 [0 0 1 0 1 0 0 1 0 1]]<jupyter_text>TF-IDF of documents<jupyter_code># create the transform
vectorizer_tfidf_test = TfidfVectorizer()
# tokenize and build vocab
vectorizer_tfidf_test.fit(text_test)
# summarize
print(vectorizer_tfidf_test.vocabulary_)
# encode document
vector_tfidf_test = vectorizer_tfidf_test.transform(text_test)
# summarize encoded vector
print(vector_tfidf_test.toarray())<jupyter_output>[[0.         0.61087812 0.         0.         0.31878155 0.
  0.61087812 0.         0.         0.38991559]
 [0.         0.         0.         0.         0.3612126  0.69218835
  0.         0.44181486 0.         0.44181486]
 [0.52130524 0.         0.         0.52130524 0.2720387  0.
  0.         0.33274238 0.52130524 0.        ]
 [0.         0.         0.69218835 0.         0.3612126  0.
  0.         0.44181486 0.         0.44181486]]<jupyter_text>Compute distances between documents<jupyter_code>dist = DistanceMetric.get_metric('euclidean')
dist.pairwise(vector_tfidf_test.toarray())
dist = DistanceMetric.get_metric('braycurtis')
dist.pairwise(vector_tfidf_test.toarray())<jupyter_output><empty_output><jupyter_text>Using Wikipedia abstracts  Import dataset<jupyter_code># import text documents from wikipedia abstracts
wiki_data=pd.read_csv('new_zeland_people.tsv',delimiter='\t', index_col='name')['abstract']
len(wiki_data)
wiki_data.head()<jupyter_output><empty_output><jupyter_text>Compute word counts<jupyter_code>name = '<NAME>'
#Extract text for a particular person
text = wiki_data[name]
#Define the count vectorizer that will be used to process the data
count_vectorizer = CountVectorizer()
#Apply this vectorizer to text to get a sparse matrix of counts
count_matrix = count_vectorizer.fit_transform([text])
#Get the names of the features
features = count_vectorizer.get_feature_names()
#Create a series from the sparse matrix
d = pd.Series(count_matrix.toarray().flatten(), 
              index = features).sort_values(ascending=False)

ax = d[:10].plot(kind='bar', figsize=(10,6), width=.8, fontsize=14, rot=45,
            title='Taika Waititi Wikipedia Article Word Counts')
ax.title.set_size(18)<jupyter_output><empty_output><jupyter_text>Compute TF-IDF<jupyter_code>#Define the TFIDF vectorizer that will be used to process the data
tfidf_vectorizer = TfidfVectorizer()
#Apply this vectorizer to the full dataset to create normalized vectors
tfidf_matrix = tfidf_vectorizer.fit_transform(wiki_data)
#Get the names of the features
features = tfidf_vectorizer.get_feature_names()
#get the row that contains relevant vector
row = wiki_data.index.get_loc(name)
#Create a series from the sparse matrix
d = pd.Series(tfidf_matrix.getrow(row).toarray().flatten(), index = features).sort_values(ascending=False)

ax = d[:10].plot(kind='bar', title='<NAME> Wikipedia Article Word TF-IDF Values',
            figsize=(10,6), width=.8, fontsize=14, rot=45 )
ax.title.set_size(20)<jupyter_output><empty_output><jupyter_text>Search the kNN<jupyter_code>nbrs = NearestNeighbors(n_neighbors=10, metric='cosine').fit(tfidf_matrix)
distances, indices = nbrs.kneighbors()

def get_closest_neighs(name):
    row = wiki_data.index.get_loc(name)
    result = pd.DataFrame({'names':wiki_data.iloc[indices[row],].reset_index()['name'],
                           'distance': distances[row].flatten()})
    return result
get_closest_neighs('<NAME>')
# Find the closest neighbors for '<NAME>'
get_closest_neighs('<NAME>')<jupyter_output><empty_output><jupyter_text>Metagenomes as text documents<jupyter_code># import the Bray-curtis distance matrix and metagenome description

bc_25bp=pd.read_csv('bc_25.csv', header=None)

samples=pd.read_csv('metagenomes_description.csv', index_col='ID')['body_site']

samples
# create a 2-NN search for the samples
nbrs = NearestNeighbors(n_neighbors=2, metric='precomputed').fit(bc_25bp)
distances, indices = nbrs.kneighbors()

def get_closest_neighs(ID):
    row = samples.index.get_loc(ID)
    result = pd.DataFrame({'ID':samples.iloc[indices[row],].reset_index()['ID'], 'distance':distances[row].flatten(),
                          'body_site': samples.iloc[indices[row],].reset_index()['body_site']})
    return result
# Retrieve the two NN for 'sample_6'
get_closest_neighs('sample_6')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11503697617091208, 'fraction_numerical': 0.05115036976170912, 'mean_word_length': 4.803337306317044, 'pattern_counts': {'":': 0, '<': 36, '<?xml version=': 0, '>': 36, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '98426', 'license': 'MIT', 'n_tokens_mistral': 1856, 'n_tokens_neox': 1684, 'n_words': 386, 'path': 'sprint3_retrieval/.ipynb_checkpoints/sprint3_demo_solutions-checkpoint.ipynb', 'repo_name': 'maliaca/ML_sprint', 'size': '4903'}