<jupyter_start><jupyter_text>Imports and NLP<jupyter_code># !python -m spacy download en_core_web_lg
import pandas as pd 
import numpy as np
import pickle
import spacy
from sklearn.neighbors import NearestNeighbors
import time
import re
%load_ext google.colab.data_table
import spacy.cli
spacy.cli.download("en_core_web_lg")
nlp = spacy.load("en_core_web_lg")<jupyter_output>[38;5;2m✔ Download and installation successful[0m
You can now load the model via spacy.load('en_core_web_lg')<jupyter_text>Examine Data<jupyter_code># @title Retrieve Data

url = "https://raw.githubusercontent.com/vjmiyagi/DS_BuildWeek_Dec_2020/main/csv/data.csv"
df_main = pd.read_csv(url)
df_artist = pd.read_csv("https://raw.githubusercontent.com/vjmiyagi/DS_BuildWeek_Dec_2020/main/csv/data_by_artist.csv")
df_genre = pd.read_csv("https://raw.githubusercontent.com/vjmiyagi/DS_BuildWeek_Dec_2020/main/csv/data_by_genres.csv")
df_year = pd.read_csv("https://raw.githubusercontent.com/vjmiyagi/DS_BuildWeek_Dec_2020/main/csv/data_by_year.csv")
df_w = pd.read_csv("https://raw.githubusercontent.com/vjmiyagi/DS_BuildWeek_Dec_2020/main/csv/data_w_genres.csv")

new_row = {"valence": 0.404, "year": 1962, "acousticness": 0.707, 
            "artists": "['<NAME>']", "danceability": 0.415, 
            "duration_ms": 189520, "energy": 0.359, "explicit": 1, 
            "id": "6meIeOX3DHdaCnaNw67abE", "instrumentalness": 0.0000653,
            "key": 7, "liveness": 0.319, "loudness": -11.094, "mode": 1, 
            "name": "上を向いて歩こう Ue Wo Muite <NAME>", "popularity": 50,
            "release_date": "2010-01-01", "speechiness": 0.0334, "tempo": 146.029}

df_main = df_main.append(new_row, ignore_index=True)

#@title Heads or Tails  { run: "auto" }
show = "Heads" #@param ["Heads", "Tails"]
from __future__ import print_function

from google.colab import widgets
from google.colab import output


a = df_main.head()
b = df_main.tail()
c = df_artist.head()
d = df_artist.tail()
e = df_genre.head()
f = df_genre.tail()
g = df_year.head()
h = df_year.tail()
i = df_year.head()
j = df_year.tail()

heads = [a, c, e, f, i]
tails = [b, d, f, h, j]

def TB():
    tb = widgets.TabBar(["Data", "Artist", "Genre", "Year", "Genre with Artist"])
    l_tab = show
    for x in range(5):
        with tb.output_to(x):
            if l_tab == "Heads":
                print(heads[x])
            elif l_tab == "Tails":
                print(tails[x])

dash = '=' * 81
print(dash)
t7 = "Shapes"
print(f'{t7:^81}')
print()
s1, s2, s3, s4, s5 = ("Main", "Artist", "Genre",
                              "Year", "Genre with Artist")
print(f'{s1:^16}{s2:^16}{s3:^16}{s4:^16}{s5:^16}')
t1, t2, t3, t4, t5 = (str(df_main.shape), str(df_artist.shape), 
                          str(df_genre.shape), str(df_year.shape),
                          str(df_w.shape))

print(f'{t1:^16}{t2:^16}{t3:^16}{t4:^16}{t5:^16}')
print()
print(dash)





TB()
#@title Explore Data  { run: "auto" }
dataset = "Data" #@param ["Data", "Artist", "Genre", "Year", "Genre with Artist"]
if dataset == "Data":
    df = df_main
    title = "Data"
elif dataset == "Artist":
    df = df_artist
    title = "Data by Artist"
elif dataset == "Genre":
    df = df_genre
    title = "Data by Genre"
elif dataset == "Year":
    df = df_year
    title = "Data by Year"
else:
    df = df_w
    title = "Genre with Artist"
# pd.set_option("display.precision", 2)
dash = "="*50

print(f"{title:^50}")
print(dash)
df.info()
#@title Heads or Tails ... a closer look  { run: "auto" }
show = "Tails" #@param ["Heads", "Tails"]
from __future__ import print_function

from google.colab import widgets
from google.colab import output

z1 = len(df_main) - 1
z2 = len(df_artist) - 1
z3 = len(df_genre) - 1
z4 = len(df_year) - 1
z5 = len(df_w) - 1
a = df_main.loc[0]
b = df_main.loc[z1]
c = df_artist.loc[0]
d = df_artist.loc[z2]
e = df_genre.loc[0]
f = df_genre.loc[z3]
g = df_year.loc[0]
h = df_year.loc[z4]
i = df_w.loc[0]
j = df_w.loc[z5]

heads = [a, c, e, f, i]
tails = [b, d, f, h, j]

def TB():
    tb = widgets.TabBar(["Data", "Artist", "Genre", "Year", "Genre with Artist"])
    l_tab = show
    for x in range(5):
        with tb.output_to(x):
            if l_tab == "Heads":
                print(heads[x])
            elif l_tab == "Tails":
                print(tails[x])

dash = '=' * 81
print(dash)
t7 = "Shapes"
print(f'{t7:^81}')
print()
s1, s2, s3, s4, s5 = ("Main", "Artist", "Genre",
                              "Year", "Genre with Artist")
print(f'{s1:^16}{s2:^16}{s3:^16}{s4:^16}{s5:^16}')
t1, t2, t3, t4, t5 = (str(df_main.shape), str(df_artist.shape), 
                          str(df_genre.shape), str(df_year.shape),
                          str(df_w.shape))

print(f'{t1:^16}{t2:^16}{t3:^16}{t4:^16}{t5:^16}')
print()
print(dash)





TB()
#@title Heads or Tails ... a closer look  { run: "auto" }
show = "Tails" #@param ["Heads", "Tails"]
from __future__ import print_function

from google.colab import widgets
from google.colab import output

z1 = len(df_main) - 1
z2 = len(df_artist) - 1
z3 = len(df_genre) - 1
z4 = len(df_year) - 1
z5 = len(df_w) - 1
a = df_main.loc[0]
b = df_main.loc[z1]
c = df_artist.loc[0]
d = df_artist.loc[z2]
e = df_genre.loc[0]
f = df_genre.loc[z3]
g = df_year.loc[0]
h = df_year.loc[z4]
i = df_w.loc[0]
j = df_w.loc[z5]

heads = [a, c, e, f, i]
tails = [b, d, f, h, j]

def TB():
    tb = widgets.TabBar(["Data", "Artist", "Genre", "Year", "Genre with Artist"])
    l_tab = show
    for x in range(5):
        with tb.output_to(x):
            if l_tab == "Heads":
                print(heads[x])
            elif l_tab == "Tails":
                print(tails[x])

dash = '=' * 81
print(dash)
t7 = "Shapes"
print(f'{t7:^81}')
print()
s1, s2, s3, s4, s5 = ("Main", "Artist", "Genre",
                              "Year", "Genre with Artist")
print(f'{s1:^16}{s2:^16}{s3:^16}{s4:^16}{s5:^16}')
t1, t2, t3, t4, t5 = (str(df_main.shape), str(df_artist.shape), 
                          str(df_genre.shape), str(df_year.shape),
                          str(df_w.shape))

print(f'{t1:^16}{t2:^16}{t3:^16}{t4:^16}{t5:^16}')
print()
print(dash)





TB()
df
df["artists"]
df_w['artists']

print("df:", df.shape)
print(df.head(1))
print("df_artist:", df_artist.shape)
print(df_artist.head(1))
print("df_genre:", df_genre.shape)
print(df_genre.head(1))
print("df_year:", df_year.shape)
print(df_year.head(1))
print("df_w:", df_w.shape)
print(df_w.head(1))

df_w.tail(1)<jupyter_output><empty_output><jupyter_text>Look for file:
vectorized.csv

The following cell takes approximately 20 minutes to run.  As long as no changes have been made to the original dataframe, except for the addition of the 1 Japanese song, you can skip it and just do the following in a new cell:

```
df2 = pd.DataFrame("vecorized.csv")
```<jupyter_code>df['name_artist'] = df['name'] + df['artists']
df['name_artist']
# Making computer sense out of all these words
then = time.time() #Time before the operations start

print("Grab a cup of coffee, this is gonna take a while.")
x = [nlp(str(item)).vector for item in df['name_artist']]
now = time.time() #Time after it finished

z = (now-then)/60
print(f"It took: {z:0.1f} minutes to run this cell")

df2 = pd.DataFrame(x)
print(df2.shape)
df2.head()
df2.to_csv("vectorized.csv")<jupyter_output><empty_output><jupyter_text>Start Playing Here!<jupyter_code>#@title Get the Model Right  { run: "auto" }
#@markdown ### Algorithm:
#@markdown ## Auto will choose for you
algorithm = "kd_tree" #@param ["ball_tree", "kd_tree", "brute", "auto"]
#@markdown ### Leaf Size default is 30
#@markdown ### This can affect the speed of the construction and query, as well as the memory requierd to store the tree.  The optimal value depends on the nature of the problem.
leaf_size = 30 #@param {type:"integer"}
#@markdown ### Metric  default = minkowski
metric = "minkowski" #@param {type:"string"}
#@markdown ### Metric Parameters
metric_params = "None" #@param {type:"string"}
#@markdown ### number of jobs
n_jobs =  1#@param {type:"integer"}
#@markdown # Nearest Neighbors
n_neighbors = 50 #@param {type:"integer"}

"""
Original Code From Yuan Jin
nn = NearestNeighbors(n_neighbors = 3, algorithm='ball_tree')
"""



nn = NearestNeighbors(algorithm=algorithm, leaf_size=leaf_size, 
                      n_neighbors=n_neighbors, n_jobs=n_jobs)




nn.fit(df2)


#@title Default title text  { run: "auto" }
test = "<NAME>" #@param {type:"string"}


test_vect = nlp(test).vector
vect = test_vect.reshape(1, -1)
results = nn.kneighbors(vect)

s = [df['name'].iloc[x] for x in results[1][0]]
a = [df['artists'].iloc[x] for x in results[1][0]]
y = [df['release_date'].iloc[x] for x in results[1][0]]
     

# print(songs)
# print(results[1][0])
cn = ["Song", "Artist", "Year", "Row #"]
print(f"{cn[0]:<51}{cn[1]:<45}{cn[2]:<15}{cn[3]:^25}")
print("=" * 126)
for i in range(len(results[1][0])):

    print(f"{s[i][0:50]:<51}{a[i][0:44]:<45}{y[i]:^15}{results[1][0][i]:>16}")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.18051703095528682, 'fraction_numerical': 0.03905469876844558, 'mean_word_length': 3.6825974025974024, 'pattern_counts': {'":': 29, '<': 24, '<?xml version=': 0, '>': 20, 'https://': 5, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '253234', 'license': 'MIT', 'n_tokens_mistral': 3906, 'n_tokens_neox': 3696, 'n_words': 836, 'path': 'spotify.ipynb', 'repo_name': 'vjmiyagi/DS_BuildWeek_Dec_2020', 'size': '9038'}