<jupyter_start><jupyter_text><jupyter_code>import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from ast import literal_eval
import ast
from sklearn import datasets
from sklearn.cluster import KMeans, DBSCAN
from sklearn import metrics
from sklearn import cluster
import matplotlib.pyplot as plt
from sklearn.preprocessing import scale
from sklearn.preprocessing import QuantileTransformer
import seaborn as sb
from sklearn.metrics import pairwise_distances_argmin_min
from google.colab import drive
import os
drive.mount('/content/gdrive')
# Establecer ruta de acceso en drive
import os
print(os.getcwd())
os.chdir("/content/gdrive/My Drive")
%cd '/content/gdrive/MyDrive/PYFINAL'
movies = pd.read_csv('movies_metadata.csv', encoding='iso8859')
movie = pd.read_csv('movies_metadata.csv', encoding='iso8859')

movie.shape
movie['genres'] = movie['genres'].fillna('[]').apply(literal_eval).apply(lambda x: [i['id'] for i in x] if isinstance(x, list) else [])
 # Sacamos el id de generos 
#Se agrega en el df movie el id del genero de cada pelicula.
gp=movie['genres'].values
y = []
for i in range(len(gp)):
  if len(gp[i]) > 0:
    y.append(gp[i][0])
  else:
    y.append(0)

movie.loc[:,'genid'] = y


#Se obtiene el nombre de cada genero del df
movies['genres'] = movies['genres'].fillna('[]').apply(literal_eval).apply(lambda x: [i['name'] for i in x] if isinstance(x, list) else [])
#Se agrega en el df movie el nombre del genero de cada pelicula.
gp1=movies['genres'].values
y1 = []
for i in range(len(gp1)):
  if len(gp1[i]) > 0:
    y1.append(gp1[i][0])
  else:
    y1.append('NA')

movie.loc[:,'genero'] = y1
#Se obtiene el nombre de cada compañia de produccion del df movies
movies = pd.read_csv('movies_metadata.csv', encoding='iso8859')
movies['production_companies'] = movies['production_companies'].fillna('[]').apply(literal_eval).apply(lambda x: [i['id'] for i in x] if isinstance(x, list) else [])
cp=movies['production_companies'].values
z = []
for i in range(len(cp)):
  if len(cp[i]) > 0:
    z.append(cp[i][0])
  else:
    z.append('0')

movie.loc[:,'compid'] = z
movies = pd.read_csv('movies_metadata.csv', encoding='iso8859')

movies['production_companies'] = movies['production_companies'].fillna('[]').apply(literal_eval).apply(lambda x: [i['name'] for i in x] if isinstance(x, list) else [])
cp1=movies['production_companies'].values
z1 = []
for i in range(len(cp1)):
  if len(cp1[i]) > 0:
    z1.append(cp1[i][0])
  else:
    z1.append('0')

movie.loc[:,'compname'] = z1

#Se borra del df movie todas las columnas que no son numericas y que creemos que no nos sirven
movie = movie.drop('status', axis=1)
movie = movie.drop('belongs_to_collection', axis=1)
movie = movie.drop('adult', axis=1)
movie = movie.drop('genres', axis=1)
movie = movie.drop('homepage', axis=1)
movie = movie.drop('imdb_id', axis=1)
movie = movie.drop('original_language', axis=1)
movie = movie.drop('overview', axis=1)
movie = movie.drop('poster_path', axis=1)
movie = movie.drop('production_companies', axis=1)
movie = movie.drop('production_countries', axis=1)
movie = movie.drop('release_date', axis=1)
movie = movie.drop('revenue', axis=1)
movie = movie.drop('runtime', axis=1)
movie = movie.drop('spoken_languages', axis=1)
movie = movie.drop('tagline', axis=1)
movie = movie.drop('video', axis=1)
movie = movie.dropna() #borramos los nulos
movie.head()
#Se revisa que diferencia se encuentra en la columna de titulo de la pelicula original con la del titulo, y revisar si podemos eliminar alguna columna que no nos sirva
movie[movie['original_title'] != movie['title']][['title', 'original_title']].head()


#Se decide borrar la columna original_title
movie = movie.drop('original_title', axis=1)
movie.head()
#Copiamos el df Movie en movie1
movie1=movie
#Se carga credits para poder sacar los actores y directores
credits = pd.read_csv('credits.csv', encoding='iso8859')
credits.head()
#Funcion para convertir en entero
def convertint(x):
    try:
        return int(x)
    except:
        return np.nan
movie['id'] = movie['id'].apply(convertint)

#Se hace un join de movie con credits por el id de la pelicula
movie = movie.merge(credits, on='id')
movie.shape
movie['cast'] = movie['cast'].fillna('[]').apply(literal_eval).apply(lambda x: [i['id'] for i in x] if isinstance(x, list) else [])
movie.head(5)


#Se agrega en el df movie el id del actor principal de cada pelicula.
act=movie['cast'].values
m = []
for i in range(len(act)):
  if len(act[i]) > 0:
    m.append(act[i][0])
  else:
    m.append('0')

movie.loc[:,'actid'] = m
movie.head(5)


moviedef=movie
movie1 = movie1.merge(credits, on='id')
movie1.shape

movie1['cast'] = movie1['cast'].fillna('[]').apply(literal_eval).apply(lambda x: [i['name'] for i in x] if isinstance(x, list) else [])
#Se agrega en el df moviedef el nombre del actor principal de cada pelicula.
actname=movie1['cast'].values
g = []
for i in range(len(actname)):
  if len(actname[i]) > 0:
    g.append(actname[i][0])
  else:
    g.append('NA')

moviedef.loc[:,'actname'] = g
moviedef.head(5)
moviedef['id'] = moviedef['id'].apply(convertint)
moviedef = moviedef.drop('cast', axis=1)#Borramos la columna cast ya que no la necesitamos
movie['crew'] = movie['crew'].apply(ast.literal_eval)

def get_director(x):#funcion para obtener el id del director de la columna crew y los que no tengan los ponga en null
    for i in x:
        if i['job'] == 'Director':
            return i['id']
    return np.nan
moviedef['directid'] = movie['crew'].apply(get_director)
def get_director(x):
    for i in x:
        if i['job'] == 'Director':
            return i['name']
    return np.nan
moviedef['director'] = movie['crew'].apply(get_director)
moviedef.head()
moviedef = moviedef.drop('crew', axis=1)#Borramos la columna crew ya que obtuvimos el director
def convertint(x):#Funcion para convertir en entero
    try:
        return int(x)
    except:
        return np.nan
moviedef['directid'] = moviedef['directid'].apply(convertint)
moviedef['vote_count'] = moviedef['vote_count'].apply(convertint)
#Crear un vector con los pesos de las caracteristicas el numero de votos contra el voto promedio

vote_counts = moviedef[moviedef['vote_count'].notnull()]['vote_count'].astype('int')
vote_averages = moviedef[moviedef['vote_average'].notnull()]['vote_average'].astype('int')
C = vote_averages.mean()
C
f=vote_counts.quantile(0.80)
f
wvote=moviedef['vote_average'].values
wvote1=moviedef['vote_count'].values
h = []
for i in range(len(wvote)):
  if vote_counts[i] >= vote_counts.quantile(0.9754):
    v = wvote1[i]
    R = wvote[i]
    m=vote_counts.quantile(0.98)
    wr=(v/(v+m) * R) + (m/(m+v) * C)
    h.append(wr)
  elif vote_counts[i] >= vote_counts.quantile(0.95) and vote_counts[i] < vote_counts.quantile(0.9754):
    v = wvote1[i]
    R = wvote[i]
    m=vote_counts.quantile(0.95)
    wr=(v/(v+m) * R) + (m/(m+v) * C)
    wr=wr*0.95
    h.append(wr)
  elif vote_counts[i] >= vote_counts.quantile(0.85) and vote_counts[i] < vote_counts.quantile(0.95)   :
    v = wvote1[i]
    R = wvote[i]
    m=vote_counts.quantile(0.85)
    wr=(v/(v+m) * R) + (m/(m+v) * C)
    wr=wr*0.85
    h.append(wr)
  elif vote_counts[i] >= vote_counts.quantile(0.80) and vote_counts[i] < vote_counts.quantile(0.85)   :
    v = wvote1[i]
    R = wvote[i]
    m=vote_counts.quantile(0.80)
    wr=(v/(v+m) * R) + (m/(m+v) * C)
    wr=wr*0.70
    h.append(wr)
  elif vote_counts[i] < vote_counts.quantile(0.80) :
    h.append(0)
moviedef.loc[:,'vote'] = h
moviedef.head(5)
#creamos un vector con el valor de 0 si la pelicula tiene un promedio de votos mayor de 6 y sino un valor de 1
promvote=moviedef['vote'].values
r = []
for i in range(len(promvote)):
  if promvote[i] >= 6:
    r.append(0)
  else:
    r.append(1)
moviedef.loc[:,'target'] = r
moviedef.head(5)
mdef=moviedef
calificadas = mdef.sort_values('vote', ascending=False).head(20000)
calificadas.head(5)
mdef = mdef.drop('title', axis=1)
#mdef = mdef.drop('vote_count', axis=1)
mdef = mdef.drop('genero', axis=1)
mdef = mdef.drop('compname', axis=1)
mdef = mdef.drop('actname', axis=1)
mdef = mdef.drop('director', axis=1)
mdef = mdef.drop('vote', axis=1)
mdef.head()
calificadas.describe()
calificadas['budget'].describe()
calificadas['budget'] = pd.to_numeric(calificadas['budget'], errors='coerce')
calificadas['budget'] = calificadas['budget'].replace(np.nan, 0)
calificadas['budget'].describe()
calificadas['popularity'].describe()
calificadas['popularity'] = pd.to_numeric(calificadas['popularity'], errors='coerce')
calificadas['popularity'] = calificadas['popularity'].replace(np.nan, 0)
calificadas['popularity'].describe()
calificadas = calificadas.dropna()
#calificadas['count'] = 1
calificadas.groupby(['budget']).count()#['count']
calificadas.groupby(['popularity']).count()#['count']
correlations = calificadas.corr()
print(correlations)
c1=calificadas
c1 = c1.drop('title', axis=1)
#mdef = mdef.drop('vote_count', axis=1)
c1 = c1.drop('genero', axis=1)
c1 = c1.drop('compname', axis=1)
c1 = c1.drop('actname', axis=1)
c1 = c1.drop('director', axis=1)
#c1 = c1.drop('count', axis=1)
X= c1[['popularity', 'vote']].values

fig,axs = plt.subplots(1,2, figsize=(15,4))
axs[0].scatter(X[:,0],X[:,1],s=10)
axs[0].set_title('original')
for max_iter,fig in zip([10000],[1]):
  kmeans_clustering = KMeans(n_clusters=5, max_iter= max_iter, init='random')
  clusters = kmeans_clustering.fit_predict(X)
  #print(clusters)
  axs[fig].set_title('iteraciones '+str(max_iter))
  axs[fig].scatter(X[:,0],X[:,1],c=clusters,s=10)
  clusters_center=kmeans_clustering.cluster_centers_
  axs[fig].scatter(clusters_center[:,0],clusters_center[:,1], marker='*',c='r',s=50)
  
c1["Clus_km"] = clusters
calificadas["Clus_km"] = clusters
c1.head(25)
from sklearn.neighbors import NearestNeighbors # importing the library
x=c1[['popularity', 'vote']].values
neighb = NearestNeighbors(n_neighbors=2) # creating an object of the NearestNeighbors class
nbrs=neighb.fit(x) # fitting the data to the object
distances,indices=nbrs.kneighbors(x) # finding the nearest neighbours
distances = np.sort(distances, axis = 0) # sorting the distances
distances = distances[:, 1] # taking the second column of the sorted distances
plt.rcParams['figure.figsize'] = (5,3) # setting the figure size
plt.plot(distances) # plotting the distances
plt.show()
X= c1[['popularity', 'vote']].values
fig,axs = plt.subplots(1,2, figsize=(15,4))
axs[0].scatter(X[:,0],X[:,1],s=10)
axs[0].set_title('original')
for fig, eps, ms in zip([1], [0.9], [0.01]):
   dbs = DBSCAN(eps =eps, min_samples = ms)
   clustersdb = dbs.fit_predict(X)
  #print(clusters)
   #print(clusters)
   axs[fig].set_title('eps '+str(eps))
   axs[fig].scatter(X[:,0],X[:,1],c=clustersdb,s=10)
c1["Clus_db"] = clustersdb
calificadas["Clus_db"] = clustersdb

from sklearn.cluster import AgglomerativeClustering
fig,axs = plt.subplots(figsize=(15,4))

X= c1[['popularity', 'vote']].values

agl_c = AgglomerativeClustering(n_clusters= 5)
clustersag = agl_c.fit_predict(X)
axs.scatter(X[:,0],X[:,1], c = clustersag)
c1["Clus_ag"] = clustersag
calificadas["Clus_ag"] = clustersag
calificadas.head()

clusag = c1['Clus_ag'].value_counts().reset_index()
clusag.columns = ['Clus_ag', 'numero']
clusag.head(11)
cluskm = c1['Clus_km'].value_counts().reset_index()
cluskm.columns = ['Clus_km', 'numero']
cluskm.head(11)
X1= c1[['vote', 'vote_count']].values

fig,axs = plt.subplots(1,2, figsize=(15,4))
axs[0].scatter(X1[:,0],X1[:,1],s=10)
axs[0].set_title('original')
for max_iter,fig in zip([500],[1]):
  kmeans_clustering1 = KMeans(n_clusters=3, max_iter= max_iter, init='random')
  clusters1 = kmeans_clustering1.fit_predict(X1)
  #print(clusters)
  axs[fig].set_title('iteraciones '+str(max_iter))
  axs[fig].scatter(X1[:,0],X1[:,1],c=clusters1,s=20)
  clusters_center=kmeans_clustering1.cluster_centers_
  axs[fig].scatter(clusters_center[:,0],clusters_center[:,1], marker='*',c='r',s=30)
c1["Clus_km1"] = clusters1
calificadas["Clus_km1"] = clusters1

from sklearn.cluster import AgglomerativeClustering
fig,axs = plt.subplots(figsize=(15,4))

X1= c1[['vote', 'vote_count']].values

agl_c = AgglomerativeClustering(n_clusters= 3)
clustersag = agl_c.fit_predict(X1)
axs.scatter(X1[:,0],X1[:,1], c = clustersag)
c1["Clus_ag1"] = clustersag
calificadas["Clus_ag1"] = clustersag
calificadas.head()
cluskm1 = c1['Clus_km1'].value_counts().reset_index()
cluskm1.columns = ['Clus_km1', 'numero']
cluskm1.head(11)
clusag1 = c1['Clus_ag1'].value_counts().reset_index()
clusag1.columns = ['Clus_ag1', 'numero']
clusag1.head(11)
pop_gen = calificadas['genero'].value_counts().reset_index()
pop_gen.columns = ['genero', 'peliculas']
pop_gen.head(50)
pop_act = calificadas['actname'].value_counts().reset_index()
pop_act.columns = ['actname', 'No peliculas']
pop_act.head(20)
pop_direct = calificadas['director'].value_counts().reset_index()
pop_direct.columns = ['director', 'No_peliculas']
pop_direct.head(20)

pop_comp = calificadas['compname'].value_counts().reset_index()
pop_comp.columns = ['compania', 'No_peliculas']
pop_comp.head(20)

moviedef['popularity'] = pd.to_numeric(moviedef['popularity'], errors='coerce')
moviedef[['title', 'popularity','vote_average','vote_count']].sort_values('vote_count', ascending=False).head(10)

import random
from sklearn.metrics.cluster import adjusted_rand_score, normalized_mutual_info_score, silhouette_score, homogeneity_score
X= c1[['popularity', 'vote']].values
y = c1['target'].values
fig,axs = plt.subplots(1,4, figsize=(15,4))
axs[0].scatter(X[:,0], X[:,1], c=y)
axs[0].set_title('original')
N_class = 5
cluster_methodos = [KMeans(n_clusters=N_class,max_iter=500,init='random'),
                    AgglomerativeClustering(n_clusters=6),
                    DBSCAN(eps=10,min_samples=2)]

nom = ['KMeans', 'AgglomerativeClustering','DBSCAN']
for f, c_m,n in zip(range(3),cluster_methodos,nom):
  cluster=c_m.fit_predict(X)
  axs[f+1].scatter(X[:,0], X[:,1], c=cluster)
  axs[f+1].set_title(format(n))
  print(c_m.__class__.__name__)
  print("NMI: "+"{:.2f}".format(normalized_mutual_info_score(y,cluster)*100))
  print("ARI: "+"{:.2f}".format(adjusted_rand_score(y,cluster)*100))
  print("silhouette: "+"{:.2f}".format(silhouette_score(X,cluster)*100))
  print("Homobeneity: "+"{:.2f}".format(homogeneity_score(y,cluster)*100))
c2=calificadas
c2.head()
def recomendar(genre):
  qualified = c2[(c2['genero'] == genre ) & (c2['Clus_km']==0)][['title', 'vote','vote_count', 'popularity', 'compname', 'director']]
  qualified.head(100)
  return qualified


recomendar('Romance')
def get_recommendations(title):
  df = c2[c2['title'] == title]
  qualified = c2[#(c2['genid'] == df['genid'].values[0] ) & 
                 (c2['Clus_km']==2)& (c2['actid'] == df['actid'].values[0] )][['title','genero','actname', 'vote','vote_count', 'popularity', 'compname', 'director']]
  
  return qualified
 
  
get_recommendations('Fast & Furious 6')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1783823627956845, 'fraction_numerical': 0.026737251222944448, 'mean_word_length': 5.8933025404157044, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '32570', 'license': 'MIT', 'n_tokens_mistral': 6032, 'n_tokens_neox': 5780, 'n_words': 1153, 'path': 'proyectofinal.ipynb', 'repo_name': 'ricacero/Datascience300', 'size': '14923'}