<jupyter_start><jupyter_text>Load data<jupyter_code>column_items = ['movie id', 'movie title', 'release date', 'video release date', 'IMDb URL', 'unknown', 'Action', 'Adventure', 'Animation', "Children's", 'Comedy', 'Crime', 'Documentary', 'Drama', 'Fantasy', 'Film-Noir', 'Horror', 'Musical', 'Mystery', 'Romance', 'Sci-Fi','Thriller', 'War', 'Western']
items = pd.read_csv('ml-100k/u.item', delimiter='|',names=column_items, header = None, encoding='latin-1')
items.head()
genres = items.iloc[:, 5:]
genres.index = items['movie id']
genres.head()<jupyter_output><empty_output><jupyter_text>Dimension reduction adn classificationNow we try to classify the data. Since every movie has several genres, we choose to select first appeared genre as its only feature. We compare the dimension reduction ability of PCA, isomap and tsne.<jupyter_code>genre_arr = genres.to_numpy()
genre_arr[0]
new_genre_arr = np.zeros((len(np.unique(items['movie id'])), len(genre_arr[0])))
for i in range(len(items['movie id'])):
    if np.sum(genre_arr[i]) ==1:
            new_genre_arr[i] = genre_arr[i]
    
    else:
        if np.sum(genre_arr[i]) >1:
            for j in range(len(genre_arr[0])):
                if genre_arr[i,j]==1:
                    new_genre_arr[i,j]=1
                    break
np.sum(new_genre_arr, axis = 1)
y_type = []
for i in range(len(items['movie id'])):
    for j in range(len(new_genre_arr[0])):
        if new_genre_arr[i,j]==1:
            y_type += [j]
y_type = np.asarray(y_type)
classes = np.unique(y_type)
classes
from sklearn.decomposition import PCA
from sklearn.manifold import TSNE, Isomap

# This cell can take a few minutes to run
run_this_cell = True

if run_this_cell:
    # In 2d
    embeddings = [PCA(n_components=2, copy=True, whiten=True, tol=1e-5),
                  Isomap(n_components=2, n_neighbors=5),
                  TSNE(n_components=2)]

    for embedding in embeddings:
        X_embedded = embedding.fit_transform(new_genre_arr)
        fig = plt.figure()
        for i in classes:
            mask = y_type == i
            plt.scatter(X_embedded[mask, 0], X_embedded[mask, 1], label=i)
        plt.legend()
        plt.title('Embedding method: '+ type(embedding).__name__)
        plt.show()

    # In 3d
    embeddings = [PCA(n_components=3, copy=True, whiten=True, tol=1e-5),
                  Isomap(n_components=3, n_neighbors=5),
                  TSNE(n_components=3)]

    for embedding in embeddings:
        X_embedded = embedding.fit_transform(new_genre_arr)
        fig = plt.figure()
        ax = Axes3D(fig)
        for i in classes:
            mask = y_type == i
            ax.scatter(X_embedded[mask, 0], X_embedded[mask, 1], X_embedded[mask, 2], label=i)
        ax.legend()
        ax.title.set_text('Embedding method: '+ type(embedding).__name__)
        plt.show()<jupyter_output><empty_output><jupyter_text>From the graphs above, we can notice tsne did the best job, so we apply this method for further steps. Besides, it is easy to see the points that form a cluster normally close to each other and a cluster is usually far away from another oen, so we apply K-means for classification.<jupyter_code>plt.figure(figsize=(10,8))
embedding = TSNE(n_components=2)
X_embedded = embedding.fit_transform(new_genre_arr)
km = KMeans(n_clusters=19)
y_pred = km.fit(X_embedded).labels_.astype(int)

for i in classes:
    mask = y_pred == i
    plt.scatter(X_embedded[mask, 0], X_embedded[mask, 1], label=i)

plt.xlim(-40, 60)
plt.legend(loc='upper right', shadow=True, fontsize='x-large')
plt.title("Visualization of movie clusters classified by K-means")
plt.show()


plt.figure(figsize=(10,8))
for i in classes:
    mask = y_type == i
    plt.scatter(X_embedded[mask,0], X_embedded[mask, 1], label=i)
    
plt.legend(loc='upper right', shadow=True, fontsize='x-large')
plt.xlim(-40, 60)
plt.title("Visualization of true movie clusters")
plt.show()
np.unique(y_pred)
plt.figure(figsize=(10,8))
embedding = TSNE(n_components=2)
X_embedded = embedding.fit_transform(new_genre_arr)
km = KMeans(n_clusters=19)
y_pred = km.fit(X_embedded).labels_.astype(int)

fig, ax = plt.subplots(1, 2, figsize=(18,8)) 
scatter1 = ax[0].scatter(X_embedded[:,0], X_embedded[:,1], c=y_type, cmap=plt.cm.get_cmap("Spectral")) 
ax[0].scatter(X_embedded[:,0], X_embedded[:,1], c=y_pred, cmap=plt.cm.get_cmap("Spectral")) 
ax[0].set_title("Visualization of movie clusters classified by K-means", fontsize=10) 
ax[0].legend(*scatter1.legend_elements(),loc='upper right', shadow=True, fontsize='x-large')

scatter = ax[1].scatter(X_embedded[:,0], X_embedded[:,1], c=y_type, cmap=plt.cm.get_cmap("Spectral")) 
ax[1].scatter(X_embedded[:,0], X_embedded[:,1], c=y_type, cmap=plt.cm.get_cmap("Spectral")) 
ax[1].set_title("Visualization of true movie clusters", fontsize=10) 
ax[1].legend(*scatter.legend_elements(),loc='upper right', shadow=True, fontsize='x-large')
plt.show()
import scipy
def find_permutation(n_clusters, real_labels, labels):
    permutation=[]
    for i in range(n_clusters):
        idx = labels == i
        new_label=scipy.stats.mode(real_labels[idx])[0][0]  # Choose the most common label among data points in the cluster
        permutation.append(new_label)
    return permutation
permutation = find_permutation(19, y_type, km.labels_)
print(permutation)
from sklearn.metrics import accuracy_score
new_labels = [ permutation[label] for label in km.labels_]   # permute the labels
print("Accuracy score is", accuracy_score(y_type, new_labels))
# Import `metrics` from `sklearn`
from sklearn import metrics

# Print out the confusion matrix with `confusion_matrix()`
print(metrics.confusion_matrix(y_type, y_pred))
from sklearn.metrics import homogeneity_score, completeness_score, v_measure_score, adjusted_rand_score, adjusted_mutual_info_score, silhouette_score
print('% 9s' % 'inertia   homo    compl  v-meas     ARI     AMI   silhouette')
print(' %i     %.3f   %.3f   %.3f    %.3f   %.3f   %.3f'
          %(km.inertia_,
      homogeneity_score(y_type, y_pred),
      completeness_score(y_type, y_pred),
      v_measure_score(y_type, y_pred),
      adjusted_rand_score(y_type, y_pred),
      adjusted_mutual_info_score(y_type, y_pred),
      silhouette_score(X_embedded, y_pred, metric='euclidean')))
from matplotlib.colors import ListedColormap

ListedColormap(['r','b','g'])
scatter.legend_elements()
from sklearn.datasets import load_iris
iris = load_iris()
features = iris.data.T

plt.scatter(features[0], features[1], alpha=0.2,
            s=100*features[3], c=iris.target, cmap='viridis')
plt.xlabel(iris.feature_names[0])
plt.ylabel(iris.feature_names[1]);
iris.target
y_pred
rng = np.random.RandomState(0)
for marker in ['o', '.', ',', 'x', '+', 'v', '^', '<', '>', 's', 'd']:
    plt.plot(rng.rand(5), rng.rand(5), marker,
             label="marker='{0}'".format(marker))
plt.legend(numpoints=1)
plt.xlim(0, 1.8);<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14568579274461627, 'fraction_numerical': 0.017054487642722937, 'mean_word_length': 4.314900153609831, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '371794', 'license': 'MIT', 'n_tokens_mistral': 2625, 'n_tokens_neox': 2482, 'n_words': 544, 'path': 'exploration/classification.ipynb', 'repo_name': 'dinotuku/ntds-2019-project-team-7', 'size': '6919'}