<jupyter_start><jupyter_text>Download data from https://www.kaggle.com/deepmatrix/imdb-5000-movie-dataset<jupyter_code>data_frame = pd.read_csv('data/2_2.csv')
data_frame.head()
# helper method for categorical feature processing
def categorize_columns(data_frame, keys_to_catorize):
    headers = data_frame.columns.tolist()
    for header in headers:
        if header not in keys_to_catorize:
            continue
        one_hot = pd.get_dummies(data_frame[header], prefix=header)
        data_frame = data_frame.drop(header, axis=1)
        data_frame = data_frame.join(one_hot)
    return data_frame
# process data into features

keys_to_keep = ['budget', 'duration', 'facenumber_in_poster', 'imdb_score', 'title_year']
keys_to_drop = [k for k in data_frame.keys() if k not in keys_to_keep]
keys_to_catorize = ['content_rating', 'country']

data_frame = categorize_columns(data_frame, keys_to_catorize)
data_frame = data_frame.fillna(0.0)
keys_to_drop = [k for k in keys_to_drop if k in data_frame.keys()]
trimmed_data_frame = data_frame.drop(keys_to_drop, axis=1)
features = np.array(trimmed_data_frame)
genres = list(data_frame['genres'])
durations = list(data_frame['duration'])
features = preprocessing.scale(features)

Y = []
X = []
genre_map = {}
for i in range(len(genres)):
    genre = genres[i]
    x = features[i,:]
    g_rep = None
    genre_map[genre] = genre_map.get(genre, 0) + 1
    if 'Horror' in genre:
        g_rep = 0
    elif 'Action' in genre:
        g_rep = 1
    elif 'Romance' in genre:
        g_rep = 2
    elif 'Family' in genre:
        g_rep = 3
    if g_rep is not None and durations[i] > 1.0:
        X.append(x)
        Y.append(g_rep)

X = np.array(X)
Y = np.array(Y)
pd.DataFrame(Y, dtype="category").describe()
X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=1/4., random_state=0)
print(X_train.shape)
print(Y_train.shape)
svm_model = svm.SVC(kernel='rbf', C=1.0)
svm_model.fit(X_train, Y_train)

Y_train_pred = svm_model.predict(X_train)
Y_test_pred = svm_model.predict(X_test)

train_mse = sklearn.metrics.mean_squared_error(Y_train, Y_train_pred)
test_mse = sklearn.metrics.mean_squared_error(Y_test, Y_test_pred)
print("Train MSE {}".format(train_mse))
print("Test MSE {}".format(test_mse))

train_accuracy = sklearn.metrics.accuracy_score(Y_train, Y_train_pred)
test_accuracy = sklearn.metrics.accuracy_score(Y_test, Y_test_pred)
print("Train Accuracy {}%".format(train_accuracy*100.0))
print("Test Accuracy {}%".format(test_accuracy*100))
svm_model.predict(X[:2,:])
cf_matrix = confusion_matrix(Y_test, Y_test_pred)
# adapted from:
# https://github.com/scikit-learn/scikit-learn/blob/master/examples/model_selection/plot_confusion_matrix.py
def plot_confusion_matrix(cm, classes,
                          normalize=False,
                          title='Confusion matrix',
                          cmap=plt.cm.Blues):
    """
    This function prints and plots the confusion matrix.
    Normalization can be applied by setting `normalize=True`.
    """
    if normalize:
        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
        print("Normalized confusion matrix")
    else:
        print('Confusion matrix, without normalization')

    print(cm)

    plt.imshow(cm, interpolation='nearest', cmap=cmap)
    plt.title(title)
    plt.colorbar()
    tick_marks = np.arange(len(classes))
    plt.xticks(tick_marks, classes, rotation=45)
    plt.yticks(tick_marks, classes)

    fmt = '.2f' if normalize else 'd'
    thresh = cm.max() / 2.
    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
        plt.text(j, i, format(cm[i, j], fmt),
                 horizontalalignment="center",
                 color="white" if cm[i, j] > thresh else "black")

    plt.ylabel('True label')
    plt.xlabel('Predicted label')
    plt.show()
    
plot_confusion_matrix(cf_matrix, classes=["Horror", "Action", "Romance", "Family"],
                      title='Confusion matrix, without normalization')
 
plot_confusion_matrix(cf_matrix, classes=["Horror", "Action", "Romance", "Family"],
                      title='Confusion matrix, normalization', normalize=True)
plt.show()<jupyter_output>Normalized confusion matrix
[[ 0.46715328  0.23357664  0.24817518  0.05109489]
 [ 0.11406844  0.47148289  0.27376426  0.14068441]
 [ 0.1372549   0.22352941  0.50588235  0.13333333]
 [ 0.01086957  0.07608696  0.09782609  0.81521739]]
{'fraction_non_alphanumeric': 0.1292732623952909, 'fraction_numerical': 0.04097803939325334, 'mean_word_length': 3.7917570498915403, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 6, 'https://': 2, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '571897', 'license': 'MIT', 'n_tokens_mistral': 1705, 'n_tokens_neox': 1584, 'n_words': 313, 'path': 'Sec2/2.2 Notebook.ipynb', 'repo_name': 'PacktPublishing/Machine-learning-with-Sci-kit-Learn-and-Tensorflow-V-', 'size': '4417'}