<jupyter_start><jupyter_text>EDD There are missing values and outliers in _Time_taken_<jupyter_code>sns.boxplot(y="Time_taken", data=df)<jupyter_output><empty_output><jupyter_text>There are outliers in _Collection_<jupyter_code>sns.boxplot(y="Collection", data=df)<jupyter_output><empty_output><jupyter_text>There are outliers in _Marketing expense_<jupyter_code>sns.boxplot(y="Marketing expense", data=df)<jupyter_output><empty_output><jupyter_text>There are outliers in _Trailer_views_<jupyter_code>sns.boxplot(y="Trailer_views", data=df)<jupyter_output><empty_output><jupyter_text>There are outliers in _Twitter_hastags_<jupyter_code>sns.boxplot(y="Twitter_hastags", data=df)<jupyter_output><empty_output><jupyter_text>There are outliers in _Num_multiplex_<jupyter_code>sns.boxplot(y="Num_multiplex", data=df)<jupyter_output><empty_output><jupyter_text>Categorical values<jupyter_code>sns.countplot(y="Start_Tech_Oscar", data=df)
sns.countplot(y="3D_available", data=df)<jupyter_output><empty_output><jupyter_text>_MPAA_film_rating_ is constant<jupyter_code>sns.countplot(y="MPAA_film_rating", data=df)
sns.countplot(y="Genre", data=df)<jupyter_output><empty_output><jupyter_text>Removing outliers<jupyter_code>lv_collection = np.percentile(df.Collection, [5])[0]
uv_collection = np.percentile(df.Collection, [90])[0]
df.Collection[(df.Collection < lv_collection)] = lv_collection
df.Collection[(df.Collection > uv_collection)] = uv_collection
df.Collection.describe()
sns.boxplot(y="Collection", data=df)
uv_marketing_expense = np.percentile(df["Marketing expense"], [87])[0]
uv_marketing_expense
df["Marketing expense"][(df["Marketing expense"] > uv_marketing_expense)] = uv_marketing_expense
sns.boxplot(y="Marketing expense", data=df)
df["Marketing expense"].describe()
lv_trailer_views = np.percentile(df.Trailer_views, [2])[0]
lv_trailer_views
df.Trailer_views[(df.Trailer_views < lv_trailer_views)] = lv_trailer_views
sns.boxplot(y="Trailer_views", data=df)
uv_twitter = np.percentile(df.Twitter_hastags, [99])[0]
df.Twitter_hastags[(df.Twitter_hastags > uv_twitter)] = uv_twitter
sns.boxplot(y="Twitter_hastags", data=df)
uv_num_multiplex = np.percentile(df.Num_multiplex, [99])[0]
df.Num_multiplex[(df.Num_multiplex > uv_num_multiplex)] = uv_num_multiplex
sns.boxplot(y="Num_multiplex", data=df)<jupyter_output><empty_output><jupyter_text>Missing values<jupyter_code>df.info()
df = df.fillna(df.mean())
df.info()<jupyter_output><empty_output><jupyter_text>Constant values<jupyter_code>del df["MPAA_film_rating"]
df.info()<jupyter_output><empty_output><jupyter_text>Transform Data Main character rating<jupyter_code>df["main_character_ratin"] = (df.Lead_Actress_rating + df["Lead_ Actor_Rating"]) / 2
del df["Lead_Actress_rating"]
del df["Lead_ Actor_Rating"]
df.main_character_ratin.describe()<jupyter_output><empty_output><jupyter_text>Dummy Variables<jupyter_code>df = pd.get_dummies(df)
df.describe()
del df["3D_available_NO"]
del df["Genre_Thriller"]
df.describe()<jupyter_output><empty_output><jupyter_text>Training Models  Logistic Regression<jupyter_code>from sklearn.linear_model import LogisticRegression
from sklearn.metrics import confusion_matrix, precision_score, recall_score, roc_auc_score
X = df.loc[:, df.columns != 'Start_Tech_Oscar']
y = df['Start_Tech_Oscar']
logistic_regression = LogisticRegression(max_iter = 2000)
logistic_regression.fit(X, y)
logistic_regression.coef_
logistic_regression.intercept_
y_pred_03 = (logistic_regression.predict_proba(X)[:, 1] >= 0.3).astype(bool)
y_pred_05 = (logistic_regression.predict_proba(X)[:, 1] >= 0.5).astype(bool)
y_pred_07 = (logistic_regression.predict_proba(X)[:, 1] >= 0.7).astype(bool)<jupyter_output><empty_output><jupyter_text>0.3<jupyter_code>confusion_matrix(y, y_pred_03)
precision_score(y, y_pred_03)
recall_score(y, y_pred_03)
roc_auc_score(y, y_pred_03)<jupyter_output><empty_output><jupyter_text>0.5<jupyter_code>confusion_matrix(y, y_pred_05)
precision_score(y, y_pred_05)
recall_score(y, y_pred_05)
roc_auc_score(y, y_pred_05)<jupyter_output><empty_output><jupyter_text>0.7<jupyter_code>confusion_matrix(y, y_pred_07)
precision_score(y, y_pred_07)
recall_score(y, y_pred_07)
roc_auc_score(y, y_pred_07)<jupyter_output><empty_output><jupyter_text>We can conclude that 0.5 provide us the best model  Linear Discriminant Analysis<jupyter_code>from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
linear_discriminant_analysis = LinearDiscriminantAnalysis()
linear_discriminant_analysis.fit(X, y)
y_pred_lda_05 = linear_discriminant_analysis.predict(X)
confusion_matrix(y, y_pred_lda_05)
roc_auc_score(y, y_pred_lda_05)
confusion_matrix(y, y_pred_05)
roc_auc_score(y, y_pred_05)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12754668930390492, 'fraction_numerical': 0.01697792869269949, 'mean_word_length': 13.281818181818181, 'pattern_counts': {'":': 0, '<': 75, '<?xml version=': 0, '>': 80, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '18', 'dir': 'jupyter-structured-clean-dedup', 'id': '56297', 'license': 'MIT', 'n_tokens_mistral': 1958, 'n_tokens_neox': 1850, 'n_words': 218, 'path': 'jupyter/classification_models/main.ipynb', 'repo_name': 'juandausa/machinelearning', 'size': '4712'}