<jupyter_start><jupyter_text>Importing Libraries<jupyter_code>import pandas as pd 
import numpy as np 
from sklearn.ensemble import RandomForestRegressor
from sklearn.model_selection import train_test_split
import ast
import json
from numpy import nan
import lightgbm as lgb
from sklearn.preprocessing import MinMaxScaler<jupyter_output><empty_output><jupyter_text>Pathing to Data<jupyter_code>data_train = '/home/vbgupta/Desktop/School/ds310/TMDB-Box-Office-Predictions/train.csv'
train = pd.read_csv(data_train)
print(len(train))
data_test = '/home/vbgupta/Desktop/School/ds310/TMDB-Box-Office-Predictions/test.csv'
test = pd.read_csv(data_test)

train.tail()
test.head()<jupyter_output><empty_output><jupyter_text>Preprocessing1. Combine the training and test sets for easier and better Preprocessing.2. Figure out what the columns that can impact the revenue a movie makes3. Look at the column typ and makes necessary changes to it. Training Set and Testing Set<jupyter_code>full_data = pd.concat([train, test], axis = 0)
print(full_data.shape )
full_data.head()
dataframe = full_data.loc[:,['budget', 'original_language', 'popularity', 'runtime']]
dataframe = dataframe.fillna(0)
dataframe['original_language'] = dataframe['original_language'].astype('category')
dataframe['original_language'] = dataframe['original_language'].cat.codes

# Using the main dataframe `full_data` for this column
full_data['genres'] = full_data['genres'].astype('str')
dataframe.head()


for i in range(len(full_data.genres)):

  try:
    x = eval(full_data.genres[i])
    genres = [sub['name'] for sub in x]
    full_data.genres[i] = genres
  except:
    pass
genr = full_data['genres'].copy()
dataframe = pd.concat([dataframe, genr], axis = 1)
dataframe.head()
dataframe['genres'] = dataframe['genres'].astype(str)
dataframe['genres'] = dataframe['genres'].str.split(",",n = 3, expand = True)
dataframe['genres'] = dataframe['genres'].replace('"[', '').replace(']"', '').replace("'", '')
dataframe['genres'] = dataframe['genres'].str.strip('[]').astype('category')
dataframe['genres'] = dataframe['genres'].cat.codes
dataframe.head()
y = train['revenue']
print(len(y))
y.head()<jupyter_output>3000<jupyter_text>Lets now separate the dataframe in X_train and X_test<jupyter_code>X_train = dataframe.iloc[:3000, :]
X_test = dataframe.iloc[3000:, :]
# Filling in the missing values in `runtime` column

X_train['runtime'].fillna(X_train['runtime'].median(), inplace = True)
X_test['runtime'].fillna(X_test['runtime'].median(), inplace = True)<jupyter_output>/home/vbgupta/Desktop/School/ds310/school_env/lib/python3.8/site-packages/pandas/core/series.py:4463: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  return super().fillna(
/home/vbgupta/Desktop/School/ds310/school_env/lib/python3.8/site-packages/pandas/core/series.py:4463: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  return super().fillna(<jupyter_text>Implimenting MinMaxScaler<jupyter_code>scaler = MinMaxScaler()
X_train = scaler.fit_transform(X_train)
X_train = pd.DataFrame(X_train, columns= dataframe.columns)
X_train.head()
scaler = MinMaxScaler()
X_test = scaler.fit_transform(X_test)
X_test = pd.DataFrame(X_test, columns= dataframe.columns)
X_test.head()
print("Training Set Length is:", len(X_train))
print("Testing Set Length is:", len(X_test))<jupyter_output>Training Set Length is: 3000
Testing Set Length is: 4398<jupyter_text>ML Model to Predict `revenue`<jupyter_code># RandomForestRegressor
rf = RandomForestRegressor(random_state=44)
rf.fit(X_train, y)
y_preds_rf = rf.predict(X_test)
y_preds_rf # RSMLE = 2.6<jupyter_output><empty_output><jupyter_text>Hyperparameter Tuning `RandomForest`<jupyter_code>params = {'bootstrap': [True, False],
 'max_depth': [10, 20, 30, None],
 'max_features': ['auto', 'sqrt'],
 'min_samples_leaf': [1, 2, 4],
 'min_samples_split': [2, 5],
 'n_estimators': [200, 400, 600, 800]}

from sklearn.model_selection import GridSearchCV

rf_gs = GridSearchCV(RandomForestRegressor(n_jobs=-1, random_state=44), param_grid=params, cv = 10, verbose=True, refit= True)
rf_gs.fit(X_train, y)
y_preds_rf_gs = rf_gs.predict(X_test)
y_preds_rf_gs<jupyter_output>Fitting 10 folds for each of 1512 candidates, totalling 15120 fits
ERROR:root:Internal Python error in the inspect module.
Below is the traceback from this internal error.

Exception in thread Thread-5:
Traceback (most recent call last):
  File "/usr/lib/python3.8/threading.py", line 932, in _bootstrap_inner
    self.run()
  File "/home/vbgupta/Desktop/School/ds310/school_env/lib/python3.8/site-packages/joblib/externals/loky/process_executor.py", line 558, in run
    self.terminate_broken(bpe)
  File "/home/vbgupta/Desktop/School/ds310/school_env/lib/python3.8/site-packages/joblib/externals/loky/process_executor.py", line 736, in terminate_broken
    self.kill_workers()
  File "/home/vbgupta/Desktop/School/ds310/school_env/lib/python3.8/site-packages/joblib/externals/loky/process_executor.py", line 766, in kill_workers
    recursive_terminate(p)
  File "/home/vbgupta/Desktop/School/ds310/school_env/lib/python3.8/site-packages/joblib/externals/loky/backend/utils.py", line 28, in recursi[...]<jupyter_text>Creating a validation set for LGBMRegression Model<jupyter_code># Creating a validation set

X_train, X_val, y_train, y_val = train_test_split(X_train, y, test_size=0.2)
params = {'objective':'regression',
         'num_leaves' : 30,
         'min_data_in_leaf' : 20,
         'max_depth' : 9,
         'learning_rate': 0.004,
         #'min_child_samples':100,
         'feature_fraction':0.9,
         "bagging_freq": 1,
         "bagging_fraction": 0.9,
         'lambda_l1': 0.2,
         "bagging_seed": 11,
         "metric": 'rmse',
         #'subsample':.8, 
          #'colsample_bytree':.9,
         "random_state" : 11,
         "verbosity": -1}
record = dict()
model = lgb.train(params
                      , lgb.Dataset(X_train, y_train)
                      , num_boost_round = 100000
                      , valid_sets = [lgb.Dataset(X_val, y_val)]
                      , verbose_eval = 500
                      , early_stopping_rounds = 500
                      , callbacks = [lgb.record_evaluation(record)]
                     )
best_idx = np.argmin(np.array(record['valid_0']['rmse']))

test_pred = model.predict(X_test, num_iteration = model.best_iteration)# LGBMRegressor
test_pred<jupyter_output>Training until validation scores don't improve for 500 rounds
[500]	valid_0's rmse: 8.27828e+07
[1000]	valid_0's rmse: 7.98227e+07
[1500]	valid_0's rmse: 7.99821e+07
Early stopping, best iteration is:
[1098]	valid_0's rmse: 7.974e+07<jupyter_text>Creating Submission File<jupyter_code>preds = pd.DataFrame(y_preds_rf_gs, columns=['revenue'])
submission = pd.concat([test['id'], preds], axis=1)
submission.head()
submission.to_csv('/home/vbgupta/Desktop/School/ds310/submission.csv', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13389178797033782, 'fraction_numerical': 0.031447404559187035, 'mean_word_length': 5.064113238967527, 'pattern_counts': {'":': 5, '<': 32, '<?xml version=': 0, '>': 32, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '380397', 'license': 'MIT', 'n_tokens_mistral': 2795, 'n_tokens_neox': 2560, 'n_words': 508, 'path': 'TMDB.ipynb', 'repo_name': 'vbgupta/TMDB-Box-Office-Predictions', 'size': '7282'}