<jupyter_start><jupyter_text>check sales value from dealer and individual each<jupyter_code>seller_sales = df.groupby('Seller_Type')['Selling_Price'].sum()
print(seller_sales)<jupyter_output>Seller_Type
Dealer        1310.73
Individual      92.32
Name: Selling_Price, dtype: float64<jupyter_text>check sales unit from dealer and individual each<jupyter_code>seller_sales = df.groupby('Seller_Type')['Selling_Price'].count() #or agg('count'), also could add other operation
print(seller_sales)<jupyter_output>Seller_Type
Dealer        195
Individual    106
Name: Selling_Price, dtype: int64<jupyter_text>Check missing value or null<jupyter_code>df.isnull().sum()
df.describe()<jupyter_output><empty_output><jupyter_text>create new feature, age of the car, based on 'year' feature<jupyter_code>list(df) #or df.columns
#remove car name

final_dataset = df[['Year',
 'Selling_Price',
 'Present_Price',
 'Kms_Driven',
 'Fuel_Type',
 'Seller_Type',
 'Transmission',
 'Owner']].copy()
final_dataset.head()
final_dataset['Current_Year'] = 2020
final_dataset.head()
final_dataset['No_Years'] = final_dataset['Current_Year']-final_dataset['Year']
final_dataset.head()
final_dataset.drop(['Year'], axis=1, inplace=True) #axis 1 column, 0 row
 final_dataset.head()
final_dataset.drop(['Current_Year'], axis=1, inplace=True)
final_dataset.head()<jupyter_output><empty_output><jupyter_text>Manage categorical feature<jupyter_code>#one hot encoding
final_dataset = pd.get_dummies(final_dataset, drop_first=True) #drop first column, prevent dummy variable trap
final_dataset.head()<jupyter_output><empty_output><jupyter_text>Correlation between features<jupyter_code>final_dataset.corr()
#plot the correlation
import seaborn as sns
sns.pairplot(final_dataset)
#plot as heatmap
import matplotlib.pyplot as plt
%matplotlib inline

corrmat = final_dataset.corr()
top_corr_features = corrmat.index
plt.figure(figsize=(20,20))

#plot the heatmap

g = sns.heatmap(final_dataset[top_corr_features].corr(), annot=True, cmap="RdGy_r") #RdY1Gn<jupyter_output><empty_output><jupyter_text>X,Y, Split train, test, valid<jupyter_code>#independent feature
X = final_dataset.iloc[:,1:]

#dependent feature, predict target
y = final_dataset.iloc[:,0]

X.head()
y.head()
#feature importance
from sklearn.ensemble import ExtraTreesRegressor

model = ExtraTreesRegressor()
model.fit(X,y)
print(model.feature_importances_)
#plot the feature importance for convenience

feat_importances = pd.Series(model.feature_importances_, index=X.columns)
feat_importances.nlargest(5).plot(kind='barh')
plt.show()
#split data
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X,y, test_size=0.2) 
X_train.head()
X_train.shape
from sklearn.ensemble import RandomForestRegressor
rf_random = RandomForestRegressor()
#hyperparameter to be used
#use RandomizedSearchCV to search for best parameters value
import numpy as np
from sklearn.model_selection import RandomizedSearchCV #vs grid search cv?

#number of trees in random forest
n_estimators = [int(x) for x in np.linspace(start=100, stop=1200, num=12)]
print(n_estimators)
#continue...
#number of features to consider at every split
max_features = ['auto', 'sqrt']

#maximum number of levels on tree
max_depth = [int(x) for x in np.linspace(5,30, num=6)]
#max_depth.append(None)

#Minimum number of samples required to split a node
min_samples_split = [2,5,10,15,100]

#minimum number of samples required at each leaf node
min_samples_leaf = [1,2,5,10]
#Create random grid
random_grid = {'n_estimators' : n_estimators,
               'max_features' : max_features,
              'max_depth' : max_depth,
              'min_samples_split' : min_samples_split,
              'min_samples_leaf' : min_samples_leaf}

print(random_grid)
#So now lets use the random grid to search for best hyperparameters
#first create the base model to tune
rf = RandomForestRegressor()
rf_random = RandomizedSearchCV(estimator=rf, param_distributions=random_grid, scoring='neg_mean_squared_error', n_iter=10, cv=5, verbose=2, random_state=42, n_jobs=1)
rf_random.fit(X_train, y_train)
predictions = rf_random.predict(X_test)
predictions
 #use distplot to display price prediction
sns.distplot(y_test-predictions)    
 #distplot shows the difference, nominal normal distribution for y_test - pedictions, means the model is giving good result
#scatter plot to show if the result is linear means the prediction is good
plt.scatter(y_test, predictions)<jupyter_output><empty_output><jupyter_text>DEPLOY<jupyter_code>#dump the model using pickle
import pickle

#set where we want to store the data set the file name
file = open('random_forest_regression_model.pkl', 'wb') #write bytes mode

#dump the info to file
pickle.dump(rf_random, file)<jupyter_output><empty_output><jupyter_text>then create requirement file in cli or in this kernel such as:<jupyter_code>#!pip freeze > requirements.txt 
#but the best is in cli bcos we have to choose the env, else it will list all packages inside base env
#instead of working env
#but if we install packages via conda only, use conda list --export > requirements.txt
#to install requirements.txt via conda : conda create --name <envname> --file requirements.txt
#!conda list --export > requirements.txt<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10103675777568331, 'fraction_numerical': 0.01357210179076343, 'mean_word_length': 6.701015965166909, 'pattern_counts': {'":': 0, '<': 36, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '239395', 'license': 'MIT', 'n_tokens_mistral': 1829, 'n_tokens_neox': 1755, 'n_words': 469, 'path': 'Car Prediction Car dekho 2021.ipynb', 'repo_name': 'allifizzuddin89/cardekho-carprice-prediction', 'size': '5305'}