<jupyter_start><jupyter_text>Data Processing<jupyter_code># Seperating the data into features and what is to be predicted
# Data preprocessing


"""
We can use .drop() to remove the specified cols and then use dummies to covert the named cols into 0s and 1s
and remove the rest

X = data.drop(['Survived', 'Ticket', 'Name'], axis =1)    
X = pd.get_dummies(X[features])

"""

# if we directly use get_dummies. It will convert the named cols into 0s & 1s, and automaticaly 
# remove the rest....

features = ["Pclass", "Sex", "SibSp", "Parch","Fare"]

X = pd.get_dummies(data[features])

Y = data['Survived']                                     # Y is the ground truth to be predicted

X
# Splitting data into train and test set

X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size = 0.2, random_state = 42)

#Applying Feature scaling to Fare  :: Puting Feature Scaling of Age under trail

sc = StandardScaler()   # Standard Scaler
mms = MinMaxScaler()    # Min Max Scaler

X_test[['Fare']] = mms.fit_transform(X_test[['Fare']])
X_train[['Fare']] = mms.fit_transform(X_train[['Fare']])
X_train<jupyter_output>/Users/samarpan/opt/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:6: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  
/Users/samarpan/opt/anaconda3/lib/python3.7/site-packages/pandas/core/indexing.py:965: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  self.obj[item] = s
/Users/samarpan/opt/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:7: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead[...]<jupyter_text>Using Random Forest Classifier<jupyter_code>#predict  

rfc = RandomForestClassifier(n_estimators = 15)
rfc.fit(X_train, Y_train)

pred_rfc = rfc.predict(X_test)




# accuracy report
# pred_rfc uses the X_test and predicts vals of Y__test and we also have the Y-test(original) and we thus can 
# test them against each oter


print(accuracy_score(Y_test, pred_rfc))
print(classification_report(Y_test, pred_rfc))<jupyter_output>0.7653631284916201
              precision    recall  f1-score   support

           0       0.74      0.91      0.82       105
           1       0.82      0.55      0.66        74

    accuracy                           0.77       179
   macro avg       0.78      0.73      0.74       179
weighted avg       0.78      0.77      0.75       179<jupyter_text>COURSE TO FINE testing to find out the ideal value of n_estimator<jupyter_code>##### 15 seems to be the best for n_estimator

report = []
estimator = []

i = 100
while(i <= 5000):
    if(i % 100 == 0):
        
        temp = i
        rfc = RandomForestClassifier(n_estimators = temp)
        rfc.fit(X_train, Y_train)
        pred_rfc = rfc.predict(X_test)
        report.append(accuracy_score(Y_test, pred_rfc)) 
        estimator.append(temp)
        
        i+=100
        



     
plt.plot(estimator, report)
plt.show()<jupyter_output><empty_output><jupyter_text>Part to be submitted<jupyter_code>test_data = pd.read_csv("test.csv")




XX_test = pd.get_dummies(test_data[features])
#print(XX_test)

XX_test[['Fare']] = mms.transform(XX_test[['Fare']])


print(XX_test.isnull().sum())

                             # Y is the ground truth to be predicted


    
    
    
#rfc = RandomForestClassifier(n_estimators=15,max_features="sqrt",verbose=True)



my_imputer = SimpleImputer()
XX_test = my_imputer.fit_transform(XX_test)
print(X_test.shape)

#X_train = mms.fit_transform(X_train)
#rfc.fit(X_train, Y_train)

#print(XX_test)
predictions = rfc.predict(XX_test)

output = pd.DataFrame({'PassengerId': test_data.PassengerId, 'Survived': predictions})
output.to_csv('my_submission.csv', index=False)
print("Your submission was successfully saved!")<jupyter_output>Pclass        0
SibSp         0
Parch         0
Fare          1
Sex_female    0
Sex_male      0
dtype: int64
(179, 6)
Your submission was successfully saved!
{'fraction_non_alphanumeric': 0.10558170813718896, 'fraction_numerical': 0.02869311813494732, 'mean_word_length': 3.361681329423265, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 14, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '283797', 'license': 'MIT', 'n_tokens_mistral': 1685, 'n_tokens_neox': 1584, 'n_words': 390, 'path': 'Titanic problem using Scikit Learn.ipynb', 'repo_name': 'SamarpanDas/Titanic---Problem_Kaggle', 'size': '4461'}