<jupyter_start><jupyter_text>Data   TaskTo predict the product sales based on the historical records.<jupyter_code>data = pd.read_csv("SalesKaggle3.csv")
data
data.info()
data['SoldFlag'].value_counts()<jupyter_output><empty_output><jupyter_text>Data Preprocessing - We will remove all the unique identifiers like Order, SKU_number etc.- We gonna use a tree-based model so no need to scale the data.<jupyter_code>def data_preparation(df):
  df = df.copy()

  #only using "Historical" records in File_Type column using query() function
  df = df.query("File_Type == 'Historical'")

  """drop unnecessary columns i.e Unique Identifiers and also File_Type as we only have Horizontal records.
  Remove SoldCount because it is a function for our target SoldFlag i.e we can't give hint to our model so better
  to remove it.
  """
  df = df.drop(["Order", "File_Type", "SKU_number", "SoldCount"], axis=1)

  #Split the data
  X = df.drop("SoldFlag", axis=1)
  y = df['SoldFlag']

  X_train, X_test, y_train, y_test = train_test_split(X,y, train_size=0.7, shuffle=True, random_state=1)

  return X_train, X_test, y_train, y_test
 X_train, X_test, y_train, y_test = data_preparation(data)
print(len(X_train))
print(len(X_test))
print(len(y_train))
print(len(y_test))
X_train
y_test<jupyter_output><empty_output><jupyter_text>Build the Pipeline for Model Training - One hot encoder for MarketType and ReleaseNumber.- Pipeline for defining the steps<jupyter_code>def model_pipeline():
    binary_transformer = Pipeline(steps=[
        ('onehot', OneHotEncoder(sparse=False, drop='if_binary'))
    ])
    
    nominal_transformer = Pipeline(steps=[
        ('onehot', OneHotEncoder(sparse=False, handle_unknown='ignore'))
    ])
    
    preprocessor = ColumnTransformer(transformers=[
        ('binary', binary_transformer, ['MarketingType']),
        ('nominal', nominal_transformer, ['ReleaseNumber'])
    ], remainder='passthrough')
    
    model = Pipeline(steps=[
        ('preprocessor', preprocessor),
        ('classifier', RandomForestClassifier(random_state=1))
    ])
    
    return model
!pip3 install -U scikit-learn
train_model = model_pipeline()
clf = train_model.fit(X_train, y_train)<jupyter_output><empty_output><jupyter_text>Evaluation<jupyter_code>y_train_pred = clf.predict(X_train)
y_test_pred = clf.predict(X_test)
# Evaluation on the train set
print("Evaluation on training set \n")
print("--------------------------------------")
model_train_f1 = f1_score(y_train, y_train_pred, average='binary')
print("The f1-score for the training set is: ", np.round(model_train_f1*100), "%")
print("--------------------------------------")

model_train_score = clf.score(X_train, y_train)
print("Model score on the train set is: ", np.round(model_train_score*100), "%")
print("--------------------------------------")

model_train_mcc = matthews_corrcoef(y_train, y_train_pred)
print("Model MCC on the test set is: ", np.round(model_train_mcc*100), "%")
# Evaluation on the train set
print("Evaluation on the test set \n")
print("--------------------------------------")
model_test_f1 = f1_score(y_test, y_test_pred, average='binary')
print("The f1-score for the test set is: ", np.round(model_test_f1*100), "%")
print("--------------------------------------")

model_test_score = clf.score(X_test, y_test)
print("Model score on the test set is: ", np.round(model_test_score*100), "%")

print("--------------------------------------")

model_test_mcc = matthews_corrcoef(y_test, y_test_pred)
print("Model MCC on the test set is: ", np.round(model_test_mcc*100), "%")
plot_confusion_matrix(clf, X_test, y_test, labels=clf.classes_)
clr = classification_report(y_test, y_pred)
print(clr)<jupyter_output>precision    recall  f1-score   support

         0.0       0.85      0.98      0.91     18923
         1.0       0.55      0.15      0.24      3876

    accuracy                           0.83     22799
   macro avg       0.70      0.56      0.57     22799
weighted avg       0.80      0.83      0.79     22799
{'fraction_non_alphanumeric': 0.17077420960916107, 'fraction_numerical': 0.02539208364451083, 'mean_word_length': 3.288153681963714, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '609905', 'license': 'MIT', 'n_tokens_mistral': 1375, 'n_tokens_neox': 1314, 'n_words': 322, 'path': 'MCC_Evaluation/MCC.ipynb', 'repo_name': 'AhmedMohsenElgarh/Machine-Learning-Models-Implementation', 'size': '4017'}