<jupyter_start><jupyter_text>Prerpcessing<jupyter_code># a copy of data
df = df.copy()
# ID is a unique number for each row and not needed for precidtion
df = df.drop('ID', axis =1)<jupyter_output><empty_output><jupyter_text>missing values in shipment methods<jupyter_code>missing_shipment = df[df['Shipment Mode'].isna()].index
df = df.drop(missing_shipment, axis=0).reset_index(drop= True)
df.isna().sum()
df.isna().mean()<jupyter_output><empty_output><jupyter_text>missing values in dosage column<jupyter_code>df.Dosage.nunique()
df.Dosage.mode()
df['Dosage'] = df['Dosage'].fillna(df['Dosage'].mode()[0])
df.isna().sum()<jupyter_output><empty_output><jupyter_text>Line Item Insurance (USD) missin values<jupyter_code>df['Line Item Insurance (USD)'] = df['Line Item Insurance (USD)'].fillna(df['Line Item Insurance (USD)'].mean())
df.isna().sum()<jupyter_output><empty_output><jupyter_text>Date columns<jupyter_code>dates = ['PQ First Sent to Client Date',
        'PO Sent to Vendor Date',
        'Scheduled Delivery Date',
        'Delivered to Client Date',
        'Delivery Recorded Date']
for column in dates:
    print(column,':',pd.to_datetime(df[column], errors = 'coerce').isna().mean().round(2))
# I am going to drop the first two
df = df.drop(['PQ First Sent to Client Date',
        'PO Sent to Vendor Date'], axis = 1)
Dates = ['Scheduled Delivery Date','Delivered to Client Date','Delivery Recorded Date']
for column in Dates:
    df[column] = pd.to_datetime(df[column])
    df[column + ' year'] = df[column].apply(lambda x: x.year)
    df[column + ' month'] = df[column].apply(lambda x: x.month)
    df[column + ' day'] = df[column].apply(lambda x: x.day)
df
df = df.drop(Dates, axis = 1)<jupyter_output><empty_output><jupyter_text>Weight (Kilograms) column<jupyter_code>df['Weight (Kilograms)'].unique()
def notnumeric(df,col):
    counter = 0
    for item in df[col]:
        if item.isnumeric():
            pass
        else:
            counter = counter +1
    return round(counter/ len(df[col]) * 100, 2)
notnumeric(df,'Weight (Kilograms)')
notnumeric(df,'Freight Cost (USD)')
df['Weight (Kilograms)'].apply(lambda x: np.NaN if not x.isnumeric() else x ).isna().mean()
# dropping this column because of many missing values
df = df.drop('Weight (Kilograms)', axis = 1)<jupyter_output><empty_output><jupyter_text>Freight Cost (USD)<jupyter_code>df['Freight Cost (USD)'].apply(lambda x: np.NaN if not x.isnumeric() else x ).isna().mean()
# dropping this column because of many missing values
df = df.drop('Freight Cost (USD)', axis = 1)<jupyter_output><empty_output><jupyter_text>PQ, PO / SO, ASN/DN columns<jupyter_code>cols = ['PQ #','PO / SO #','ASN/DN #']
for item in cols:
    print(item, df[item].isna().mean())<jupyter_output>PQ # 0.0
PO / SO # 0.0
ASN/DN # 0.0<jupyter_text>As this columns have a huge number of unique values without a missing value. if we get dummies out of them we will have thousands of extra columns which could cause a huge processing time and even overfitting. so I am going to drop them.<jupyter_code>df = df.drop(cols, axis =1)
df.info()
df.select_dtypes('object').info()
df.select_dtypes('object')
df.select_dtypes('object').nunique()
df.select_dtypes('float64').nunique()
df.select_dtypes('int64').nunique()<jupyter_output><empty_output><jupyter_text>it seems we have a reasonable number of object variavles and 2 binary variavles. first we can handle those 2<jupyter_code>cols_bin = ['Fulfill Via', 'First Line Designation']
for item in cols_bin:
    print(item, df[item].unique())
df['Fulfill Via'] = df['Fulfill Via'].replace({'Direct Drop':0, 'From RDC':1 })
df['First Line Designation'] = df['First Line Designation'].replace({'Yes':1, 'No':0}) 
df.head()<jupyter_output><empty_output><jupyter_text>One hot encoding<jupyter_code>for column in df.select_dtypes('object').columns.drop('Shipment Mode'):
    dummies = pd.get_dummies(df[column], prefix = column)
    df = pd.concat([df,dummies], axis =1)
    df = df.drop(column, axis =1)
df.shape
df<jupyter_output><empty_output><jupyter_text>Split train and test<jupyter_code>X = df.drop('Shipment Mode', axis =1)
y = df['Shipment Mode']
y.unique()
g = {} 
for item in y:
    if item in g:
        g[item] = g[item]+1
    else:
        g[item] = 1
g
LABEL_MAP = {'Air':0, 'Truck':1, 'Air Charter':2, 'Ocean':3}
y= y.map({'Air':0, 'Truck':1, 'Air Charter':2, 'Ocean':3})
y
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X,y, random_state = 42, test_size = 0.3, shuffle = True)<jupyter_output><empty_output><jupyter_text>Scaling<jupyter_code>from sklearn.preprocessing import StandardScaler
scaler = StandardScaler()
scaler.fit(X_train)
X_train = pd.DataFrame(scaler.transform(X_train), index = X_train.index, columns = X_train.columns)
X_test =  pd.DataFrame(scaler.transform(X_test), index = X_test.index, columns = X_test.columns)<jupyter_output><empty_output><jupyter_text>Training<jupyter_code>X_train.shape
import tensorflow as tf

inputs = tf.keras.Input(shape=(771,))

x = tf.keras.layers.Dense(128, activation='relu')(inputs)
x = tf.keras.layers.Dense(128, activation='relu')(x)

outputs = tf.keras.layers.Dense(4, activation='softmax')(x)

model = tf.keras.Model(inputs=inputs, outputs=outputs)

model.compile(
    optimizer='adam',
    loss='sparse_categorical_crossentropy',
    metrics=['accuracy']
)

history = model.fit(
    X_train,
    y_train,
    validation_split=0.2,
    batch_size=32,
    epochs=100,
    callbacks=[
        tf.keras.callbacks.EarlyStopping(
            monitor='val_loss',
            patience=3,
            restore_best_weights=True
        )
    ]
)
from sklearn.metrics import confusion_matrix, classification_report

y_pred = np.argmax(model.predict(X_test), axis=1)

cm = confusion_matrix(y_test, y_pred, labels=list(LABEL_MAP.values()))
clr = classification_report(y_test, y_pred, labels=list(LABEL_MAP.values()), target_names=list(LABEL_MAP.keys()))

print("Test Set Accuracy: {:.2f}%".format(model.evaluate(X_test, y_test, verbose=0)[1] * 100))

plt.figure(figsize=(8, 8))
sns.heatmap(cm, annot=True, fmt='g', vmin=0, cmap='Blues', cbar=False)
plt.xticks(ticks=[0.5, 1.5, 2.5, 3.5], labels=list(LABEL_MAP.keys()))
plt.yticks(ticks=[0.5, 1.5, 2.5, 3.5], labels=list(LABEL_MAP.keys()))
plt.xlabel("Predicted")
plt.ylabel("Actual")
plt.title("Confusion Matrix")
plt.show()

print("Classification Report:\n----------------------\n", clr)<jupyter_output>Test Set Accuracy: 90.47%
{'fraction_non_alphanumeric': 0.16037590509936836, 'fraction_numerical': 0.014789708827607457, 'mean_word_length': 5.248315688161694, 'pattern_counts': {'":': 0, '<': 55, '<?xml version=': 0, '>': 55, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '214061', 'license': 'MIT', 'n_tokens_mistral': 2459, 'n_tokens_neox': 2361, 'n_words': 520, 'path': 'shpment method prediction.ipynb', 'repo_name': 'MSadriAghdam/Supply-Chain-Prediction_Neural-Network-ML', 'size': '6491'}