<jupyter_start><jupyter_text>AI-Enabled FinTech B2B Invoice Management Application  Importing the libraries<jupyter_code>import numpy as np
import pandas as pd
import datetime as dt
from scipy import stats

import seaborn as sns
import matplotlib.pyplot as plt
%matplotlib inline 
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LinearRegression
from sklearn import metrics
from sklearn.ensemble import RandomForestRegressor<jupyter_output><empty_output><jupyter_text>Importing the dataset<jupyter_code>df= pd.read_csv('C:\\Users\shrey\Downloads\H2HBABBA3224.csv',error_bad_lines=False)     #Read the Dataset
df = pd.DataFrame(df)   #Make it dataframe 
df.columns  
df.head()            # To see each column of Dataset of first 5 columns.<jupyter_output><empty_output><jupyter_text>Data Pre-processing<jupyter_code>df.info()          # To see the description of each column of Dataset.
df.isnull().sum()            # To see number of NULL values in each attribute/ column
df1=df[['business_code','cust_number','name_customer','clear_date','buisness_year','doc_id','posting_date','document_create_date','due_in_date','invoice_currency','document type','posting_id','total_open_amount','baseline_create_date','cust_payment_terms','invoice_id','isOpen']]
# Made new Dataframe except the columns which we do not want.
df1.invoice_id.duplicated().sum()              # To find the number of duplicate values in "invoice_id" attribute.
df1.shape                                   # To see shape of Data frame.
df1.drop_duplicates(subset ="invoice_id",keep = 'first', inplace = True)              
# dropped the duplicate values from the "invoice_id" attribute because invoice_id always have unique value 
df1.invoice_id.duplicated().sum()              #To check the number of Duplicate values in "invoice_id".
df1.dropna(subset=['invoice_id'], inplace=True)        # To Remove NULL values from "invoice_id" attribute.
df1.isnull().sum()                       # To see number of NULL values in each attribute/ column
df1              # To see the dataset
df1.info()                       # To see the information of each column.
df1=df1.reset_index(drop=True)          # To reset the Index of the Dataframe and "drop=true" dropping the old index.
df1<jupyter_output><empty_output><jupyter_text>Heat Map of all the columns<jupyter_code>correlation = df1.corr()
sns.heatmap(correlation, xticklabels = correlation.columns, yticklabels=correlation.columns , annot=True)<jupyter_output><empty_output><jupyter_text>Heat Map of 4 Columns<jupyter_code>df10 = df1[['document_create_date','baseline_create_date','due_in_date','total_open_amount']]
correlation = df10.corr()
sns.heatmap(correlation, xticklabels = correlation.columns, yticklabels=correlation.columns , annot=True)<jupyter_output><empty_output><jupyter_text>Converting dates columns/ attribute to date time format<jupyter_code>df1['document_create_date']=pd.to_datetime(df1['document_create_date'].astype(str), format='%Y%m%d')              #Coverting date from integer format to datetime format          
df1['due_in_date']=pd.to_datetime(df1['due_in_date'].astype(str), format='%Y%m%d.0')                              #Coverting date from float format to datetime format
df1['baseline_create_date']=pd.to_datetime(df1['baseline_create_date'].astype(str), format='%Y%m%d.0')            #Coverting date from float format to datetime format
df1['clear_date']=pd.to_datetime(df1.clear_date)                                                                  #Coverting date from object format to datetime format
df1['posting_date']=pd.to_datetime(df1.posting_date)                                                              #Coverting date from object format to datetime format
df1
df1.dtypes                     # Datatypes of all the attributes
df1['invoice_id'] = df1['invoice_id'].astype('int64')      #Converting Datatype from float to integer.
df1['doc_id'] = df1['doc_id'].astype('int64')            #Converting Datatype from float to integer.
df1['buisness_year'] = df1['buisness_year'].astype('int64')   #Converting Datatype from float to integer.
df1<jupyter_output><empty_output><jupyter_text>Splitting the Dataset into train and Real world dataset<jupyter_code>td = df1.loc[df1['isOpen'] == 0]      #Splitting the Data into Train dataset where "isopen=0" means "cleardate" is given.
train_df = pd.DataFrame(td)
train_df
td1 = df1.loc[df1['isOpen'] == 1]    #Splitting the Data into Test dataset where "isopen=1" means "cleardate" is given or NULL.
test_df = pd.DataFrame(td1)
test_df
train_df.sort_values("posting_date",ascending=True)         # Sorting the Training Dataset using "posting_date" in ascending order means oldest data first.<jupyter_output><empty_output><jupyter_text>Adding New Target Column<jupyter_code>train_df['delay'] =  train_df['clear_date'] - train_df['due_in_date']      #Inserting New column named "delay" which is the difference of days between "clear_date" and "due_in_date"
train_df['delay']=train_df['delay'].dt.days       # concerting the Delay into days format
#Q1 = train_df['delay'].quantile(0.25)
#Q3 = train_df['delay'].quantile(0.75)
#IQR = Q3 - Q1    #IQR is interquartile range. 

#filter = (train_df['delay'] >= Q1 - 5.0 * IQR) & (train_df['delay'] <= Q3 + 5.0 *IQR)
#train_df=train_df.loc[filter] 
train_df
train_df.info()
print(pd.to_datetime(train_df['document_create_date']).min())       #First Date 
print(pd.to_datetime(train_df['document_create_date']).max())       #Earliest Date   
train_df['isOpen'].unique                 # Unique Values in isOpen
print(dict(train_df['business_code'].value_counts()))         # Unique Values int that Attribute with their Count
# Unique Values int that Attribute with their Count

print(dict(train_df['invoice_currency'].value_counts()))
print(dict(train_df['isOpen'].value_counts()))
print(dict(train_df['document type'].value_counts()))
print(dict(train_df['posting_id'].value_counts()))
print(dict(train_df['cust_payment_terms'].value_counts()))
train_df.describe()
train_df = train_df.drop(['posting_id', 'document type'], axis = 1)       # Dropping 2 columns posting_id and document_type
train_df.head()
sns.pairplot(train_df)       #Pairpots using Seaborn<jupyter_output><empty_output><jupyter_text>Exploratory data analysis(EDA)<jupyter_code>#Graph between business_code , delay
temp = pd.DataFrame(train_df.groupby(['business_code'], axis=0, as_index=False)['delay'].mean())
plt.figure(figsize=(10,6))
sns.barplot(x="business_code", y="delay",data=temp,linewidth=2.5, facecolor=(1, 1, 1, 0),
                 errcolor=".4", edgecolor="Blue")<jupyter_output><empty_output><jupyter_text>Encoding business code column<jupyter_code># It is clear that U002 afterwards all business codes are meager so we take it as others.

train_df = train_df.replace(['U002', 'U005', 'U007'], 'other')
business = pd.get_dummies(train_df['business_code']) 
train_df = pd.concat([train_df, business], axis=1)
train_df.head()<jupyter_output><empty_output><jupyter_text>Encoding invoice_currency column<jupyter_code>train_df['invoice_currency'] = train_df['invoice_currency'].map({'USD': 0,'CAD': 1})<jupyter_output><empty_output><jupyter_text>Exploratory data analysis(EDA)<jupyter_code>#Graph between invoice_currency and delay
temp1 = pd.DataFrame(train_df.groupby(['invoice_currency'], axis=0, as_index=False)['delay'].mean())
plt.figure(figsize=(6,4))
sns.barplot(x="invoice_currency", y="delay",data=temp1,linewidth=1.5, facecolor=(1, 1, 1, 0),
                 errcolor=".2", edgecolor="blue")
#Graph of Total_open_Amount
plt.figure(figsize=(8,8))
plt.figure(1)
sns.distplot(train_df['total_open_amount'],color='green')
#Heatmap
correlation = train_df.corr()
sns.heatmap(correlation, xticklabels = correlation.columns, yticklabels=correlation.columns , annot=True)<jupyter_output><empty_output><jupyter_text>Feature Engineering<jupyter_code># Log Transformation Function and applying to 'total_open_amount' attribute
def Transform(x):
    return np.log(x)

train_df['total_open_amount'] = train_df['total_open_amount'].apply(Transform)
train_df['cust_number'].nunique()        # Number of unique values in customer column
# Label Encode Function
from sklearn.preprocessing import LabelEncoder
import numpy as np
class LabelEncoderExt(object):  #asssigning 0 for unknown
    def __init__(self):
        self.label_encoder = LabelEncoder()

    def fit(self, data_list):
        self.label_encoder = self.label_encoder.fit(list(data_list) + ['Unknown'])
        self.classes_ = self.label_encoder.classes_
        return self

    def transform(self, data_list):
        new_data_list = list(data_list)
        for unique_item in np.unique(data_list):
            if unique_item not in self.label_encoder.classes_:
                new_data_list = ['Unknown' if x==unique_item else x for x in new_data_list]
        return self.label_encoder.transform(new_data_list)
#Encoding Cust_number
custnumber_enc=LabelEncoderExt()
custnumber_enc.fit(train_df['cust_number'])
train_df['cust_number_enc']=custnumber_enc.transform(train_df['cust_number'])
# Dropping 3 columns which are not in use
train_df = train_df.drop([ 'cust_number','due_in_date', 'isOpen'], axis = 1)
train_df.head(1)
#distplot of Delay 
plt.figure(figsize=(6,6))
plt.figure(2)
sns.distplot(train_df['delay'],color='blue')
#Made new column of quarter from baseline create date
train_df['baseline_quater']= pd.to_datetime(train_df['baseline_create_date']).dt.quarter<jupyter_output><empty_output><jupyter_text>Encoding  Customer Payment terms<jupyter_code>train_df = train_df.replace(['NAA8', 'NAAX', 'NAAW'], 'NAA')
train_df = train_df.replace(['NAGD', 'NAUZ', 'NAVF', 'NAVE', 'NAVR', 'NAWN', 'NAVQ',
                                  'NAWP', 'NAVM', 'NACB', 'NABG','NAVD', 'NAVL', 'NAUP', 'NATJ', 'NATH', 
                                  'NATU', 'NA8Q', 'NAVC', 'NANC', 'NATK', 'NAUI', 'NAWM',
                                  'NATL', 'NAUY', 'NATW', 'NATX', 'NAWU','NATV','NAUX','NATM','NATY','NACE','NATZ','NAUN'], 'NA_place')

train_df = train_df.replace(['NAH4', 'NAC6', 'NAM4', 'NAD1', 'NAU5', 'NAG2', 'NAX2', 'NAD5', 'NAM1',
                                  'NAD4','NAB1','NAM2','NAM3', 'NAV9',  'NAV2', 'NAD8'], 'NA_letter_number')

train_df = train_df.replace(['NA3B', 'NA3F', 'NA9X'], 'NA_number_letter')

train_df = train_df.replace(['NA32', 'NA10', 'NA38', 'NA84', 'NA25', 'NA31','NA39'], 'NA_number_number')

train_df = train_df.replace(['CA10', 'CA30', 'C106', 'CAX2', 'CAB1', 'C129', 'CA60','90M7'], 'CA')


train_df = train_df.replace(['BR56', 'B030', 'B052', 'BR12', 'B112','BR60'], 'B')

train_df['cust_payment_terms'].value_counts()
payment = pd.get_dummies(train_df['cust_payment_terms'])

# In the cust_payment_terms column we can see they are categorical ones, so we make new feature
X_final = pd.concat([train_df, payment], axis=1)

X_final
# Heatmap
plt.figure(figsize=(15,15))
sns.heatmap(X_final.corr(),annot=True,cmap='viridis',linewidths=.5)<jupyter_output><empty_output><jupyter_text>Feature Selection<jupyter_code>X_final = X_final.drop(['doc_id', 'invoice_id','business_code', 'name_customer', 'clear_date',
                       'posting_date', 'document_create_date', 'baseline_create_date', 'cust_payment_terms',
                       'invoice_id'], axis = 1)
X_final<jupyter_output><empty_output><jupyter_text>Splitting the Dataset into Training and Testing<jupyter_code>y = X_final['delay']
X_final = X_final.drop(['delay'], axis = 1)
Xtrain ,Xtest, ytrain, ytest = train_test_split(X_final, y, test_size = 0.2, random_state=0)
y
l = list(np.isfinite(ytrain))
print([x for x in l if x == False])
lp = list(np.isfinite(Xtrain))
print([x for x in lp if x == False])
Xtrain.info()
ytrain<jupyter_output><empty_output><jupyter_text>Training the Model<jupyter_code>random_model = RandomForestRegressor(n_estimators=300)
#from sklearn.linear_model import LogisticRegression
#classifier = LogisticRegression(random_state = 0)
#classifier.fit(Xtrain, ytrain)
random_model.fit(Xtrain, ytrain)
# Predicting the output for Test Dataset
y_pred3 = random_model.predict(Xtest)

#Checking the accuracy
#random_model_accuracy = round(random_model.score(Xtrain, ytrain)*100,2)
#print(round(random_model_accuracy,2),'%')
#y_pred = classifier.predict(Xtest)
print(y_pred3)
y_pred3 = y_pred3.astype('int64')<jupyter_output><empty_output><jupyter_text>Evaluation of the Model<jupyter_code>from sklearn.metrics import accuracy_score
print(accuracy_score(ytest, y_pred3))
from sklearn.metrics import mean_squared_error
mean_squared_error(ytest, y_pred3,squared=False)
y_out = random_model.predict(Xtest)
y_out= y_out.astype('int64')
y_out[0:3]
from sklearn.metrics import mean_absolute_error
from sklearn.metrics import r2_score
from sklearn.metrics import mean_absolute_error
from sklearn.metrics import explained_variance_score
def evaluate_metrics(x, y_true,y_pred, mod):
    mean_abs_error = mean_absolute_error(y_true,y_pred)
    mean_sq_error = mean_squared_error(y_true,y_pred)
    root_mean_sq_error = mean_squared_error(y_true,y_pred)**0.5
    r2_scr = r2_score(y_true,y_pred)
    explained_variance = explained_variance_score(y_true,y_pred)
    return mean_abs_error,mean_sq_error,root_mean_sq_error,r2_scr,explained_variance
print(evaluate_metrics(Xtest, ytest , y_pred3 , RandomForestRegressor))
print(r2_score(ytest, y_pred3))<jupyter_output>0.3682727406480647<jupyter_text>Mapping<jupyter_code>def Mapping(x):
    temp = []
    for i in list(x):
        if i < 0:
          temp.append("Payment done before due date.")
        elif i >= 0 and i < 15:
          temp.append("Payment done within 15 days")
        elif i >=16 and i<=30:
          temp.append("Payment done after 15 days and within 30 days")
        elif i >=31 and i<=45:
          temp.append("Payment done after 30 days and within 45 days")
        elif i>=46 and i<=60:
          temp.append("Payment done after 30 days and within 45 days")
        else:
          temp.append("Payment done after 60 days")
    return temp

col2 = Mapping(y_out)
bucket = pd.DataFrame()
bucket['Predictions'] = y_out
bucket['Bucket_list'] = col2
bucket<jupyter_output><empty_output><jupyter_text>Predicting Delay and Clear_Date Real World Problem<jupyter_code>test_df
print(dict(test_df['business_code'].value_counts()))
print(dict(test_df['invoice_currency'].value_counts()))
print(dict(test_df['isOpen'].value_counts()))
print(dict(test_df['document type'].value_counts()))
print(dict(test_df['posting_id'].value_counts()))
print(dict(test_df['cust_payment_terms'].value_counts()))
test_df1 = test_df.drop(['posting_id', 'document type'], axis = 1)
test_df1['invoice_currency'] = test_df1['invoice_currency'].map({'USD': 0,'CAD': 1})
def Transform(x):
    return np.log(x)

test_df1['total_open_amount'] = test_df1['total_open_amount'].apply(Transform)
custnumber_test_enc=LabelEncoderExt()
custnumber_test_enc.fit(test_df1['cust_number'])
test_df1['cust_number_test_enc']=custnumber_test_enc.transform(test_df1['cust_number'])
test_df1 = test_df1.drop([ 'cust_number','due_in_date', 'isOpen'], axis = 1)
test_df1['baseline_quater']= pd.to_datetime(test_df1['baseline_create_date']).dt.quarter
test_df1 = test_df1.replace(['NAA8', 'NAAX', 'NAAW'], 'NAA')
test_df1 = test_df1.replace(['NAGD', 'NAUZ', 'NAVF', 'NAVE', 'NAVR', 'NAWN', 'NAVQ',
                                  'NAWP', 'NAVM', 'NACB', 'NABG','NAVD', 'NAVL', 'NAUP', 'NATJ', 'NATH', 
                                  'NATU', 'NA8Q', 'NAVC', 'NANC', 'NATK', 'NAUI', 'NAWM',
                                  'NATL', 'NAUY', 'NATW', 'NATX', 'NAWU','NATV','NAUX','NATM','NATY',
                           'NACE','NATZ','NAUN','NACH','NACG','NAUW','NABD'], 'NA_place')

test_df1 = test_df1.replace(['NAH4', 'NAC6', 'NAM4', 'NAD1', 'NAU5', 'NAG2', 'NAX2', 'NAD5', 'NAM1',
                                  'NAD4','NAB1','NAM2','NAM3', 'NAV9',  'NAV2', 'NAD8'], 'NA_letter_number')

test_df1 = test_df1.replace(['NA3B', 'NA3F', 'NA9X'], 'NA_number_letter')

test_df1 = test_df1.replace(['NA32', 'NA10', 'NA38', 'NA84', 'NA25', 'NA31','NA39'], 'NA_number_number')

test_df1 = test_df1.replace(['CA10', 'CA30', 'C106', 'CAX2', 'CAB1', 'C129', 'CA60','90M7'], 'CA')


test_df1 = test_df1.replace(['BR56', 'B030', 'B052', 'BR12', 'B112','BR60'], 'B')

test_df1['cust_payment_terms'].value_counts()
test_df1 = test_df1.replace(['U002', 'U005', 'U007'], 'other')
business = pd.get_dummies(test_df1['business_code']) 
test_df1 = pd.concat([test_df1, business], axis=1)
test_df1
payment = pd.get_dummies(test_df1['cust_payment_terms'])

# Above in the cust_payment_terms column we can see they are categorical ones, so we make new feature
# out of them 
X_test2 = pd.concat([test_df1, payment], axis=1)

X_test2
 X_test2 = X_test2.drop(['doc_id', 'invoice_id','business_code', 'name_customer', 'clear_date',
                       'posting_date', 'document_create_date', 'baseline_create_date', 'cust_payment_terms',
                       'invoice_id'], axis = 1)
X_test2
X_test2['delay']=random_model.predict(X_test2)
test_df['delay']=X_test2['delay'].astype('int')
test_df['clear_date'] = test_df['due_in_date'] + pd.to_timedelta(test_df['delay'], unit='d')
test_df['delay'].value_counts()
def Mapping(x):
    temp = []
    for i in list(x):
        if i < 0:
          temp.append("Payment done before due date.")
        elif i >= 0 and i < 15:
          temp.append("Payment done within 15 days")
        elif i >=16 and i<=30:
          temp.append("Payment done after 15 days and within 30 days")
        elif i >=31 and i<=45:
          temp.append("Payment done after 30 days and within 45 days")
        elif i>=46 and i<=60:
          temp.append("Payment done after 30 days and within 45 days")
        else:
          temp.append("Payment done after 60 days")
    return temp

col2 = Mapping(test_df['delay'])
bucket = pd.DataFrame()
bucket['Predictions'] = test_df['delay']
bucket['Bucket_list'] = col2
bucket
test_df['Bucket_list']=bucket['Bucket_list']
test_df
test_df['delay'].value_counts()
test_df.to_csv(r"C:\Users\shrey\Downloads\babba.csv")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14142526709990086, 'fraction_numerical': 0.02467232074016962, 'mean_word_length': 4.987141444114738, 'pattern_counts': {'":': 0, '<': 91, '<?xml version=': 0, '>': 89, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 15, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '20', 'dir': 'jupyter-structured-clean-dedup', 'id': '401661', 'license': 'MIT', 'n_tokens_mistral': 6605, 'n_tokens_neox': 6239, 'n_words': 1281, 'path': 'milestone3.ipynb', 'repo_name': 'ShreyasG07/AI-Enabled-Fintech-B2B-Invoice-Management-Application', 'size': '18158'}