<jupyter_start><jupyter_text>Global Function Text Classification<jupyter_code>import pandas as pd
import numpy as np
import nltk
import string
from matplotlib import pyplot as plt
%matplotlib inline
import re
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import KFold, cross_val_score, train_test_split, GridSearchCV, cross_val_predict
from sklearn.metrics import precision_recall_fscore_support as score
from sklearn.metrics import classification_report, confusion_matrix
import seaborn as sns
import warnings
warnings.filterwarnings('always') 
#reading the file
global_func= pd.read_csv("globalfunc_16aug2021.csv", engine="python")

#printing top 5 rows
global_func.head()
#checking the shape of the df
global_func.shape
print(global_func.columns)#we have two columns
print()
print(global_func.category.unique()) #we have seven unique categories 
#counting total number of descriptions in each category
print("Observations in Epidemic_Pandemic_Prep={}".format(len(global_func[global_func["category"]=="Epidemic_Pandemic_Prep"])))
print("Observations in Knowledge_Gen_Sharing={}".format(len(global_func[global_func["category"]=="Knowledge_Gen_Sharing"])))
print("Observations in not_global_func={}".format(len(global_func[global_func["category"]=="not_global_func"])))
print("Observations in Health_Advocacy={}".format(len(global_func[global_func["category"]=="Health_Advocacy"])))
print("Observations in Control_Cross_Border_Disease={}".format(len(global_func[global_func["category"]=="Control_Cross_Border_Disease"])))
print("Observations in AMR_Response={}".format(len(global_func[global_func["category"]=="AMR_Response"])))
print("Observations in Response_Unhealthy_Products={}".format(len(global_func[global_func["category"]=="Response_Unhealthy_Products"])))

#creating a list of number of observation in each category
cat_size=[len(global_func[global_func["category"]=="Epidemic_Pandemic_Prep"]),
             len(global_func[global_func["category"]=="Knowledge_Gen_Sharing"]),
             len(global_func[global_func["category"]=="not_global_func"]),
             len(global_func[global_func["category"]=="Health_Advocacy"]),
             len(global_func[global_func["category"]=="Control_Cross_Border_Disease"]),
             len(global_func[global_func["category"]=="AMR_Response"]),
             len(global_func[global_func["category"]=="Response_Unhealthy_Products"])]

print(cat_size)

#gettign the unique categories in the data frame
label_cat=global_func.category.unique()

print(label_cat)
#creating a df of cat_size and cat_label

df={"label_cat":label_cat,
    "cat_size":cat_size
   }
df=pd.DataFrame(df)
df
#create a sorted dataframe
df_sorted=df.sort_values("cat_size")
#plotting the observation cat
plt.figure(figsize=(14,6))
plt.bar(df_sorted["label_cat"], df_sorted["cat_size"])
plt.xticks(rotation=90)

for i, v in enumerate(df_sorted["cat_size"]):
    plt.text(i, v, str(v))

#we can see there is huge data imbalance between various categories. 
#Not a global function has a lot of data than the other categories
#This will be amjor problem when we run our model.
# There is very high probability that our model will become biased due to the large imbalance in the data
#checking the percentage of each categories
print("Percentage of Epidemic_Pandemic_Prep={}%".
      format(round(len(global_func[global_func["category"]=="Epidemic_Pandemic_Prep"])/(global_func.shape[0]),4)*100))
print("Percentage of Knowledge_Gen_Sharing={}%".
      format(round(len(global_func[global_func["category"]=="Knowledge_Gen_Sharing"])/(global_func.shape[0]),3)*100))
print("Percentage of not_global_func={}%".
      format(round(len(global_func[global_func["category"]=="not_global_func"])/(global_func.shape[0]),4)*100))
print("Percentage of Health_Advocacy={}%".
      format(round(len(global_func[global_func["category"]=="Health_Advocacy"])/(global_func.shape[0]),4)*100))
print("Percentage of Control_Cross_Border_Disease={}%".
      format(round(len(global_func[global_func["category"]=="Control_Cross_Border_Disease"])/(global_func.shape[0]),3)*100))
print("Percentage of AMR_Response={}%".
      format(round(len(global_func[global_func["category"]=="AMR_Response"])/(global_func.shape[0]),4)*100))
print("Percentage of Response_Unhealthy_Products={}%".
      format(round(len(global_func[global_func["category"]=="Response_Unhealthy_Products"])/(global_func.shape[0]),4)*100))<jupyter_output>Percentage of Epidemic_Pandemic_Prep=7.33%
Percentage of Knowledge_Gen_Sharing=15.6%
Percentage of not_global_func=53.93%
Percentage of Health_Advocacy=10.73%
Percentage of Control_Cross_Border_Disease=9.4%
Percentage of AMR_Response=2.19%
Percentage of Response_Unhealthy_Products=0.91%<jupyter_text>Features engineering<jupyter_code>#creating feature for text message length
global_func["body_len"]=global_func["description"].apply(lambda x: len(x)-x.count(" "))
global_func.head()
#creating feature of number of sentences in the text
def count_sent(text):
    return len(nltk.sent_tokenize(text))

global_func["body_sent"]=global_func["description"].apply(lambda x: count_sent(x))

global_func.head()
#creating feature for % of punctuation in the text message

def count_punt(text):
    count= sum([1 for char in text if char in string.punctuation])
    return round(count/(len(text)-text.count(" ")), 3)*100

global_func["punct%"]=global_func["description"].apply(lambda x: count_punt(x))

global_func.head()
#lets evaluate the new features that we created and see if any of these need any transformation
#looking at text length
bins=np.linspace(0, 2500, 60)
plt.figure(figsize=(14,10))
plt.subplot(331)
plt.hist(global_func[global_func["category"]=="Epidemic_Pandemic_Prep"]["body_len"], bins, density=True, label="Epidemic_Pandemic_Prep")
plt.legend(loc="upper right")

plt.subplot(332)
plt.hist(global_func[global_func["category"]=="Knowledge_Gen_Sharing"]["body_len"], bins, density=True, label="Knowledge_Gen_Sharing")
plt.legend(loc="upper right")

plt.subplot(333)
plt.hist(global_func[global_func["category"]=="not_global_func"]["body_len"],  bins, density=True, label="not_global_func")
plt.legend(loc="upper right")

plt.subplot(334)
plt.hist(global_func[global_func["category"]=="Health_Advocacy"]["body_len"],  bins, density=True, label="Health_Advocacy")
plt.legend(loc="upper right")

plt.subplot(335)
plt.hist(global_func[global_func["category"]=="Control_Cross_Border_Disease"]["body_len"], bins, density=True, label="Control_Cross_Border_Disease")
plt.legend(loc="upper right")

plt.subplot(336)
plt.hist(global_func[global_func["category"]=="AMR_Response"]["body_len"], bins, density=True, label="AMR_Response")
plt.legend(loc="upper right")

plt.subplot(337)
plt.hist(global_func[global_func["category"]=="Response_Unhealthy_Products"]["body_len"], bins, density=True, label="Response_Unhealthy_Products")
plt.legend(loc="upper right")


plt.show();<jupyter_output><empty_output><jupyter_text>It doesn't look like there is major difference in the length of text for each category. It almost similaryly distributed with a lot of text in each category with length below 500, and the a long tail. Since, we have a lot of data from "not_global_function" category so we can see a right skew very clearly.Since there is this right skew, we will normalize this so that the model does not pay undue attention to these long text.<jupyter_code>#looking at punctuation percentage
bins=np.linspace(0, 100, 40)
plt.figure(figsize=(14,10))

plt.subplot(331)
plt.hist(global_func[global_func["category"]=="Epidemic_Pandemic_Prep"]["punct%"], bins, density=True, label="Epidemic_Pandemic_Prep")
plt.legend(loc="upper right")

plt.subplot(332)
plt.hist(global_func[global_func["category"]=="Knowledge_Gen_Sharing"]["punct%"], bins, density=True, label="Knowledge_Gen_Sharing")
plt.legend(loc="upper right")

plt.subplot(333)
plt.hist(global_func[global_func["category"]=="not_global_func"]["punct%"],  bins, density=True, label="not_global_func")
plt.legend(loc="upper right")

plt.subplot(334)
plt.hist(global_func[global_func["category"]=="Health_Advocacy"]["punct%"],  bins, density=True, label="Health_Advocacy")
plt.legend(loc="upper right")

plt.subplot(335)
plt.hist(global_func[global_func["category"]=="Control_Cross_Border_Disease"]["punct%"], bins, density=True, label="Control_Cross_Border_Disease")
plt.legend(loc="upper right")

plt.subplot(336)
plt.hist(global_func[global_func["category"]=="AMR_Response"]["punct%"], bins, density=True, label="AMR_Response")
plt.legend(loc="upper right")

plt.subplot(337)
plt.hist(global_func[global_func["category"]=="Response_Unhealthy_Products"]["punct%"], bins, density=True, label="Response_Unhealthy_Products")
plt.legend(loc="upper right")


plt.show();<jupyter_output><empty_output><jupyter_text>We can see from the above histogram pf punctuation that there is not much of a difference in the punctuation use among various categories. Also, here is not skew. So, no transoformation will be required for this feature. However, we will still leave this feature for model to learn something<jupyter_code>#looking at number of sentence

bins=np.linspace(0, 25, 40)
plt.figure(figsize=(14,10))

plt.subplot(331)
plt.hist(global_func[global_func["category"]=="Epidemic_Pandemic_Prep"]["body_sent"], bins, density=True, label="Epidemic_Pandemic_Prep")
plt.legend(loc="upper right")

plt.subplot(332)
plt.hist(global_func[global_func["category"]=="Knowledge_Gen_Sharing"]["body_sent"], bins, density=True, label="Knowledge_Gen_Sharing")
plt.legend(loc="upper right")

plt.subplot(333)
plt.hist(global_func[global_func["category"]=="not_global_func"]["body_sent"],  bins, density=True, label="not_global_func")
plt.legend(loc="upper right")

plt.subplot(334)
plt.hist(global_func[global_func["category"]=="Health_Advocacy"]["body_sent"],  bins, density=True, label="Health_Advocacy")
plt.legend(loc="upper right")

plt.subplot(335)
plt.hist(global_func[global_func["category"]=="Control_Cross_Border_Disease"]["body_sent"], bins, density=True, label="Control_Cross_Border_Disease")
plt.legend(loc="upper right")

plt.subplot(336)
plt.hist(global_func[global_func["category"]=="AMR_Response"]["body_sent"], bins, density=True, label="AMR_Response")
plt.legend(loc="upper right")

plt.subplot(337)
plt.hist(global_func[global_func["category"]=="Response_Unhealthy_Products"]["body_sent"], bins, density=True, label="Response_Unhealthy_Products")
plt.legend(loc="upper right")


plt.show();<jupyter_output><empty_output><jupyter_text>There is right skew in the above plot as well. We also perform transformation on this feature as well.  Now lets look at which features might require transformation<jupyter_code>#looking at aggregated data histogram of the new features to see which are the best candidate for transformation
#looking at text length
bins=np.linspace(0, 2500, 60)
plt.figure(figsize=(6,6))

plt.hist(global_func["body_len"], bins)
plt.title("Body Length Distribution")
plt.show();


#crealy there is right skew in the data. So, we will transform this data so that the histogram looks like bell curve
#looking at punctuation percentage
bins=np.linspace(0, 100, 40)
plt.figure(figsize=(6,6))

plt.hist(global_func["punct%"], bins)
plt.title("Punctuation Percent Distribution")
plt.show();

#there is no skew. So, we won't transfer this.
#looking at sentence length

bins=np.linspace(0, 25, 40)
plt.figure(figsize=(6,6))

plt.hist(global_func["body_sent"], bins)
plt.title("Number of Sentence Distribution")
plt.show();

#there is right skew. We will transfer this feature as well so that the data is concetrated around some central value.<jupyter_output><empty_output><jupyter_text>Box-cox power transformation  We will use the box-cox power transformation to transform "body_len" and "body_sent" feature vectors<jupyter_code># checking different exponents on body len and printing the hist to see how the distribution changes

for i in [1,2,4,6,7, 8, 9]:
    plt.hist((global_func["body_len"])**(1/i), bins=60)
    plt.title("Transformations: 1/{}".format(str(i)))
    plt.show();
    
plt.hist(np.log10(global_func["body_len"]), bins=60)
plt.title("Transformations: log")
plt.show();

#after 6th power the histograms look very similar. However, the log transformation look better than than other powers
#applying logrithmic transformation on "body_len" column s
global_func["body_len"]=global_func["body_len"].apply(lambda x: np.log10(x))
## checking different exponents on number of sentences and printing the hist to see how the distribution changes
for i in [1,2,4,6,7, 8, 9]:
    plt.hist((global_func["body_sent"])**(1/i), bins=40)
    plt.title("Transformations: 1/{}".format(str(i)))
    plt.show();
    
plt.hist(np.log10(global_func["body_sent"]), bins=40)
plt.title("Transformations: log")
plt.show();
#applying logrithmic transformation on "body_len" column s
global_func["body_sent"]=global_func["body_sent"].apply(lambda x: np.log10(x))
global_func.head()<jupyter_output><empty_output><jupyter_text>Now we will clean our data. This means we will do the following step:1. First we remove the punctuations.2. then we will remove the stopwords like the, a, etc.3. Finally we will stem or lemmatize the work. We will explore both stemming and lemmatization.<jupyter_code>#checking punctuations available in string package
string.punctuation
#initializing the stopwords and stemmer and lemmetizer
stopwords=nltk.corpus.stopwords.words("English")
#ps=nltk.PorterStemmer()
wn=nltk.WordNetLemmatizer()
#creating the cleaning function
def clean_text(text):
    
    #removing punctuation
    clean_punct="".join([char for char in text if char not in string.punctuation])
    
    #spitting into token based on word end
    token=re.split("\W+",clean_punct)
    
    #decided to use lemmetizer instead of stemmer as lemmetizer provids better results and also understands language semantics
    clean_lemma= [wn.lemmatize(word.upper()) for word in token if word not in stopwords] 
    
    #removing "NA" token
    remove_na= [word for word in clean_lemma if word !="NA"]
    
    #returning the cleaned text
    return remove_na
#applying the clean_text() function on the description column using lambda function
#global_func["descrip_clean"]=global_func["description"].apply(lambda x: clean_text(x))<jupyter_output><empty_output><jupyter_text>Vectorizing Raw Data   Applying CountVectorizer<jupyter_code>count_vect= CountVectorizer(analyzer=clean_text)
x_counts= count_vect.fit_transform(global_func["description"])
print(x_counts.shape)
print(count_vect.get_feature_names())
#vectorizer outputs a sparse matrix. Lets take a look at the matrix
x_counts_df=pd.DataFrame(x_counts.toarray())

x_counts_df

#we can see below that the matrix is full of zeors. So, it is a sparese matrix. Each column is a unique word
#Each row is number of text description in our dataset.
#The column value will be other than zero if the unique word exist in the text 
# we can also see what words each column represent
x_counts_df.columns= count_vect.get_feature_names()

x_counts_df.head()
#not clear which text has ÒAG. So, checking which rowa/text have this word
x_counts_df[x_counts_df["ÒAG"]==1]
#extracting the text 1238 from description to see how word ÒAG is used
print(global_func.loc[1238, "description"])<jupyter_output>AFRIKAGRUPPERNA FRAME 2017-2021. Framework agreement 2017-2021The current frame agreement between the Africa groups (AG) of Sweden and Sida runs from 2017-2021. Implementation is made by partner organizations in Angola, Mozambique, Namibia, South Africa and Zimbabwe. AG applies a rights based approach and works within four thematic areas; òGovernance of natural resourcesòFood sovereigntyòSRHR and HIV òWorking and living conditions. During the appraisal the following dialogue objectives was identified: òAG and its POs has strengthened capacity related to risk analysis and risk management at program and organization level.òAG has strengthened the integration of conflict sensitivity.òAG has adjusted and developed systems for internal control related to cooperation with new rights-oriented actors and networks have been developed.òSecuring and implementing the enhanced gender equality work at the organization and program level.òStrengthened capacity for monitoring and internal control at th[...]<jupyter_text>Applying tfi-df<jupyter_code>tfidf_vect=TfidfVectorizer(analyzer=clean_text)
x_tfidf=tfidf_vect.fit_transform(global_func["description"])
print(x_tfidf.shape)
print(tfidf_vect.get_feature_names())
#the x_tfidf is sparse matrix. Lets look at it in data frame form
x_tfidf_df= pd.DataFrame(x_tfidf.toarray())

x_tfidf_df

#tfidf is very similar to count vectorizer. The matrix is full of zeros. Each column represents a word
#And which ever text (rows) the word comes we have the tfidf calculated. The values of tfidf will be different from 1.
# lets change the name of coulmns from number to actual words which these coulmns represent
x_tfidf_df.columns= tfidf_vect.get_feature_names()

x_tfidf_df<jupyter_output><empty_output><jupyter_text>Building machine learning classifier<jupyter_code>#concatenating all  the features
x_features= pd.concat([global_func["body_len"], global_func["body_sent"], global_func["punct%"],pd.DataFrame(x_counts.toarray())],
                      axis=1)

x_features<jupyter_output><empty_output><jupyter_text>Exploring Random Forest Classifier through Cross-Validation to check how our model does before running it on a separate holdout set<jupyter_code>#splitting the data into train and test set
X_train, X_test, y_train, y_test=train_test_split(x_features, global_func["category"], test_size=0.2,shuffle=True)
#instantiating the classifier
rf = RandomForestClassifier (n_estimators=50, max_depth=200, n_jobs = -1)
# generate your cross-validation prediction with 10 fold Stratified sampling
y_pred=cross_val_predict(rf, X_train, y_train, cv=5, n_jobs=-1)
y_pred.shape
#printing the classification result
print(classification_report(y_train, y_pred))
#creating confusion matrix
confusion_matrix(list(y_train), list(y_pred))
#creating confusion matrix heatmap
conf_mat_rf=confusion_matrix(list(y_train),list(y_pred))
conf_mat_df_rf= pd.DataFrame(conf_mat_rf,
                     index = ["AMR_Response", "Control_Cross_Border_Disease", "Epidemic_Pandemic_Prep", 
                             "Health_Advocacy", "Knowledge_Gen_Sharing", "Response_Unhealthy_Products", 
                             "not_global_func"], 
                     columns = ["AMR_Response", "Control_Cross_Border_Disease", "Epidemic_Pandemic_Prep", 
                             "Health_Advocacy", "Knowledge_Gen_Sharing", "Response_Unhealthy_Products", 
                              "not_global_func"])

plt.figure(figsize=(10, 8))
sns.heatmap(conf_mat_df_rf, annot=True, cmap="Blues", center=0, fmt='g')          
plt.ylabel('True Label')
plt.xlabel('Predicted Label')
plt.show()<jupyter_output><empty_output><jupyter_text>Applying the classifier on Holdout Test set (Train and Test set separately)<jupyter_code>#creating a train and test set
X_train, X_test, y_train, y_test=train_test_split(x_features, global_func["category"], test_size=0.2,shuffle=True)

#instantiating the model and fitting it on the train set
rf=RandomForestClassifier(n_estimators=150, max_depth=None)

#fitting the model on training data
rf_model=rf.fit(X_train, y_train)
#most important list of features tht random forest uses for learning
sorted(zip(rf_model.feature_importances_, X_train.columns), reverse=True)[0:10]

#our created features are in top 10 importance
#predicting the categories using the trained model on test set 
y_pred=rf_model.predict(X_test)

#printing classification results
print(classification_report(y_test, y_pred)) 
#creating confusion matrix heatmap
conf_mat_rf=confusion_matrix(list(y_test),list(y_pred))
conf_mat_df_rf= pd.DataFrame(conf_mat_rf,
                     index = ["AMR_Response", "Control_Cross_Border_Disease", "Epidemic_Pandemic_Prep", 
                             "Health_Advocacy", "Knowledge_Gen_Sharing", "Response_Unhealthy_Products", 
                             "not_global_func"], 
                     columns = ["AMR_Response", "Control_Cross_Border_Disease", "Epidemic_Pandemic_Prep", 
                             "Health_Advocacy", "Knowledge_Gen_Sharing", "Response_Unhealthy_Products", 
                              "not_global_func"])

plt.figure(figsize=(10, 8))
sns.heatmap(conf_mat_df_rf, annot=True, cmap="Blues", center=0, fmt='g')          
plt.ylabel('True Label')
plt.xlabel('Predicted Label')
plt.show()<jupyter_output><empty_output><jupyter_text>From the above heatmap, we can see there are 23 misclassifications. Most of the misclassification are assigned to "not_global_fuction" category. Imbalance in our dataset could be one reason. Due to disproportional large number of data in this category in comparision to other categories cases our model to miscalssify into this category.<jupyter_code>#checking which which predictions do not match to actual category, and then subsetting the X_test data set based on this.
X_test[y_test!=y_pred]

#However, this X_test data frame is a vectorized data frame where we have converted text to numbers for coputer to understand.
#Therefore, for humans to understand which description were misclassified, we will used the undex from this data frame,
#and subset the global_func dataframe.
#extracting the index numbers from the X_test which have been misclassified
index_list=X_test[y_test!=y_pred].index


#using the index extracted from X_test, we subset the original global_func data frame
global_func_misclass=global_func.iloc[index_list]

#resetting the index, and getting the index to start from 0, and creating a separate coulmn for oroginal index
global_func_misclass=global_func_misclass.reset_index()

#printing the global_func misclass data frame
global_func_misclass
#subsetting the values from y_pred which it wrongly predicts
y_pred_misclass=pd.DataFrame(y_pred[y_test!=y_pred])

#concatenating the the y_pred_misclass with the global_func_misclass data frame
global_func_mispred=pd.concat([global_func_misclass, y_pred_misclass], axis=1)

#renaming the misclassification prediction column
global_func_mispred=global_func_mispred.rename(columns={0:"class_pred"})

#creating the data frame with only the columns we are interedted in 
global_func_mispred=global_func_mispred[["index", "description", "category", "class_pred"]] 

#printing the data frame
global_func_mispred<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11240669581732256, 'fraction_numerical': 0.013338633452586016, 'mean_word_length': 6.93899018232819, 'pattern_counts': {'":': 2, '<': 51, '<?xml version=': 0, '>': 51, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '13', 'dir': 'jupyter-structured-clean-dedup', 'id': '418797', 'license': 'MIT', 'n_tokens_mistral': 7615, 'n_tokens_neox': 7125, 'n_words': 1829, 'path': 'Global_Function_NLP_Aug2021.ipynb', 'repo_name': 'pyarasid/GlobalFunction', 'size': '22641'}