<jupyter_start><jupyter_text>Read file<jupyter_code># Read the csv file into a pandas DataFrame
df = pd.read_csv('training_list3withtype.csv')
df.head()
# Input of the training will be on column'filename' and using column 'doctype' for classification/categorise
df.info()
# Check column names
df.columns
# Drop (duplicates) and incomplete rows
print(df.count())
df = df.dropna()
df = df.drop_duplicates()
print(df.count())
# Check variation/number of document types we have in the list before training
print(df['doctype'].value_counts())
print(df['doctype'].value_counts(normalize=True))
# Check porportions of each doctype
# we have unbalance category, 'Well log' type contains the larget portion for this training
count = df['doctype'].value_counts()
count.plot(kind='barh')
plt.title("Proportion of document types")
plt.xlabel("Count")
plt.ylabel("Document category")
# Initialize Label Encoder
le = LabelEncoder()
df['doctype_enc'] = le.fit_transform(df['doctype'])
df.head()
#Catagory: operations and drilling(0),palynology and core data(1),well log (2),well report(3)
df.dtypes<jupyter_output><empty_output><jupyter_text>Data cleaning and pre-preparation<jupyter_code># Input of the training will be on column'filename' and using classification/category from column 'doctype_enc'
# Remove special characters using regex
sp_chars = ["!",'"',"#","%","&","'","(",")",
              "*","+",",","-",".","/",":",";","<",
              "=",">","?","@","[","\\","]","^","_",
              "`","{","|","}","~","–"]
for char in sp_chars:
    df['filename'] = df['filename'].str.replace(char, ' ')
# Remove number from 'filename' using regex and convert to lower case
df['filename'] = df['filename'].str.replace('\d+', ' ').str.lower()
df
# Remove file format as shown in column 'ext' (pdf, las, doc)
file_format = df.ext.unique()
for ext in file_format:
    df['filename'] = df['filename'].str.replace(ext, '')
df
# Remove proper noun (such as well name that usually find in 'file name')
well_list = ['appendix','chevron','slb','swallow', 'corvus', 'clio','franklin','fish','river','banka','garganey','yyche','backpacker','coogee','charon','cisticola','geelvink','koel','lilac','livet','munia','moogooree','curvus','hyde']
for well in well_list:
    df['filename'] = df['filename'].str.replace(well, ' ')
df
# remove words less than 3 characters
# df['filename'].str.findall('\w{3,}').str.join(' ')
df['filename'] = df['filename'].str.split().map(lambda sl: " ".join(s for s in sl if len(s) > 2))
df
# # Remove whitespace(s) between words and then rejoin
# df['filename'] = df['filename'].str.split().str.join(" ")
# df
# Re-check and Drop (duplicates) and incomplete rows
print(df.count())
df = df.dropna()
df = df.drop_duplicates()
print(df.count())<jupyter_output>doctype        1357
subtype        1357
path           1357
filename       1357
ext            1357
doctype_enc    1357
dtype: int64
doctype        1357
subtype        1357
path           1357
filename       1357
ext            1357
doctype_enc    1357
dtype: int64<jupyter_text>Export clean data<jupyter_code># # # Export clean list as .csv
df.to_csv(r'C:\Users\nkantanong\BootCamp\Final_Project\Final_Project_NS\nlp\list_clean1.csv', index = False)<jupyter_output><empty_output><jupyter_text>Preparation data for training<jupyter_code># Check column names
df.columns
# Tokenized
tokenized1 = df['filename'].str.lower().apply(word_tokenize)
tokenized1
# Remove stop words with additional names (well lists that show in the file name)
# well_list = ['swallow', 'clio','franklin','fish','river','banka','garganey','yyche','backpacker','coogee','charon','cisticola','geelvink','koel','lilac','livet','munia','moogooree','curvus','hyde']
# def remover(tokens):
#     stopwords = set(stopwords.words('english'))
#     new_words =('swallow', 'clio','franklin','fish','river','banka','garganey','yyche','backpacker','charon','cisticola','geelvink','koel','lilac','livet','munia','moogooree','curvus','hyde')
#     for i in new_words:
#         stopwords.append(i)

# final_stop_words = all_stop_words.extend(well_list)
# remover = StopWordsRemover(inputCol="words", outputCol="filtered")
# tokenized_name = tokenized_name.apply(stopwords)
# tokenized_name
# Define function to remove stop words
def remove_stop_words(tokens):
    no_stop = []
#     stopwords = nltk.corpus.stopwords.words('english')
    for token in tokens:
        if token not in stopwords.words('english'):
            no_stop.append(token)
    return no_stop

# Apply our function to tokens
tokenized = tokenized1.apply(remove_stop_words)
tokenized
# Define function to lemmatization
def lemmatize(tokens):
    lemmatizer = WordNetLemmatizer()
    # Create the lemmatized list
    lemmatized = []
    for token in tokens:
            # Lemmatize and append
            lemmatized.append(lemmatizer.lemmatize(token))
    return " ".join(lemmatized)

# Apply our function to tokens
tokenized = tokenized1.apply(lemmatize)
tokenized
# assign tokenized to dataframe
df2 = df.assign(tokenized = tokenized)
df2.head(5)
# assign words count (tokenized count) to dataframe
df2['tokenized_count'] = [len(x.split()) for x in df2['tokenized'].tolist()]
df2
# # # Export cleaned list#2 as .csv
df2.to_csv(r'C:\Users\nkantanong\BootCamp\Final_Project\Final_Project_NS\nlp\list_clean2.csv', index = False)
# Get all document catagories
od = df2[df2['doctype']=='operations and drilling']['tokenized'].str.cat(sep=' ')
wr = df2[df2['doctype']=='well report']['tokenized'].str.cat(sep=' ')
wl = df2[df2['doctype']=='well log']['tokenized'].str.cat(sep=' ')
paly = df2[df2['doctype']=='palynology and core data']['tokenized'].str.cat(sep=' ')

# Initialize the word cloud
wc = WordCloud(width = 500, height = 500, min_font_size = 10, background_color ='white')

# Generate the world clouds for each type of message
od_wc = wc.generate(od)

# plot the world cloud for operations and drilling documents                     
plt.figure(figsize = (5, 5), facecolor = None) 
plt.imshow(od_wc) 
plt.axis("off") 
plt.title("Common words in operations and drilling")
plt.tight_layout(pad = 0) 
plt.show() 

wr_wc = wc.generate(wr)

# plot the world cloud for Well Report                   
plt.figure(figsize = (5, 5), facecolor = None) 
plt.imshow(wr_wc) 
plt.axis("off") 
plt.title("Common words in Well Reports")
plt.tight_layout(pad = 0) 
plt.show() 

# ##########
wl_wc = wc.generate(wl)

# plot the world cloud for Well log                   
plt.figure(figsize = (5, 5), facecolor = None) 
plt.imshow(wl_wc) 
plt.axis("off") 
plt.title("Common words in Well Log")
plt.tight_layout(pad = 0) 
plt.show() 

# Generate the world clouds for Palynology
paly_wc = wc.generate(paly)

# plot the world cloud for operations and drilling documents                     
plt.figure(figsize = (5, 5), facecolor = None) 
plt.imshow(paly_wc) 
plt.axis("off") 
plt.title("Common words in palynology and core data")
plt.tight_layout(pad = 0) 
plt.show()<jupyter_output><empty_output><jupyter_text>Build model<jupyter_code># use stratify parameter of train_test_split since our data is not balanced
X = df2['tokenized']
y = df2['doctype_enc']
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=34, stratify=y)
# Create the tf-idf vectorizer
vectorizer = TfidfVectorizer(strip_accents='ascii')

# create training and testing set using the same vectorizer
tfidf_train = vectorizer.fit_transform(X_train)
tfidf_test = vectorizer.transform(X_test)
# Initialize the Multinomial Naive Bayes classifier
nb = MultinomialNB()

# Fit the model
nb.fit(tfidf_train, y_train)

# Print the accuracy score
print("Accuracy score:",nb.score(tfidf_test, y_test))
# Save the model as a pickle in a file
joblib.dump(nb, 'nb_model.pkl')
 
# # Load the model from the file
# nb_from_joblib = joblib.load('nb_model.pkl')
 
# # Use the loaded model to make predictions
# nb_from_joblib.predict(X_test)
# Predict the doc type
y_pred = nb.predict(tfidf_test)


# Print the Confusion Matrix
cmatrix = confusion_matrix(y_test, y_pred)
print("Confusion Matrix\n")
print(cmatrix)



# Print the Classification Report
cr = classification_report(y_test, y_pred)
print("\n\nClassification Report\n")
print(cr)



# # Get probabilities.
# y_pred_proba = nb.predict(tfidf_test)

# # # # Print the Receiver operating characteristic Auc score
# auc_score = roc_auc_score(y, y_pred_proba, multi_class='ovr')

# print("\nROC AUC Score:",auc_score)



# # # # Get False Positive rate, True Positive rate and the threshold
# fpr, tpr, thresholds = roc_curve(y_test, y_pred_proba)<jupyter_output>Confusion Matrix

[[  5   0  18   0]
 [  0  30   3   1]
 [  0   1 171   1]
 [  0   0   9  33]]


Classification Report

              precision    recall  f1-score   support

           0       1.00      0.22      0.36        23
           1       0.97      0.88      0.92        34
           2       0.85      0.99      0.91       173
           3       0.94      0.79      0.86        42

    accuracy                           0.88       272
   macro avg       0.94      0.72      0.76       272
weighted avg       0.89      0.88      0.86       272
{'fraction_non_alphanumeric': 0.1450624042040727, 'fraction_numerical': 0.022881541493321656, 'mean_word_length': 4.163934426229508, 'pattern_counts': {'":': 1, '<': 20, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '415598', 'license': 'MIT', 'n_tokens_mistral': 3280, 'n_tokens_neox': 3084, 'n_words': 771, 'path': 'Data_preparation.ipynb', 'repo_name': 'knarisara/Final_project', 'size': '9134'}