<jupyter_start><jupyter_text>Import Libraries<jupyter_code>import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import itertools
import nltk
from nltk.corpus import stopwords
import re
from nltk.tokenize import word_tokenize
from nltk.tag.perceptron import PerceptronTagger
from nltk.stem import PorterStemmer, WordNetLemmatizer
import time
from nltk.corpus import wordnet
import string<jupyter_output><empty_output><jupyter_text>Train Data Preprocessing<jupyter_code># import training data
data = pd.read_csv('./train_data_labels.csv')
data.head(6)
# check if there is any null value
print(data.isna().sum().to_string())
# visualize class labels
label_count = data.label.value_counts()
print(label_count.describe().to_string())
sns.barplot(y=label_count.index, x=label_count, palette='dark', alpha=.6);
# check for duplicated abstract
dup = data[data.duplicated('abstract', keep=False)] # show all duplicated results
print(dup.shape)
dup.head(6)
# check if the duplicate abstracts have different labels
dup_lab_count = dup.groupby('abstract')['label'].apply(set).apply(len)
dup_abs = dup_lab_count[dup_lab_count.values>1].index
# return the index if the same abstracts have different labels
dup_index = [dup[dup.abstract==dup_abs[i]].index for i in range(len(dup_abs))]
dup_index = list(itertools.chain.from_iterable(dup_index))
# remove the wrong labeled abstracts
clean_data = data.copy()
clean_data.drop(dup_index, axis=0, inplace=True)
print(clean_data.shape)
# define stopwords containing English, German and French
en = stopwords.words('english')
ge = stopwords.words('german')
fr = stopwords.words('french')
stopword = en + ge + fr
# define function for text preprocessing
def get_clean(text):
      
    text = text.lower() # return lowercase
    text = re.sub('[.]', '', text) # replace '.' by space
    text = re.sub('[/(){}\[\]\|@,;∼~`!^$%&*#+_-]', ' ', text) # replace the specified symbols by space
    text = re.sub(r'[^0-9a-z #+_]', ' ', text) # replace all other symbols by space
    text = re.sub(r'[0123456789]', ' ', text) # replace number by space
    text = re.sub(r"\b[a-zA-Z]\b", '', text) # remove single letter
    # remove stopwords and return the cleaned text
    return ' '.join(word for word in text.split() if word not in stopword)
# create new column to store cleaned abstract
clean_data['clean'] = clean_data.abstract.apply(get_clean)
print(clean_data.clean[0])
# create new column to store tokens
clean_data['tokens'] = clean_data['clean'].apply(word_tokenize)
print(clean_data.tokens[0])
# check if there is any cells that contain no tokens
# this may happen for abstract that contains only symbols
print(clean_data.tokens.apply(len).describe().to_string())
# display data that has less or equal to 2 tokens
clean_data[clean_data.tokens.apply(len)<=2]
# show data that has number of tokens >=3 and <= 5
clean_data[(clean_data.tokens.apply(len)>=3)&(clean_data.tokens.apply(len)<=5)]
# remove data that has bad abstract
# e.g. zero tokens, abstract with trivial meanings
zero_len = clean_data[clean_data.tokens.apply(len)==0].index.tolist()
bad_index = zero_len + [806, 2071, 2168, 32733, 38083, 62045, 62378, 62812, 76635, 77264, 77707, 80364, 82516]
clean_data.drop(bad_index, axis=0, inplace=True)
print(clean_data.shape)
# re-check the duplicated data
clean_data[clean_data.duplicated('clean', keep='first')]
# keep the first appeared duplicated row and remove the rest
clean_data.drop(clean_data[clean_data.duplicated('clean', keep='first')].index.tolist(), axis=0, inplace=True)
print(clean_data.shape)
# visualize the number of labels after cleaning
count = clean_data.label.value_counts()
sns_bar(count, axes=None);
# https://nlp.stanford.edu/IR-book/html/htmledition/stemming-and-lemmatization-1.html
print('The word "{}" after stemming is "{}"'.format('operate', PorterStemmer().stem('operate')))
print('The word "{}" after stemming is "{}"'.format('operational', PorterStemmer().stem('operational')))
# define function to perform lemmatization using POS tagging
# lemmatization on each word will be performed according the individual's POS tagging
# if no POS tag returned, keep the original forms of the words
tagger = PerceptronTagger()
lemm = WordNetLemmatizer()
tag_dict = {'N': wordnet.NOUN, 'V': wordnet.VERB, 'J': wordnet.ADJ, 'R': wordnet.ADV}
def get_lemmatized(token):  
    return [lemm.lemmatize(word, tag_dict.get(tagger.tag([word])[0][1][0].upper())) 
            if tag_dict.get(tagger.tag([word])[0][1][0].upper())!=None else word for word in token]
# perform lemmatization on the cleaned abstract
start = time.time()

clean_data['lemm_tokens'] = clean_data.tokens.apply(get_lemmatized)

print('The elapsed time is', (time.time() - start)/60)
# tokenize the lemmatized abstract
# save the cleaned training data to clean_train.csv
clean_data['lemm'] = clean_data.lemm_tokens.apply(lambda x: ' '.join(x))
train_output = clean_data[['clean', 'lemm', 'label']]
train_output.to_csv('./clean_train.csv', index=False)<jupyter_output><empty_output><jupyter_text>Test Data Preprocessing<jupyter_code># import the test data
test = pd.read_csv('./test_data.csv')
test.head()
# perform the text preprocessing using the same approach for training data
test['clean'] = test.abstract.apply(get_clean) 
test['tokens'] = test.clean.apply(word_tokenize)
test['lemm'] = test.tokens.apply(get_lemmatized).apply(lambda x: ' '.join(x))
# check if there is any zero token
test[test.tokens.apply(len)==0]
# replace the data with zero token by their original abstract
test['clean'][test.index==812] = '1300'
test['lemm'][test.index==812] = '1300'

test['clean'][test.index==7332] = '...'
test['lemm'][test.index==7332] = '...'
# save the cleaned test data to clean_test.csv
test_output = test[['test_id', 'clean', 'lemm']]
test_output.to_csv('./clean_test.csv', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14246715577546493, 'fraction_numerical': 0.020644941136324858, 'mean_word_length': 6.020359281437126, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '622918', 'license': 'MIT', 'n_tokens_mistral': 2044, 'n_tokens_neox': 1941, 'n_words': 509, 'path': 'text_preprocessing.ipynb', 'repo_name': 'syl1618/ACS-Text-Classification', 'size': '5861'}