<jupyter_start><jupyter_text>Pre-processing<jupyter_code>columns = ['keywords', 'urgency']
rows = []

training_data = pd.DataFrame(rows, columns=columns)
training_data = pd.read_csv("cpf_processed.csv")
u1_docs = [row['keywords'] for index,row in training_data.iterrows() if row['urgency'] == 1]

vec_u1 = CountVectorizer()
X_u1 = vec_u1.fit_transform(u1_docs)
tdm_u1 = pd.DataFrame(X_u1.toarray(), columns=vec_u1.get_feature_names())

u2_docs = [row['keywords'] for index,row in training_data.iterrows() if row['urgency'] == 2]

vec_u2 = CountVectorizer()
X_u2 = vec_u2.fit_transform(u2_docs)
tdm_u2 = pd.DataFrame(X_u2.toarray(), columns=vec_u2.get_feature_names())

u3_docs = [row['keywords'] for index,row in training_data.iterrows() if row['urgency'] == 3]

vec_u3 = CountVectorizer()
X_u3 = vec_u3.fit_transform(u3_docs)
tdm_u3 = pd.DataFrame(X_u3.toarray(), columns=vec_u3.get_feature_names())

u4_docs = [row['keywords'] for index,row in training_data.iterrows() if row['urgency'] == 4]

vec_u4 = CountVectorizer()
X_u4 = vec_u4.fit_transform(u4_docs)
tdm_u4 = pd.DataFrame(X_u4.toarray(), columns=vec_u4.get_feature_names())

u5_docs = [row['keywords'] for index,row in training_data.iterrows() if row['urgency'] == 5]

vec_u5 = CountVectorizer()
X_u5 = vec_u5.fit_transform(u5_docs)
tdm_u5 = pd.DataFrame(X_u5.toarray(), columns=vec_u5.get_feature_names())
# Frequency of words for each urgency label

word_list_u1 = vec_u1.get_feature_names();    
count_list_u1 = X_u1.toarray().sum(axis=0) 
freq_u1 = dict(zip(word_list_u1,count_list_u1))

word_list_u2 = vec_u2.get_feature_names();    
count_list_u2 = X_u2.toarray().sum(axis=0) 
freq_u2 = dict(zip(word_list_u2,count_list_u2))

word_list_u3 = vec_u3.get_feature_names();    
count_list_u3 = X_u3.toarray().sum(axis=0) 
freq_u3 = dict(zip(word_list_u3,count_list_u3))

word_list_u4 = vec_u4.get_feature_names();    
count_list_u4 = X_u4.toarray().sum(axis=0) 
freq_u4 = dict(zip(word_list_u4,count_list_u4))

word_list_u5 = vec_u5.get_feature_names();    
count_list_u5 = X_u5.toarray().sum(axis=0) 
freq_u5 = dict(zip(word_list_u5,count_list_u5))
# Write values to CSV file
import csv

with open('u1_word_frequencies.csv', 'w') as f:
    for key in freq_u1.keys():
        f.write("%s,%s\n"%(key,freq_u1[key]))
        
with open('u2_word_frequencies.csv', 'w') as f:
    for key in freq_u2.keys():
        f.write("%s,%s\n"%(key,freq_u2[key]))
        
with open('u3_word_frequencies.csv', 'w') as f:
    for key in freq_u3.keys():
        f.write("%s,%s\n"%(key,freq_u3[key]))

with open('u4_word_frequencies.csv', 'w') as f:
    for key in freq_u4.keys():
        f.write("%s,%s\n"%(key,freq_u4[key]))
        
with open('u5_word_frequencies.csv', 'w') as f:
    for key in freq_u5.keys():
        f.write("%s,%s\n"%(key,freq_u5[key]))
# For Laplace Smoothing

docs = [row['keywords'] for index,row in training_data.iterrows()]

vec = CountVectorizer()
X = vec.fit_transform(docs)

total_features = len(vec.get_feature_names())

total_cnts_features_u1 = count_list_u1.sum(axis=0)
total_cnts_features_u2 = count_list_u2.sum(axis=0)
total_cnts_features_u3 = count_list_u3.sum(axis=0)
total_cnts_features_u4 = count_list_u4.sum(axis=0)
total_cnts_features_u5 = count_list_u4.sum(axis=0)

features_dict = {}
features_dict['total'] = total_features
features_dict['u1'] = total_cnts_features_u1
features_dict['u2'] = total_cnts_features_u2
features_dict['u3'] = total_cnts_features_u3
features_dict['u4'] = total_cnts_features_u4
features_dict['u5'] = total_cnts_features_u5
with open('total_cnts_features.csv', 'w') as f:
    for key in features_dict.keys():
        f.write("%s,%s\n"%(key,features_dict[key]))
u1_prob = training_data.groupby('urgency').count()['keywords'][1]
u2_prob = training_data.groupby('urgency').count()['keywords'][2]
u3_prob = training_data.groupby('urgency').count()['keywords'][3]
u4_prob = training_data.groupby('urgency').count()['keywords'][4]
u5_prob = training_data.groupby('urgency').count()['keywords'][5]

u_probabilities = {}
u_probabilities['u1'] = u1_prob
u_probabilities['u2'] = u2_prob
u_probabilities['u3'] = u3_prob
u_probabilities['u4'] = u4_prob
u_probabilities['u5'] = u5_prob
with open('u_probabilities.csv', 'w') as f:
    for key in u_probabilities.keys():
        f.write("%s,%s\n"%(key,u_probabilities[key]))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1595035623994484, 'fraction_numerical': 0.033325672259250746, 'mean_word_length': 6.401360544217687, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '454320', 'license': 'MIT', 'n_tokens_mistral': 1835, 'n_tokens_neox': 1810, 'n_words': 262, 'path': '.ipynb_checkpoints/Ticket_Classification_Preprocessing-checkpoint.ipynb', 'repo_name': 'ShengXue97/CPF-Microsoft-Azure', 'size': '4351'}