<jupyter_start><jupyter_text>Reading the text data<jupyter_code>text_data_sentence = pd.read_csv('textdatanew.csv', encoding='ISO-8859-1')
text_data_sentence.head()<jupyter_output><empty_output><jupyter_text>Reading the Response file<jupyter_code>bess_tags = pd.read_csv('CBW_Bess_tags_final2.csv')
bess_tags.head()
#bess_tags.head()
bess_reponse = bess_tags.loc[:,['Content','Event','Type','para no','biographyID','collectionID']]
bess_reponse= bess_reponse.fillna(' ')
bess_reponse.loc[:,'Response'] = bess_reponse.loc[:,['Content','Event']].apply(lambda x: '_'.join(x),axis = 1)
top_10_persona_desc = bess_reponse[bess_reponse.Type.isin(['personaDescription'])].Response.value_counts()[0:10]
top_10_persona_desc
bess_reponse.shape
bess_reponse.loc[:,['para no','biographyID','collectionID','Response']].head()
bess_response_df = bess_reponse.loc[:,['para no','biographyID','collectionID','Response','Type']].drop_duplicates()
#bess_response_df.groupby(['para no','biographyID','collectionID','Type'],as_index = False)['Response'].apply(lambda x: ' '.join(x))
final_response_file = bess_response_df[bess_response_df.Response.isin(top_10_persona_desc.index)]
#final_response_file.melt()
final_response_file.head()
final_response_file.loc[:,'values'] = 1
final_response_wide = (final_response_file.pivot_table(index=['para no','biographyID','collectionID','Type'],\
                                       columns='Response', values='values').reset_index())
final_response_wide = final_response_wide.fillna(0)
final_response_wide.head()
text_data_sentence_final = pd.merge(text_data_sentence,final_response_wide, how = 'left', left_on = ['CollectionID','BiographyID','ParagraphNo'],\
                             right_on = ['collectionID','biographyID','para no'])
text_data_sentence_final = text_data_sentence_final[~text_data_sentence_final['para no'].isna()]
text_data_sentence_final_response = text_data_sentence_final[text_data_sentence_final.columns[-10:]]
final_response_file.Response.value_counts()<jupyter_output><empty_output><jupyter_text>Bag of Words<jupyter_code>stop_words = stopwords.words('english')
bow_model = CountVectorizer(ngram_range= (1,2),stop_words=stop_words)
Para_text_bow = bow_model.fit_transform(text_data_sentence_final.ParagraphText)<jupyter_output><empty_output><jupyter_text>Tf-IDF<jupyter_code>tf_idf_model = TfidfVectorizer(ngram_range=(1,2),stop_words = stop_words)
Para_text_tfidf = tf_idf_model.fit_transform(text_data_sentence_final.ParagraphText)<jupyter_output><empty_output><jupyter_text>Splitting data into train and test<jupyter_code>from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(Para_text_tfidf,text_data_sentence_final_response,
                                                    test_size = 0.3, random_state = 0)
y_test.shape<jupyter_output><empty_output><jupyter_text>Machine Learning Models<jupyter_code>from sklearn.ensemble import RandomForestClassifier
from sklearn.linear_model import LogisticRegression
#knn_model = KNeighborsClassifier(n_neighbors= 3, p = 1.5)
rf_model = RandomForestClassifier()
lr_model = LogisticRegression(multi_class='multinomial')
#y_train.melt()[y_train.melt().value.isin([1])].variable.value_counts()
rf_model.fit(X_train,y_train)
feature_importances = pd.DataFrame(rf_model.feature_importances_,
                                  index = bow_model.get_feature_names(),
                                  columns=['importance']).sort_values('importance',ascending=False)

feature_importances
np.array(rf_model.predict_proba(X_test)).shape
[np.argmax([int(each[1]>0.4) for each in np.array(rf_model.predict_proba(X_test))[:,each_example,:]]) if \
     sum([int(each[1]>0.4) for each in np.array(rf_model.predict_proba(X_test))[:,each_example,:]]) > 0 else 10
            for each_example in range(100)]
result = [np.argmax(each) if each.sum() != 0 else 10 for each in rf_model.predict_proba(X_test)]
pd.DataFrame(result)[0].value_counts()
y_test.melt()[y_test.melt().value.isin([1])].variable.value_counts()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13980868285504047, 'fraction_numerical': 0.01177336276674025, 'mean_word_length': 7.339468302658487, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '64522', 'license': 'MIT', 'n_tokens_mistral': 1473, 'n_tokens_neox': 1421, 'n_words': 154, 'path': 'Paragraph Annotation/Paragraph Annotation_ PersonaDesc- Machine Learning.ipynb', 'repo_name': 'UVA-DSI-2019-Capstones/CBW', 'size': '4077'}