<jupyter_start><jupyter_text><jupyter_code>from google.colab import drive
drive.mount('/content/drive')
%matplotlib inline
import re
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
import seaborn as sns
import csv
import codecs
import json

import string
import sklearn

from nltk.tokenize import TweetTokenizer
from nltk.sentiment.vader import SentimentIntensityAnalyzer as VS

import gensim
from gensim.models import word2vec

import spacy as sp
nlp = sp.load('en')

import nltk
nltk.download('vader_lexicon')
sentiment_analyzer = VS()

nltk.download("stopwords")
from nltk.corpus import stopwords
enStop = stopwords.words('english')
enStop_dict={e: 0 for e in enStop}

from nltk.stem.porter import *
stemmer = PorterStemmer()

# load oov dictionary for word correction
oov_dict = {}
with open('/content/drive/Shareddrives/trecisprep/OOV_Dictionary_V1.0 (1).tsv','r') as tsv_file:
  read_tsv = csv.reader(tsv_file,delimiter = '\t')
  for row in read_tsv:
    oov_dict[row[0]] = row[1]
  tsv_file.close()
# The pre-trained model can be downloaed from: https://crisisnlp.qcri.org/lrec2016/lrec2016.html
%time model = gensim.models.KeyedVectors.load_word2vec_format('/content/drive/Shareddrives/trecisprep/crisisNLP_word_vector.bin', binary=True)
# in order to run the cell, download three datasets from trecis2018-train,trecis2018-test, trescis2019a-test respectively.
# covert them to tsv file with format as shown in the output
data_dir="/content/drive/Shareddrives/data/"
trecis2018train=pd.read_csv(data_dir+"2018A_train_tsv.tsv",dtype=str,sep='\t',lineterminator='\n')
trecis2018train['timestamp']='null'
trecis2018train['postPriority'] = trecis2018train['postPriority'].fillna("Unknown")
idx=trecis2018train[trecis2018train['postPriority'].str.contains("Unknown")].index
trecis2018train['source']='2018train'
print(trecis2018train['post_id'].head())
trecis2018train['post_id'] = trecis2018train.post_id.apply(str)
trecis2018test=pd.read_csv(data_dir+"2018A_test_tsv.tsv",sep='\t',lineterminator='\n')
trecis2018test['source']='2018test'
trecis2018test['post_id'] = trecis2018test.post_id.apply(str)
trescis2019atest=pd.read_csv(data_dir+"2019A_test.csv")
trescis2019atest['source']='2019a'
trescis2019atest['post_id'] = trescis2019atest.post_id.apply(str)
print("Dataset size:\n trecis2018train\ttrecis2018test\ttrescis2019atest\n {}\t{}\t{}".format(len(trecis2018train),len(trecis2018test),len(trescis2019atest)))
print("------------")
print("Total:\n {}".format(len(trecis2018train)+len(trecis2018test)+len(trescis2019atest)))
trecis2018test.head()
# some changes of information types happened with the evolution of TREC-IS, so this is used for reformating information types
trecis2018train["post_categories"]=trecis2018train['post_categories'].apply(lambda x: x.replace("PastNews","ContextualInformation") if "PastNews" in x else x)
trecis2018train["post_categories"]=trecis2018train['post_categories'].apply(lambda x: x.replace("ContinuingNews","News") if "ContinuingNews" in x else x)
trecis2018train["post_categories"]=trecis2018train['post_categories'].apply(lambda x: x.replace("KnownAlready","OriginalEvent") if "KnownAlready" in x else x)
trecis2018train["post_categories"]=trecis2018train['post_categories'].apply(lambda x: x.replace("SignificantEventChange","NewSubEvent") if "SignificantEventChange" in x else x)
trecis2018train["post_categories"]=trecis2018train['post_categories'].apply(lambda x: x.replace("Unknown","Irrelevant") if "Unknown" in x else x)

trecis2018test["post_categories"]=trecis2018test['post_categories'].apply(lambda x: x.replace("PastNews","ContextualInformation") if "PastNews" in x else x)
trecis2018test["post_categories"]=trecis2018test['post_categories'].apply(lambda x: x.replace("ContinuingNews","News") if "ContinuingNews" in x else x)
trecis2018test["post_categories"]=trecis2018test['post_categories'].apply(lambda x: x.replace("KnownAlready","OriginalEvent") if "KnownAlready" in x else x)
trecis2018test["post_categories"]=trecis2018test['post_categories'].apply(lambda x: x.replace("SignificantEventChange","NewSubEvent") if "SignificantEventChange" in x else x)
trecis2018test["post_categories"]=trecis2018test['post_categories'].apply(lambda x: x.replace("Unknown","Irrelevant") if "Unknown" in x else x)

df_combined=pd.concat([trecis2018train,trecis2018test,trescis2019atest])
df_combined=df_combined.reset_index(drop=True)
# df_combined["post_categories"]=df_combined['post_categories'].apply(lambda x: x.split(","))
print(df_combined.shape)
df_combined.tail()

df_combined['post_categories'] = df_combined.post_categories.apply(lambda x: x[1:-1].split(','))

for index,row in df_combined.iterrows():
        category=row['post_categories']
        l=[]
        cnt=0
        for w in category:
          if(cnt==0):
            l.append(w[1:-1])
          else:
            l.append(w[2:-1])
          cnt=cnt+1
        row['post_categories']=l
        print(row['post_categories'])<jupyter_output><empty_output><jupyter_text>load model<jupyter_code># The pre-trained model can be downloaed from: https://crisisnlp.qcri.org/lrec2016/lrec2016.html
%time model = gensim.models.KeyedVectors.load_word2vec_format('/content/drive/Shareddrives/trecisprep/crisisNLP_word_vector.bin', binary=True)
priority_weight_dict = {'Advice': 0.22862577231414025, 'CleanUp': 0.35653795010024114,
                                  'News': 0.3270722902848146, 'Discussion': 0.19307661376501264,
                                  'Donations': 0.3612767541359874, 'EmergingThreats': 0.8537984236364585,
                                  'Factoid': 0.48917308056907677, 'FirstPartyObservation': 0.1420197717109446,
                                  'GoodsServices': 0.6959346525995092, 'Hashtags': 0.2687451987058043,
                                  'InformationWanted': 0.6688596972384949, 'Irrelevant': 0.0010061402766415601,
                                  'OriginalEvent': 0.09192532381788204, 'MovePeople': 0.9253136744180216,
                                  'MultimediaShare': 0.3128463518966703, 'Official': 0.6785618345463643,
                                  'ContextualInformation': 0.1405539758609653, 'SearchAndRescue': 1.0010061402766415,
                                  'Sentiment': 0.039519645594885515, 'ServiceAvailable': 0.7669372401298534,
                                  'NewSubEvent': 0.9176682650090996, 'ThirdPartyObservation': 0.2703955630411556,
                                  'Location': 0.03711651480530716, 'Volunteer': 0.3839639462617982,
                                  'Weather': 0.33196659834790165}
actionable_tweet = []
count_actiona = 0
important_list = ['GoodsServices','SearchAndRescue','MovePeople','EmergingThreats','NewSubEvent','ServiceAvailable']
for index,row in df_combined.iterrows():
  t = row['post_categories']
  count = 0
  flag = "GoodsServices"
  for z in t:
    if( z in important_list):
      count+=1
      if(priority_weight_dict[flag] < priority_weight_dict[z]):
        flag = z  
  if(count > 0):
    count_actiona+=1
    x = {}
    x['post_categories'] = flag
    x['text'] = row['text']
    actionable_tweet.append(x)
print(len(actionable_tweet))
count_actiona
# """
# "Request-GoodsServices",
# "Request-SearchAndRescue",
# "CallToAction-MovePeople",
# "Report-EmergingThreats", 
# "Report-NewSubEvent", 
# "Report-ServiceAvailable"
# """
def load_actionable_verbs(actionable_tweet): 
    actionnable_verb_list_dict={}
    for row in actionable_tweet:
        tweet_text=row['text']
        category=row['post_categories']
        if category not in actionnable_verb_list_dict:
            actionnable_verb_list_dict[category]=[]
        doc=nlp(tweet_text)
        verbs=[token.text for token in doc if token.pos_ == "VERB"]
        for v in verbs:
            if len(v)>=3 and v not in enStop_dict:
                actionnable_verb_list_dict[category].append(v)
    return {e:sorted(nltk.FreqDist(actionnable_verb_list_dict[e]).items(), key=lambda kv: kv[1],reverse=True) for e in actionnable_verb_list_dict}
%time sorted_actionnable_verb_dict=load_actionable_verbs(actionable_tweet)
sorted_actionnable_verb_dict2dict_top10={}
for event in sorted_actionnable_verb_dict:
    sorted_actionnable_verb_dict2dict_top10[event]={}
    for tup in sorted_actionnable_verb_dict[event][:10]:
        sorted_actionnable_verb_dict2dict_top10[event][stemmer.stem(tup[0])]=tup[1]
sorted_actionnable_verb_dict2dict_top10.keys()
print(sorted_actionnable_verb_dict['MovePeople'][:10])
print(sorted_actionnable_verb_dict['EmergingThreats'][:10])
sorted_actionnable_verb_dict['SearchAndRescue'][:10]
stopList = ["http", "https", "rt", "@", ":", "t.co", "co", "amp", "&amp;", "...", "\n", "\r"]
stopList.extend(string.punctuation)

local_tokenizer = TweetTokenizer()
def tokenizer_wrapper(text):
    return local_tokenizer.tokenize(text)

#define vectorizer using sklearn
vectorizer = sklearn.feature_extraction.text.TfidfVectorizer(
    tokenizer=tokenizer_wrapper,
    ngram_range=(1, 1),
    stop_words=stopList, #We do better when we keep stopwords
    use_idf=True,
    smooth_idf=False,
    norm=None, #Applies l2 norm smoothing
    decode_error='replace',
    max_features=10000,
    min_df=4,
    max_df=0.501)

analyzer = vectorizer.build_analyzer()

def normalize(s):
        """
        Given a text, cleans and normalizes it. Feel free to add your own stuff.
        From: https://www.kaggle.com/mschumacher/using-fasttext-models-for-robust-embeddings
        """
        s = s.lower()
        # Replace numbers and symbols with language
        s = s.replace('&', ' and ')
        s = s.replace('@', ' at ')
        s = s.replace('0', 'zero')
        s = s.replace('1', 'one')
        s = s.replace('2', 'two')
        s = s.replace('3', 'three')
        s = s.replace('4', 'four')
        s = s.replace('5', 'five')
        s = s.replace('6', 'six')
        s = s.replace('7', 'seven')
        s = s.replace('8', 'eight')
        s = s.replace('9', 'nine')
        return s

def tokenize_tweet(string):
    string=string.lower()
    # Clean and Refine (remove URL, special characters)
    giant_url_regex = ('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|'
        '[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+')
    string = re.sub(giant_url_regex, "", string)
    
    string = re.sub(r"\'s", "s", string)
    string = re.sub(r"\'ve", "ve", string)
    string = re.sub(r"\'t", "t", string)
    string = re.sub(r"\'re", "re", string)
    string = re.sub(r"\'d", "d", string)
    string = re.sub(r"\'ll", "ll", string)
    
    #Skip retweet signs, @ symbols, and special chars such as punctuations
    string = re.sub(r"[^A-Za-z]", " ", string)
    string = re.sub(r"\s{2,}", " ", string)
    tokens=[]
    for w in string.split():
        # OOV Correction
        if w in oov_dict:
            w=oov_dict[w]
        # Short words skip and Stop word removal
        if len(w) >= 3 and w not in enStop_dict:
            tokens.append(w)
    return tokens

def vectorize(sentence):
    tokenized = tokenize_tweet(sentence)
    wvs = []
    for t in tokenized:
        if t in model:
            v = model[t]
            norm = np.linalg.norm(v)
            normed_v = v / norm
            wvs.append(normed_v) 
    m = np.array(wvs)
    normed_m = np.mean(m, axis=0)
    return normed_m

# construct X matrix
def to_matrix(raw_tweet_texts):
    X_matrix = np.zeros((len(raw_tweet_texts), 300))
    for index,s in enumerate(raw_tweet_texts):
        sv = vectorize(s)
        if not np.isnan(sv).any():
            X_matrix[index,:]=sv
        else:
            print(index,s)
    return X_matrix
def getTweetTextsbyIT(it="MovePeople"):
    return df_combined[df_combined['post_categories'].astype(str).str.contains(it)].reset_index(drop=True)['text']
df_combined.head()
print(getTweetTextsbyIT('MovePeople')[8])
" ".join(tokenize_tweet(getTweetTextsbyIT('MovePeople')[20]))
test_tweet=getTweetTextsbyIT('MovePeople')[1]
test_tweet="I hate you"
print(test_tweet)
import spacy
nlp = spacy.load('en')

doc = nlp(test_tweet)

# Analyze syntax
print("Noun phrases:", [chunk.text for chunk in doc.noun_chunks])
print("Verbs:", [token.lemma_ for token in doc if token.pos_ == "VERB"])
# Find named entities, phrases and concepts
for entity in doc.ents:
    print(entity.text, entity.label_)
    
import nltk
nltk.download('vader_lexicon')
sentiment_analyzer = VS()

sentiment = sentiment_analyzer.polarity_scores("I like you")
sentiment
sorted_actionnable_verb_dict2dict_top10['EmergingThreats']
"""
Features considered for boosting are as follows:
1. No. of hashtags (numeric)
2. No. of special verbs, such as, trapped, stuck, move etc (dataset statistical analysis)
3. Sentiment polarity (categorical,, -1, 0 or 1)
4. Tweet length (word_length, char_length, numeric)
5. URL count (numeric)
6. Digit count (int, numeric)
7. Retweet check (0 or 1)
8. caps ratio (float, numeric)
9. Special chars count (@, ! and ?, normalized float numeric)
10. : in first token check (0 or 1)
11. Named Entity count (numeric)
"""
def return_features(tweet_text):
    #1. No. of hashtags (numeric)
    hashtag_count=tweet_text.count("#")
    #3. Sentiment polarity (categorical,, -1, 0 or 1)
    sentiment = sentiment_analyzer.polarity_scores(tweet_text)
    sentiment_cpd=sentiment['compound']
    sentiment_neg=sentiment['neg']
    sentiment_neu=sentiment['neu']
    sentiment_pos=sentiment['pos']
  
    #2. No. of special verbs such as, trapped, stuck, move etc (dataset statistical analysis)
    #['MovePeople', 'EmergingThreats', 'GoodsServices', 'SearchAndRescue', 'ServiceAvailable']
    tokens=tokenize_tweet(tweet_text)
    is_action1=int(any(stemmer.stem(i) in sorted_actionnable_verb_dict2dict_top10['MovePeople'] for i in tokens))
    is_action2=int(any(stemmer.stem(i) in sorted_actionnable_verb_dict2dict_top10['EmergingThreats'] for i in tokens))
    is_action3=int(any(stemmer.stem(i) in sorted_actionnable_verb_dict2dict_top10['GoodsServices'] for i in tokens))
    is_action4=int(any(stemmer.stem(i) in sorted_actionnable_verb_dict2dict_top10['SearchAndRescue'] for i in tokens))
    is_action5=int(any(stemmer.stem(i) in sorted_actionnable_verb_dict2dict_top10['ServiceAvailable'] for i in tokens))
    
    #4. Tweet length (word_length, char_length, numeric)
    tokens=tweet_text.split(" ")
    word_length=len(tokens)
    char_length=len(tweet_text)
    
    #5. URL count (numeric)
    url_count=len(re.findall('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|'
        '[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+',tweet_text))
    #6. Digit count (int, numeric)
    digital_count=len( re.findall('[0-9]+', tweet_text))
    
    #7. Retweet check (0 or 1)
    is_retweet=int(tweet_text[:2]=="RT")
    # 8. caps ratio (float, numeric)
    caps_ratio=len("".join(re.findall('[A-Z]+', tweet_text)))/char_length
    
    #9. Special chars count
    at_count=tweet_text.count("@")
    exclaim_count=tweet_text.count("!")
    question_count=tweet_text.count("?")
    
    #10. : in first token check (0 or 1)
    colon_check=int(":" in tokens[0])
    #11. Named Entity count (numeric)
    doc = nlp(tweet_text)
    ner_count=len(doc.ents)
#     print(doc.ents)
    
    return_list=[hashtag_count,sentiment_cpd,sentiment_neg,sentiment_neu,sentiment_pos,is_action1,
                    is_action2,is_action3,is_action4,is_action5,word_length,char_length,
                    url_count,digital_count,is_retweet,caps_ratio,at_count,exclaim_count,question_count,
                    colon_check,ner_count]
    return return_list

def normalize_by_columns(crafted_features_matrix,columns_to_normalize=["hashtag_count","word_length","char_length","url_count","digital_count","at_count","exclaim_count","question_count","ner_count"]):
    crafted_features_matrix_=crafted_features_matrix.copy()
    to_normalize_features_matrix=crafted_features_matrix_[columns_to_normalize]
    from sklearn import preprocessing
    x = to_normalize_features_matrix.values 
    min_max_scaler = preprocessing.MinMaxScaler()
    x_scaled = min_max_scaler.fit_transform(x)
    normalized_features_matrix = pd.DataFrame(x_scaled,columns=columns_to_normalize)
    normalized_features_matrix.head()
    for column in columns_to_normalize:
        crafted_features_matrix_[column]=normalized_features_matrix[column]
    return crafted_features_matrix_
test_tweet=getTweetTextsbyIT('MovePeople')[1]
print("test for ",test_tweet)
feature_names=["hashtag_count","sentiment_cpd","sentiment_neg","sentiment_neu",
               "sentiment_pos","is_action1","is_action2","is_action3","is_action4",
              "is_action5","word_length","char_length","url_count","digital_count",
              "is_retweet","caps_ratio","at_count","exclaim_count","question_count",
              "colon_check","ner_count"]
import datetime
start_time=datetime.datetime.now()
features=return_features(test_tweet)

for idx,fature_name in enumerate(feature_names):
    print(fature_name,"====>",features[idx])

import os
if os.path.isfile("crafted_features_matrix_2018B.csv"):
    crafted_features_matrix=pd.read_csv("crafted_features_matrix.csv")
    categories = []
    for it in crafted_features_matrix['categories']:
        categories.append(it.split(","))
    crafted_features_matrix['categories']=categories
else:
    feature_names=["post_id","hashtag_count","sentiment_cpd","sentiment_neg","sentiment_neu",
               "sentiment_pos","is_action1","is_action2","is_action3","is_action4",
              "is_action5","word_length","char_length","url_count","digital_count",
              "is_retweet","caps_ratio","at_count","exclaim_count","question_count",
              "colon_check","ner_count","categories","priority"]
    crafted_features_matrix=pd.DataFrame(columns=feature_names)
    texts=df_combined['text']
    postids=df_combined['post_id']
    categories=df_combined['post_categories']
    priority=df_combined['postPriority']
    for index,tweet_text in enumerate(texts):
        if index%1000==0:
            print("Now is processing at index",index)
        features=return_features(tweet_text)
        features.insert(0,postids[index])
        features.append(",".join(categories[index]))
        features.append(priority[index])
        crafted_features_matrix.loc[len(crafted_features_matrix)]=features
    crafted_features_matrix.to_csv("crafted_features_matrix_2018B.csv",index=False)
    
new_crafted_features_matrix=normalize_by_columns(crafted_features_matrix)
print(new_crafted_features_matrix.shape)
new_crafted_features_matrix.head()<jupyter_output>Now is processing at index 0
Now is processing at index 1000
Now is processing at index 2000
Now is processing at index 3000
Now is processing at index 4000
Now is processing at index 5000
Now is processing at index 6000
Now is processing at index 7000
Now is processing at index 8000
Now is processing at index 9000
Now is processing at index 10000
Now is processing at index 11000
Now is processing at index 12000
Now is processing at index 13000
Now is processing at index 14000
Now is processing at index 15000
Now is processing at index 16000
Now is processing at index 17000
Now is processing at index 18000
Now is processing at index 19000
Now is processing at index 20000
Now is processing at index 21000
Now is processing at index 22000
Now is processing at index 23000
Now is processing at index 24000
Now is processing at index 25000
Now is processing at index 26000
Now is processing at index 27000
(27047, 24)<jupyter_text>**ML models for combined features (crafted features + word2vec features)**<jupyter_code>word2vec_matrix=to_matrix(df_combined['text'])
word2vec_matrix.shape
combined_matrix=np.concatenate([word2vec_matrix, new_crafted_features_matrix.iloc[:,1:22].values], axis=1)

#X and y construction for priortiy classification
X_crafted_matrix=combined_matrix
print(X_crafted_matrix.shape)
X_crafted_matrix.shape

priortiy2label={"Low":0,"Unknown":0,"Medium":1,"High":2,"Critical":3}
label2priority={0: 'Low', 1: 'Medium', 2: 'High', 3: 'Critical'}
new_crafted_features_matrix_=new_crafted_features_matrix.replace({"priority":priortiy2label})
y_crafted_pri=new_crafted_features_matrix_['priority'].values
print(y_crafted_pri.shape)
print(y_crafted_pri)


y_crafted_pri=y_crafted_pri.astype('int')

from sklearn.model_selection import train_test_split
features_train, features_test, labels_train, labels_test = train_test_split(X_crafted_matrix, 
                                                                            y_crafted_pri, 
                                                                            test_size=0.2, 
                                                                            random_state=0)
# from imblearn.over_sampling import SMOTE
# oversampler=SMOTE('not majority')
# features_train,labels_train=oversampler.fit_sample(features_train,labels_train)
# print(features_train.shape)
# print(len(labels_train.shape))

# By GridSearch and KFold validation, we get the highest accurancy with parameter C=xx
# from sklearn.model_selection import GridSearchCV
# from sklearn.model_selection import StratifiedKFold
# val_clf = LogisticRegression()
# C_params = [0.001, 0.01, 0.1, 1, 10,100]
# # gamma_params=[0.01,0.1,1,10,100]
# param_grid = dict(C=C_params)
# kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=7)
# grid_search = GridSearchCV(val_clf, param_grid, n_jobs=-1, cv=kfold)
# grid_result = grid_search.fit(features_train, labels_train)
# print("Best: %f using %s" % (grid_result.best_score_, grid_result.best_params_))
# means = grid_result.cv_results_['mean_test_score']
# params = grid_result.cv_results_['params']
# for mean, param in zip(means, params):
#     print("%f  with: %r" % (mean, param))

# from sklearn.naive_bayes import GaussianNB
# gnb_pri_model = GaussianNB()
# %time gnb_pri_model.fit(features_train, labels_train)
# pri_model=gnb_pri_model

#poor on critical although hits a higher average precision and recall over GaussianNB
# from sklearn.linear_model import LogisticRegression
# from sklearn.svm import LinearSVC
# from sklearn.svm import SVC
# # lsvc_pri_model = LinearSVC(C=0.1)
# lsvc_pri_model = LogisticRegression(C=100)
# # lsvc_pri_model = SVC(C=0.1,kernel="linear")
# %time lsvc_pri_model.fit(features_train, labels_train)
# pri_model=lsvc_pri_model

# print('Accuracy of LinearSVC classifier on FULL set with oversampling for priority (Train set): {:.4f}'
#      .format(pri_model.score(features_train, labels_train)))
# print('Accuracy of LinearSVC classifier on FULL set with oversampling for priority (Test set): {:.4f}'
#      .format(pri_model.score(features_test, labels_test)))<jupyter_output>(27047,)
[0 2 2 ... 0 0 0]<jupyter_text>Logistic Regression<jupyter_code>from sklearn.linear_model import LogisticRegression
from sklearn.svm import LinearSVC
from sklearn.svm import SVC

lsvc_pri_model = LogisticRegression(C=1000,max_iter=10000)
# lsvc_pri_model = SVC(C=0.1,kernel="linear")
%time lsvc_pri_model.fit(features_train, labels_train)
pri_model_1=lsvc_pri_model
print('Accuracy of Logistic Regression classifier on FULL set with oversampling for priority (Train set): {:.4f}'
     .format(pri_model_1.score(features_train, labels_train)))
print('Accuracy of Logistic Regression classifier on FULL set with oversampling for priority (Test set): {:.4f}'
     .format(pri_model_1.score(features_test, labels_test)))<jupyter_output>CPU times: user 2min 29s, sys: 23.1 s, total: 2min 52s
Wall time: 1min 27s
Accuracy of Logistic Regression classifier on FULL set with oversampling for priority (Train set): 0.7742
Accuracy of Logistic Regression classifier on FULL set with oversampling for priority (Test set): 0.7627<jupyter_text>**Linear SVC**<jupyter_code>linearsvc_pri_model = LinearSVC(C=0.1)
%time linearsvc_pri_model.fit(features_train,labels_train)
pri_model_2 = linearsvc_pri_model
print('Accuracy of LinearSVC classifier on FULL set with oversampling for priority (Train set): {:.4f}'
     .format(pri_model_2.score(features_train, labels_train)))
print('Accuracy of LinearSVC classifier on FULL set with oversampling for priority (Test set): {:.4f}'
     .format(pri_model_2.score(features_test, labels_test)))<jupyter_output>CPU times: user 2.94 s, sys: 9.92 ms, total: 2.95 s
Wall time: 2.97 s
Accuracy of LinearSVC classifier on FULL set with oversampling for priority (Train set): 0.7634
Accuracy of LinearSVC classifier on FULL set with oversampling for priority (Test set): 0.7664<jupyter_text>**SVC Model**<jupyter_code>from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import StandardScaler
svc_model =  make_pipeline(StandardScaler(),SVC(gamma='auto'))
svc_model.fit(features_train,labels_train)
pri_model = svc_model
print('Accuracy of LinearSVC classifier on FULL set with oversampling for priority (Train set): {:.4f}'
     .format(pri_model.score(features_train, labels_train)))
print('Accuracy of LinearSVC classifier on FULL set with oversampling for priority (Test set): {:.4f}'
     .format(pri_model.score(features_test, labels_test)))
predicted_test = pri_model.predict(features_test)
from sklearn.metrics import classification_report
class_names=[label2priority[each] for each in range(0,4)]
print(classification_report(labels_test, predicted_test,target_names=class_names))
print("Evaluate on training set before SMOTE")
from sklearn.metrics import classification_report
class_names_=[label2priority[each] for each in range(0,4)]
print(classification_report(y_crafted_pri, pri_model.predict(X_crafted_matrix),target_names=class_names_))<jupyter_output>/usr/local/lib/python3.7/dist-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))<jupyter_text>**Multi-Label classification for information type**<jupyter_code>def onehot_to_labels(onehot):
    labels=[]
    for index,each in enumerate(onehot):
        if each == 1:
            labels.append(list(multilabel_binarizer.classes_)[index])
    return labels

def labels_to_onehot(label_list):
    onehot=[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
    for each in label_list:
        onehot[list(multilabel_binarizer.classes_).index(each)]=1
    return np.array(onehot)

def normalize_prob(raw_list=['0','1','2','3']):
    raw_list=[float(i) for i in raw_list]
    max_raw_score=np.max(raw_list)
    min_raw_score=np.min(raw_list)
    normalized=[]
    for each in raw_list:
        normalized.append(each/(max_raw_score-min_raw_score))
    return normalized

index2label={0: 'Advice',
 1: 'CleanUp',
 2: 'ContextualInformation',
 3: 'Discussion',
 4: 'Donations',
 5: 'EmergingThreats',
 6: 'Factoid',
 7: 'FirstPartyObservation',
 8: 'GoodsServices',
 9: 'Hashtags',
 10: 'InformationWanted',
 11: 'Irrelevant',
 12: 'Location',
 13: 'MovePeople',
 14: 'MultimediaShare',
 15: 'NewSubEvent',
 16: 'News',
 17: 'Official',
 18: 'OriginalEvent',
 19: 'SearchAndRescue',
 20: 'Sentiment',
 21: 'ServiceAvailable',
 22: 'ThirdPartyObservation',
 23: 'Volunteer',
 24: 'Weather'}

import warnings
warnings.filterwarnings('ignore')

from sklearn.preprocessing import MultiLabelBinarizer
multilabel_binarizer = MultiLabelBinarizer()
multilabel_binarizer.fit(df_combined['post_categories'])
X_matrix=X_crafted_matrix
# transform target variable
y_matrix = multilabel_binarizer.transform(df_combined['post_categories'])
print(df_combined['text'][2000])
print(onehot_to_labels(y_matrix[2000]))
y_matrix[0]
X_crafted_matrix.shape<jupyter_output><empty_output><jupyter_text>**model for info types**<jupyter_code>from sklearn.model_selection import train_test_split
features_train, features_test, labels_train, labels_test = train_test_split(X_matrix, y_matrix, test_size=0.2, random_state=9)
from sklearn.multiclass import OneVsRestClassifier
print(features_test.shape,features_train.shape)<jupyter_output>(5410, 321) (21637, 321)<jupyter_text>**logistic Regression**<jupyter_code>from sklearn.linear_model import LogisticRegression
ml_m = LogisticRegression()
clf1 = OneVsRestClassifier(ml_m)

%time clf1.fit(features_train, labels_train)
clf=clf1
clf_predicted_label = clf.predict(features_test)
print(features_test.shape,features_train.shape)
print(clf_predicted_label.shape,labels_test.shape)
print(clf_predicted_label.shape)
print(len(clf_predicted_label))
information_type = set()
a=set()
for t in range(df_combined.shape[0]):
  for z in df_combined['post_categories'][t]:
    if(df_combined['post_categories'][t]==' '):
      a.add(df_combined['post_id'])
    information_type.add(z)
# information_type
a
class_names = [index2label[each] for each in range(25)]

class_names.append('')
class_names
from sklearn.metrics import classification_report
print(classification_report(labels_test, clf_predicted_label))

print('Accuracy of Logistic Regression classifier for priority (Train set): {:.4f}'
     .format(clf.score(features_train, labels_train)))
print('Accuracy of Logistic Regression classifier for priority (Test set): {:.4f}'
     .format(clf.score(features_test, labels_test)))<jupyter_output>precision    recall  f1-score   support

           0       0.00      0.00      0.00        12
           1       0.20      0.00      0.01       308
           2       0.00      0.00      0.00        25
           3       0.82      0.29      0.43       309
           4       0.55      0.05      0.09       511
           5       0.58      0.22      0.32       130
           6       0.33      0.00      0.01       264
           7       0.59      0.16      0.25       592
           8       0.61      0.04      0.07       834
           9       0.00      0.00      0.00        21
          10       0.77      0.44      0.56      1463
          11       0.00      0.00      0.00        41
          12       0.65      0.13      0.22      1061
          13       0.57      0.08      0.13       276
          14       0.00      0.00      0.00        37
          15       0.62      0.27      0.37      1046
          16       0.00      0.00      0.00       115
          17       0.63    [...]<jupyter_text>**naive bayes**<jupyter_code>from sklearn.naive_bayes import GaussianNB
ml_m = GaussianNB()
clf2 = OneVsRestClassifier(ml_m)
%time clf2.fit(features_train, labels_train)
clf=clf2
clf_predicted_test = clf.predict(features_test)
print(classification_report(labels_test, clf_predicted_test,target_names=class_names))

print('Accuracy of BernoulliNB classifier for priority (Train set): {:.4f}'
     .format(clf.score(features_train, labels_train)))
print('Accuracy of BernoulliNB classifier for priority (Test set): {:.4f}'
     .format(clf.score(features_test, labels_test)))<jupyter_output>precision    recall  f1-score   support

               Advice       0.00      0.08      0.00        12
              CleanUp       0.12      0.75      0.20       308
ContextualInformation       0.02      0.80      0.04        25
           Discussion       0.15      0.77      0.24       309
            Donations       0.15      0.69      0.24       511
      EmergingThreats       0.15      0.78      0.25       130
              Factoid       0.13      0.67      0.22       264
FirstPartyObservation       0.26      0.68      0.38       592
        GoodsServices       0.26      0.55      0.35       834
             Hashtags       0.03      0.67      0.05        21
    InformationWanted       0.45      0.60      0.51      1463
           Irrelevant       0.03      0.68      0.06        41
             Location       0.31      0.65      0.42      1061
           MovePeople       0.14      0.79      0.24       276
      MultimediaShare       0.04      0.73      0.08  [...]<jupyter_text>**2019b Submission (on 2019btrecis-test set)**<jupyter_code>e= ['albertawildfires2019a','coloradostemshooting2019a','cyclonekenneth2019a','philippinesearthquake2019a','sandiegosynagogueshooting2019a','southafricafloods2019c']

import gzip
import json
import os
def read_json_gz(file_name):
  d = []
  with gzip.open(file_name,'r') as fin:
    for line in fin:
      json_str = line.decode('utf-8')
      ts = json.loads(json_str)
      # if ts not in d:#modified
      d.append(ts)
      # else:
      #   print(ts)
  fin.close()
  return d

def read_fol(fol_path):
  files_name = os.listdir(fol_path)
  files_path = [fol_path + x for x in files_name]
  print(files_path)
  content = []
  for f in files_path:
    content = content + read_json_gz(f)

  return content
data_fol_path = '/content/drive/Shareddrives/data/2019 B test/'
ct = read_fol(data_fol_path)
results=[]
for x in ct:
  p={}
  p['text']=x['allProperties']['text']
  p['post_id']=x['allProperties']['id']
  p['event_id']=x['topic']
  results.append(p)

print(len(results))
df_test_2019b=pd.DataFrame(results)
X_test_2019b_word2vec_matrix=to_matrix(df_test_2019b['text'])
import os
if os.path.isfile("trecis2019b_fbtest_matrix.csv"):
    X_test_2019b_fb=pd.read_csv("trecis2019b_fbtest_matrix.csv")
else:
    feature_names=["hashtag_count","sentiment_cpd","sentiment_neg","sentiment_neu",
               "sentiment_pos","is_action1","is_action2","is_action3","is_action4",
              "is_action5","word_length","char_length","url_count","digital_count",
              "is_retweet","caps_ratio","at_count","exclaim_count","question_count",
              "colon_check","ner_count"]
    trecis2019b_fbtest_matrix=pd.DataFrame(columns=feature_names)
    texts=df_test_2019b['text']
    for index,tweet_text in enumerate(texts):
        if index%1000==0:
            print("Now is processing at index",index)
        features=return_features(tweet_text)
        trecis2019b_fbtest_matrix.loc[len(trecis2019b_fbtest_matrix)]=features
    trecis2019b_fbtest_matrix.to_csv("trecis2019b_fbtest_matrix.csv",index=False)
    

X_test_2019b_fb=normalize_by_columns(X_test_2019b_fb)
X_test_2019b=np.concatenate([X_test_2019b_word2vec_matrix, X_test_2019b_fb.iloc[:,0:21].values], axis=1)
print(X_test_2019b.shape)
X_test_2019b
t = 0.51 # this is set emprically
predicted_test1_proba=clf1.predict_proba(X_test_2019b)
predicted_test1_proba=[normalize_prob(each) for each in predicted_test1_proba]
predicted_test1_proba=np.array(predicted_test1_proba)

predicted_test2_proba=clf2.predict_proba(X_test_2019b)
predicted_test2_proba=[normalize_prob(each) for each in predicted_test2_proba]
predicted_test2_proba=np.array(predicted_test2_proba)
combined_predict_proba=(predicted_test1_proba+predicted_test2_proba)/2
y_pred_prob_2019=combined_predict_proba
predicted_2019 = (y_pred_prob_2019 >= t).astype(int)

# prediction for priority
predicted_pri = pri_model_1.predict(X_test_2019b)
# predicted
predictd_labels_2019=multilabel_binarizer.inverse_transform(predicted_2019)
# predictd_labels_2019
all_predicted_categories=[]
all_predicted_categories = sum([list(tup) for tup in predictd_labels_2019],[])
len(set(all_predicted_categories))

all_predicted_categories_freq = nltk.FreqDist(all_predicted_categories)
# create dataframe
all_predicted_categories_df = pd.DataFrame({'Category': list(all_predicted_categories_freq.keys()), 
                              'Count': list(all_predicted_categories_freq.values())})
all_predicted_categories_df
df_test_2019b["event_id"].shape
runtag="baseline"
runlabel="UCD"+runtag

def normalize_priority(raw_priority=['0','1','2','3']):
    raw_priority=[float(i) for i in raw_priority]
    max_priority_score=np.max(raw_priority)
    min_priority_score=np.min(raw_priority)
    normalized=[]
    for each in raw_priority:
        normalized.append((each+1)/(max_priority_score-min_priority_score))
#         normalized.append(0.75*each/(max_priority_score-min_priority_score)+(0.25*max_priority_score-min_priority_score)/(max_priority_score-min_priority_score))
    return normalized
import math
short2longITs2019b = {'Advice': 'Other-Advice',
                      'CleanUp': 'Report-CleanUp',
                      'ContextualInformation': 'Other-ContextualInformation',
                      'Discussion': 'Other-Discussion',
                      'Donations': 'CallToAction-Donations',
                      'EmergingThreats': 'Report-EmergingThreats',
                      'Factoid': 'Report-Factoid',
                      'FirstPartyObservation': 'Report-FirstPartyObservation',
                      'GoodsServices': 'Request-GoodsServices',
                      'Hashtags': 'Report-Hashtags',
                      'InformationWanted': 'Request-InformationWanted',
                      'Location': 'Report-Location',
                      'MovePeople': 'CallToAction-MovePeople',
                      'MultimediaShare': 'Report-MultimediaShare',
                      'NewSubEvent': 'Report-NewSubEvent',
                      'News': 'Report-News',
                      'Official': 'Report-Official',
                      'OriginalEvent': 'Report-OriginalEvent',
                      'SearchAndRescue': 'Request-SearchAndRescue',
                      'Sentiment': 'Other-Sentiment',
                      'Irrelevant': 'Report-Irrelevant',
                      'ServiceAvailable': 'Report-ServiceAvailable',
                      'ThirdPartyObservation': 'Report-ThirdPartyObservation',
                      'Volunteer': 'CallToAction-Volunteer',
                      'Weather': 'Report-Weather'
                      }

event2test = {"albertaWildfires2019": "TRECIS-CTIT-H-Test-029",
                          "cycloneKenneth2019": "TRECIS-CTIT-H-Test-030",
                          "philippinesEarthquake2019": "TRECIS-CTIT-H-Test-031",
                          "coloradoStemShooting2019": "TRECIS-CTIT-H-Test-032",
                          "southAfricaFloods2019": "TRECIS-CTIT-H-Test-033",
                          "sandiegoSynagogueShooting2019": "TRECIS-CTIT-H-Test-034"}
# A naive approach: based on the averaged priority weights computed by information types in training set

######modified
info_type_priority_weight_dict = {'Advice': 0.22862577231414025, 'CleanUp': 0.35653795010024114,
                                  'News': 0.3270722902848146, 'Discussion': 0.19307661376501264,
                                  'Donations': 0.3612767541359874, 'EmergingThreats': 0.8537984236364585,
                                  'Factoid': 0.48917308056907677, 'FirstPartyObservation': 0.1420197717109446,
                                  'GoodsServices': 0.6959346525995092, 'Hashtags': 0.2687451987058043,
                                  'InformationWanted': 0.6688596972384949, 'Irrelevant': 0.0010061402766415601,
                                  'OriginalEvent': 0.09192532381788204, 'MovePeople': 0.9253136744180216,
                                  'MultimediaShare': 0.3128463518966703, 'Official': 0.6785618345463643,
                                  'ContextualInformation': 0.1405539758609653, 'SearchAndRescue': 1.0010061402766415,
                                  'Sentiment': 0.039519645594885515, 'ServiceAvailable': 0.7669372401298534,
                                  'NewSubEvent': 0.9176682650090996, 'ThirdPartyObservation': 0.2703955630411556,
                                  'Location': 0.03711651480530716, 'Volunteer': 0.3839639462617982,
                                  'Weather': 0.33196659834790165,'':0.0}

# priority prediction
normalized_predicted_pri=normalize_priority(predicted_pri)

info_type_priority_weight_list=list(info_type_priority_weight_dict.values())
max_pri=np.max([float(i) for i in info_type_priority_weight_list])
min_pri=np.min([float(i) for i in info_type_priority_weight_list])

# Step 2: Convert the submission file to the standard format
# generate submission file
# import core.label2code as l2c
# index2label = l2c.index2label
str_pre = ""

import random
sub_dict={}
for index,each in enumerate(predictd_labels_2019):
    each=list(each)
    if len(each)==0:
        each=['Irrelevant']
    elif len(each)>1 and 'Irrelevant' in each:
        each.remove('Irrelevant')
    elif 'Unknown' in each:
        each.remove('Unknown')
        
    p=np.mean([float(info_type_priority_weight_dict[e]) for e in each])
#     p = float(info_type_priority_weight_dict[index2label[each]])
    # low=0.25, medium=0.5, high=0.75 and critical=1.0. 
    # equation for priority calculation where p is parameter =0.5 by default
    # p*predicted_score+(1-p)*statistic_weight
    parameter_p=0.5
    predicted_score=normalized_predicted_pri[index]
#   statistic_weight=normalize_priority([p])[0]
    statistic_weight=p/(max_pri-min_pri)
#     statistic_weight=0.75*p/(max_pri-min_pri)+(0.25*max_pri-min_pri)/(max_pri-min_pri)
    p_final=parameter_p*predicted_score+(1-parameter_p)*statistic_weight

    event_id=df_test_2019b.iloc[index]['event_id']
    test_id=event2test[event_id]
    post_id=df_test_2019b.iloc[index]['post_id']

#     labels=[short2longITs2019A[index2label[each]] for each in multi_label_predicted[index]]
    
######modified    
    # print(each)
    for i in each:
      if(i==''):
        each.remove(i)
#######modified end



    labels=[short2longITs2019b[e] for e in each]

    if math.isnan(p_final):
        p_final=0.0
    if p_final>1.0:
        p_final=1.0
        
    line_sub=test_id+"\tQ0\t"+str(post_id)+"\t"+ "#" + "\t" + str(p_final) +"\t"+str(labels)+"\t"+runlabel+ "\n"
#     str_pre +=line_sub
    if event_id not in sub_dict:
#         sub_dict[post_id]=[event_id]
        sub_dict[event_id]={line_sub:p_final}
    else:
        sub_dict[event_id][line_sub]=p_final
str_pre=""
for each in sub_dict:
    rank_inevent=1
    sub_inevent=sub_dict[each]
    sub_inevent_sorted=sorted(sub_inevent.items(), key=lambda kv: kv[1],reverse=True)
    for tup in sub_inevent_sorted:
        line_sub=tup[0].replace("#",str(rank_inevent))
        str_pre+=line_sub
        rank_inevent+=1
with open(runlabel+".txt", "w") as f:
    f.write(str_pre)

print("Submission for "+runlabel+" is done")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11902267098494951, 'fraction_numerical': 0.05832063250142884, 'mean_word_length': 3.8772357723577238, 'pattern_counts': {'":': 14, '<': 38, '<?xml version=': 0, '>': 44, 'https://': 3, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 54, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '153921', 'license': 'MIT', 'n_tokens_mistral': 15858, 'n_tokens_neox': 14110, 'n_words': 2640, 'path': 'newucd.ipynb', 'repo_name': 'vikrantkumar256/Incident-Stream-IIT-BHU', 'size': '41992'}