<jupyter_start><jupyter_text>`WHEN Frequency for Activity AU Description` `Created/Modified By: <NAME>` * `1, Rewrite architecture of the code`* `2, Fix a warning "a value is trying to be set on a copy"; using df.at[] or df.loc[] instead of df[][]`* `3, Modify code so as to capture adhoc & ad hoc` * `4, Add similarity attribute to compare predictions and ground truth`* `5, Modify code so as to capture WHEN containing conjunction such as annual and quarterly `* `6, Remove stop words on predictions and ground truth before comparing similarity`* `7, Modify code so as to capture WHEN containing ... as ... basis (monthly as needed basis)`* `8, Be able to capture bi-..., bi ..., bi... etc`* `9, Solve error, "float is not subscriptable", desc = str(desc)`* `10, Add confidence, starting index, ending index`<jupyter_code>%%time

import os
import re
import torch
import numpy as np
import pandas as pd
from tqdm import tqdm
import matplotlib.pyplot as plt
from sklearn.feature_extraction.text import TfidfVectorizer
import torch.nn.functional as F

#Deep Learning Libraries
import spacy
from transformers import BertForQuestionAnswering
from transformers import BertTokenizer

from sentence_transformers import SentenceTransformer
from transformers import AutoTokenizer, AutoModel, AutoModelForQuestionAnswering,pipeline

import nltk
# nltk.data.path.append('/home/mca_fix/share/nltk_data/') 
nltk.data.path.append('/home/mcafixmlpython/lib/nltk_data/') 
from nltk.stem import WordNetLemmatizer
wordnet_lemmatizer = WordNetLemmatizer()


# nlp = spacy.load("/home/mca_fix/share/en_core_web_md-3.0.0/en_core_web_md/en_core_web_md-3.0.0")
nlp = spacy.load("/home/mcafixmlpython/lib/en_core_web_md-3.0.0/en_core_web_md/en_core_web_md-3.0.0")

# Loading BERT-BASE-NLI-MEAN-TOKENS for similarity

# simi_PATH = '/home/mca_fix/share/bert-base-nli-mean-tokens/'
simi_PATH = '/home/mcafixmlpython/lib/bert-base-nli-mean-tokens/'

simi_tokenizer=AutoTokenizer.from_pretrained(simi_PATH)
simi_model=AutoModel.from_pretrained(simi_PATH,local_files_only=True)

import warnings
warnings.filterwarnings('ignore')
%%time

def similar(sent):
    tokens={'input_ids':[],'attention_mask':[]}
    for sentence in sent:
        new_tokens=simi_tokenizer.encode_plus(sentence,max_length=128,truncation=True,padding='max_length',return_tensors='pt')
        tokens['input_ids'].append(new_tokens['input_ids'][0])
        tokens['attention_mask'].append(new_tokens['attention_mask'][0])
    #reformat list of tensors into single tensor
    tokens['input_ids']=torch.stack(tokens['input_ids'])
    tokens['attention_mask']=torch.stack(tokens['attention_mask'])
    #processing tokens
    outputs=simi_model(**tokens)
    outputs.keys()
    embeddings=outputs.last_hidden_state
    attention_mask=tokens['attention_mask']
    mask=attention_mask.unsqueeze(-1).expand(embeddings.size()).float()
    masked_embeddings=embeddings * mask
    summed = torch.sum(masked_embeddings,1)
    summed_mask=torch.clamp(mask.sum(1),min=1e-9)
    mean_pooled=summed/summed_mask
    from sklearn.metrics.pairwise import cosine_similarity
    mean_pooled=mean_pooled.detach().numpy()
    x=cosine_similarity([mean_pooled[0]],mean_pooled[1:])
    return x[0][0]
from nltk.corpus import stopwords

def remove_stopwords(input_text):
    #print("in remove_stopwords\n",input_text)
       
    stopwords_list = stopwords.words('english')
    newStopWords = ['citi']
    stopwords_list.extend(newStopWords)
        
    # Some words which might indicate a certain sentiment are kept via a whitelist
    #whitelist = ["n't", "not", "no"]
      
    whitelist = ["n't"]
    words = input_text.split() 
    clean_words = [word for word in words if (word not in stopwords_list or word in whitelist) and len(word) > 2]          
    return " ".join(clean_words)<jupyter_output><empty_output><jupyter_text>`20220113_ARCMs_sp.xlsx`<jupyter_code># %%time

# df = pd.read_excel('20220113_ARCMs_sp.xlsx', sheet_name="AU-ARCM Details", engine='openpyxl')

# # df = df.applymap(str)

# print(df.shape)
# df.head()<jupyter_output><empty_output><jupyter_text>`1000 ARCMs.xlsx`<jupyter_code>%%time 

df = pd.read_excel(io='1000 ARCMs.xlsx', sheet_name="1000 ARCMs", engine='openpyxl')

# df = df[df["Activity Description contains > 20 words"] == "Pass"]
df = df[["Activity Instance Id (for AU)", "Activity AU Description", "Frequency of the activity"]]
df.reset_index(drop=True, inplace=True)

df.dropna(axis=0, inplace=True)

# # coerce type conversion to every element of a DataFrame
# df = df.applymap(str)

# text preprocessing
# df['Activity AU Description'] = df['Activity AU Description'].str.strip().str.lower()
# df['Activity AU Description'] = df['Activity AU Description'].str.replace('[^\w\s]','') # semi-annual -> semiannual

print(df.shape)
df.head()
when_frequency_lst = ["daily", "weekly", 
#                       "biweekly", "realtime", "event driven", # *** from control#
                      "monthly", "quarterly", "yearly", "semiannual", "semiannually",
                      "annual", # *** from activity# "ongoing", "needed", 
                      "annually", 
                      # outliers # 
                      "each month", "every month", "calendar month",  
                      # "needed basis", "required basis",
                      "ad hoc", "adhoc"]

from typing import Tuple

def activity_when(desc: str) -> Tuple[str, str, int, int]:
    """
    
    
    """
    
    # Text preprocessing
    desc = str(desc)
#     desc = re.sub('[^\w\s]', '', desc.strip().lower())
    desc = desc.lower()
    doc = nlp(desc)
    
    activity_when_answer = None
    
    
    ########################### semi/bi, semi-annually, bi-weekly... ###########################
    semi_bi = ["semi", "bi"]
    for word in when_frequency_lst:
        
        for semi_bi_word in semi_bi:
            
            if f"""{semi_bi_word}-{word}""" in desc:
        
                activity_when_answer = f"""{semi_bi_word}-{word}"""
        
    
    
    ########################### annual and quarterly basis; adhoc or weekly basis... ########################### 
    for token in doc:
        
        # Exclude last three tokens 
        try:
        
            if (token.text in when_frequency_lst) and (doc[token.i+1].pos_ == "CCONJ") and (doc[token.i+2].text in when_frequency_lst) and (doc[token.i+3].text in ["basis", "cadence"]):
                
                span = " ".join([doc[token.i].text, doc[token.i+1].text, doc[token.i+2].text, doc[token.i+3].text])

                if any(word in when_frequency_lst for word in span.split()) and (activity_when_answer is None):
  
                    activity_when_answer = span
        
        except:
            
            pass
    
    
    
    ########################### on ... basis... ########################### 
    for token in doc:
        
        # Exclude the last token 
        try:
            
            if (token.text in when_frequency_lst) and (doc[token.i+1].text in ["basis", "cadence"]):
                
                # ... as ... basis (monthly as needed basis)
                if doc[token.i-2].text in when_frequency_lst:
                    
                    span = " ".join([doc[token.i-2].text, doc[token.i-1].text, token.text, doc[token.i+1].text])
                
                # on a/an ... basis ..
                elif (doc[token.i-2].text == "on") and ((doc[token.i-1].text == "a") or (doc[token.i-1].text == "an")):
                    
                    span = " ".join([doc[token.i-2].text, doc[token.i-1].text, token.text, doc[token.i+1].text])
                
                # on ... basis ...
                elif doc[token.i-1].text == "on":
                    
                    span = " ".join([doc[token.i-1].text, token.text, doc[token.i+1].text])
                
                else:
                    
                    span = " ".join([token.text, doc[token.i+1].text])

                if any(word in when_frequency_lst for word in span.split()) and (activity_when_answer is None):

                        activity_when_answer = span
        
        except:
            
            pass
            
              
    ########################### monthly, quarterly, annually, semi-annually... ###########################       
    for token in doc:
        
        # Exclude the last token 
        try:
        
            if ((token.dep_ == "amod") and (token.pos_ == "ADJ")) or ((token.dep_ == "advmod") and (token.pos_ == "ADV")):
                
                span = token.text

                if any(word in when_frequency_lst for word in span.split()) and (activity_when_answer is None):

                    activity_when_answer = span
        
        except:
            
            pass
                
                
                
    ########################### other outliers... ###########################
    if activity_when_answer is None:
    
        for word in when_frequency_lst:
            
            if word in desc:
                
                activity_when_answer = word
    
    
    
    ########################### confidence ###########################
    if activity_when_answer is None:
        
        confidence = "Low"
        
    else:
        
        confidence = "High"
        
        
        
    ########################### starting index, ending index ###########################
    if activity_when_answer is None:
        
        start, end = None, None
    
    else:
        
        start = desc.find(activity_when_answer)
        
        if start == -1:
            
            start, end = None, None
        
        else:
            
            end = start + len(activity_when_answer)
        
    
    
    return activity_when_answer, confidence, start, end
%%time

# df["when_ans_freq"] = df["Activity AU Description"].apply(func=activity_when)
df[["when_ans_freq", "confidence", "start", "end"]] = df["Activity AU Description"].\
    apply(func=lambda row: activity_when(row)[0:4]).to_list()
df["when_ans_freq"] = df["when_ans_freq"].apply(func=lambda row : "missing" if row is None else row)
df["found"] = (~df["when_ans_freq"].isna())
df['review_priority'] = np.where((df['found'] == False), "High Priority Review because of Missing When Frequency", "No Need to Review")      
df['missing_or_not'] = np.where((df['found'] == False), "Missing", "Not Missing")
df["Frequency of the activity"] = df["Frequency of the activity"].str.strip().str.lower()
df["when_ans_freq"] = df["when_ans_freq"].str.strip().str.lower()
df["similarity"] = df.apply(func=lambda row: similar(sent=[row["Frequency of the activity"], row["when_ans_freq"]]), 
                            axis=1)<jupyter_output>CPU times: user 26min 46s, sys: 1min 40s, total: 28min 26s
Wall time: 7min 26s
{'fraction_non_alphanumeric': 0.1601584085466937, 'fraction_numerical': 0.009946583164487014, 'mean_word_length': 2.574391046741277, 'pattern_counts': {'":': 1, '<': 13, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 3}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '117868', 'license': 'Apache-2.0', 'n_tokens_mistral': 3466, 'n_tokens_neox': 3320, 'n_words': 762, 'path': 'when_frequency_final_py.ipynb', 'repo_name': 'perryisbusy/Transfer-Learning-Playground', 'size': '10857'}