<jupyter_start><jupyter_text>Example proof of concept for a recommender system based on text similarity between incidents Adapted after (main credit goes to): https://www.datacamp.com/community/tutorials/recommender-systems-python  1. Imports<jupyter_code>import pandas as pd
import numpy as np
import time
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import linear_kernel
from nltk.corpus import wordnet, stopwords
from nltk.tokenize import word_tokenize
import re
import nltk

# progress bar
from tqdm.auto import tqdm  # for notebooks<jupyter_output><empty_output><jupyter_text>necessary downloads before using nltk locally<jupyter_code>nltk.download('punkt')
nltk.download('stopwords')<jupyter_output><empty_output><jupyter_text>2. Functions<jupyter_code>def cleaner(text):
    
    """ lowercase and tokenize text, keep only alphabetical cahrs 

    Args: 
        text            (string):      text to clean

    Returns:
        text            (string):      cleaned text

    """
    
    
    text = str(text).lower()
    # adding numbers to regex, possibly information like "BKR01" as location is helpful
    text = re.sub('[^A-Za-z0-9]', ' ', text)
    text = word_tokenize(text)
    text = [token for token in text if token not in stopwords.words('english')]
    
    
    return " ".join(text)
def check_mention_in_document(stri, text):
    
    """Return true if (sub-)string in text

    Args: 
        stri          (string):   string to search for
        text          (string):   text in which is searched

    Returns:
        bool

    """
    
    # check inside list
    if stri in text:
        return True
    else:
        return False
def dataframe_column_to_cosine_sim(df, column):

    """Creates cosine similarity for each document in a dataframe. 

    Args: 
        df              (obj):      DataFrame
        column          (string):   Name of the column

    Returns:
        corpus          (list):     list of documents as strings
        tf_idf_matrix   (obj):      scipy sparse tf-idf matrix
        cosine_sim      (array):    cosine similarity matrix

    """
    start = time.time()
    
    # convert dataframe strings into list of strings
    corpus = df[column].tolist()
    
    # use english stop words
    tfidf_vectorizer = TfidfVectorizer(stop_words='english')
    
    # generate tf-idf vectors for corpus
    tfidf_matrix = tfidf_vectorizer.fit_transform(corpus)
    
    # compute similarity matrix with pairwise scores, faster version
    cosine_sim = linear_kernel(tfidf_matrix, tfidf_matrix)

    end = time.time()

    # get total time
    print('Time taken is {} seconds'.format(round(end-start,4)))
    
    
    return corpus, tfidf_matrix, cosine_sim
def get_recommendations(title, cosine_sim, df, column, top_n):

    """Create item recommendations based on cosine similarity

    Args: 
        title           (string):   Item name
        cosine_sim      (array):    cosine similarity matrix
        df              (obj):      DataFrame
        column          (string):   Column name
        top_n           (int):      amount of similar observations

    Returns:
        df              (obj):      DataFrame with top_n recommendations

    """
    
    # check input value for top_n to be smaller than 20
    try:
        assert top_n < 20
    except:
        raise ValueError('Too high value for top_n. Enter less than 20.')
    

    # adjust index for searching
    df.reset_index(inplace=True)

    # Generate mapping between titles and index
    indices = pd.Series(df.index, index=df[column])

    # Get index of item that matches title
    idx = indices[title]

    # Sort the items based on the similarity scores
    sim_scores = list(enumerate(cosine_sim[idx]))
    sim_scores = sorted(sim_scores, key=lambda x: x[1], reverse=True)

    # Get the scores for n most similar items
    sim_scores = sim_scores[1:top_n]

    # Get the item indices
    item_indices = [i[0] for i in sim_scores]

    # Return the top n most similar items
    return df[column].iloc[item_indices]<jupyter_output><empty_output><jupyter_text>3. Read in data and check values<jupyter_code>df.head(1)
cols_to_drop = ['trans','index','contact_person','comments','loss_potential_comments','observer','shift','risk_area','control_doc', 'start_date', 'end_date']
df.drop(labels=cols_to_drop, axis = 1, inplace = True)
df.dropna(inplace = True)
df.shape<jupyter_output><empty_output><jupyter_text>3.1 Clean title text column<jupyter_code>start = time.time()


df['title_cleaned'] = df['title'].progress_apply(lambda x: cleaner(str(x)))

end = time.time()

# get total time
print('Time taken is {} seconds'.format(round(end-start,4)))<jupyter_output><empty_output><jupyter_text>3.2 Investigate amount of mentions<jupyter_code>df['title_push'] = df['title_cleaned'].apply(lambda x: check_mention_in_document('push', x))
print('amount of mentions is {} which is {} per cent'.format(df['title_push'].sum(), round(df['title_push'].sum()/len(df),4)))
title_values = ['vessel', 'water','wave']
for value in title_values:
    df['title_'+value] = df['title_cleaned'].apply(lambda x: check_mention_in_document(value, x))
for value in title_values:
    print(value+':', df['title_'+value].sum())<jupyter_output>vessel: 1623
water: 897
wave: 87<jupyter_text>Example recommendations based on title<jupyter_code># choose column to index
column = 'title_cleaned'
# prepare text, store returned variables
title_corpus, title_tfidf_matrix, title_cosine_sim = dataframe_column_to_cosine_sim(df, column)
# print cleaned title for first case
test_title = df.iloc[0,-5]

print(test_title)
print(get_recommendations(test_title, title_cosine_sim,
                          df, column, 10))<jupyter_output>21314                                          push system
15302                               seized door handle q05
15589                                   damaged push tubes
35324                             crane damaged operations
26395    bkr01 crew transfer vessel largo push fender s...
171        damage vessel fender boat landing push tube a01
13744            bkr01 beaumaris bay fender poor condition
14162                        c wind artimus damaged fender
5677                                           push lights
Name: title_cleaned, dtype: object
{'fraction_non_alphanumeric': 0.09062253743104808, 'fraction_numerical': 0.013869188337273443, 'mean_word_length': 2.926980198019802, 'pattern_counts': {'":': 0, '<': 28, '<?xml version=': 0, '>': 27, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '369173', 'license': 'MIT', 'n_tokens_mistral': 1995, 'n_tokens_neox': 1880, 'n_words': 545, 'path': 'similar_incidents_recommender.ipynb', 'repo_name': 'DanishDahaka/incident-analysis-ml', 'size': '6345'}