app/utils.py

import os
import re
import nltk
import gradio as gr
import pandas as pd
import seaborn as sns
from statistics import mean
import matplotlib.pyplot as plt
from collections import Counter
from wordcloud import WordCloud
from nltk.sentiment import SentimentIntensityAnalyzer
from preprocessing.preprocess import process_user_input


def create_input_instruction():
    gr.Markdown(
        """
        # Predict Sentiment Labels
        Get started by inputting the conversation into the text box below. The
        conversation should be in the format of `<speaker>:<message>` where speaker is
        either Visitor or Agent. There should only be one message per line.
        
        <b>For example:</b><br/>
       Visitor:  Hi! How are you?<br/>
       Visitor:  Are you there?</br>
       Agent:  Hi! I'm good. And you?
            
        You can also choose from one of the examples below to get a quickstart.
        """
    )


def format_prediction_ouptut(roles, messages, labels):
    output = ""

    for role, message, label in zip(roles, messages, labels):
        output += f"{role}: {message}\n[{label}]\n\n"

    return output


def remove_temp_dir(temp_dir):
    # List all files in the temporary directory
    files = os.listdir(temp_dir)

    # Remove each file within the directory
    for file in files:
        file_path = os.path.join(temp_dir, file)
        if os.path.isfile(file_path):
            os.remove(file_path)
            print(f"=== File '{file}' has been removed.")
        else:
            remove_temp_dir(file_path)
            print(f"=== Directory '{file}' has been removed.")

    os.rmdir(temp_dir)


SENTIMENT_GROUP_MAPPING = {
    "Acceptance": 3,
    "Openness": 3,
    "Interest": 2,
    "Curiosity": 2,
    "Informative": 1,
    "Greeting": 0,
    "None": 0,
    "Uninterested": -1,
    "Anxious": -2,
    "Confused": -2,
    "Annoyed": -2,
    "Remorse": -2,
    "Disapproval": -3,
    "Accusatory": -3,
    "Denial": -3,
    "Obscene": -3,
}

def label_analysis(labeled_conv):
    msg_pattern = r"(Agent|Visitor): (.*)\n\[(.*)\]"
    # find the components of each message, including the speaker, message, and label
    component_lst = re.findall(msg_pattern, labeled_conv)

    labels=[]

    for speaker, _, label in component_lst:
        labels.append(label)
        
    label_counts = Counter(labels)

    # Create a bar plot
    fig1, ax = plt.subplots(figsize=(12, 6))
    sns.barplot(x=list(label_counts.keys()), y=list(label_counts.values()), ax=ax)
    
    plt.title('Label Frequency Distribution')
    plt.xlabel('Labels')
    plt.ylabel('Frequency')
    plt.xticks(rotation=45, ha='right')
    
    labels_text = " ".join(labels)
    wordcloud = WordCloud(width=800, height=400, background_color='white').generate(labels_text)

    # Create a new figure for the word cloud
    fig2, ax2 = plt.subplots(figsize=(10, 5))
    ax2.imshow(wordcloud, interpolation='bilinear')
    ax2.axis('off')  # Turn off the axis

    return fig1,fig2    

def sentiment_flow_plot(labeled_conv):
    """Generate the sentiment flow plot using the output from the label predecting

    Args:
        labeled_conv (str): the displayed output of a sentiment-label prediction.

    Returns:
        _The sentiment flow plot figure.
    """
    msg_pattern = r"(Agent|Visitor): (.*)\n\[(.*)\]"
    # find the components of each message, including the speaker, message, and label
    component_lst = re.findall(msg_pattern, labeled_conv)

    visitor_Y = []
    agent_Y = []

    for speaker, _, label in component_lst:
        if speaker == "Visitor":
            visitor_Y.append(label)
            if agent_Y == []:
                agent_Y.append("None")
            else:
                agent_Y.append(agent_Y[-1])
        else:
            agent_Y.append(label)
            if visitor_Y == []:
                visitor_Y.append("None")
            else:
                visitor_Y.append(visitor_Y[-1])

    X = range(1, len(agent_Y) + 1)
    visitor_Y_converted = [SENTIMENT_GROUP_MAPPING[visitor_Y[i]] for i in range(len(X))]
    agent_Y_converted = [SENTIMENT_GROUP_MAPPING[agent_Y[i]] for i in range(len(X))]

    fig, ax = plt.subplots()
    sns.set(style="whitegrid")

    ax.plot(X, visitor_Y_converted, label="Visitor", color="red", marker="o")
    ax.plot(X, agent_Y_converted, label="Agent", color="green", marker="o")

    plt.legend(loc="upper left", bbox_to_anchor=(1, 1))
    plt.subplots_adjust(right=0.8)

    plt.yticks(ticks=[-3, -2, -1, 0, 1, 2, 3])

    plt.xlabel("Timestamp")
    plt.ylabel("Sentiment Score")
    plt.title("Sentiment Flow Plot")
    plt.close(fig)

    return fig


def sentiment_intensity_analysis(input):
    def analyze_sentiment(text):
        sia = SentimentIntensityAnalyzer()
        scores = [
            sia.polarity_scores(sentence)["compound"]
            for sentence in nltk.sent_tokenize(text)
        ]
        return mean(scores)

    result = process_user_input(input)

    if not result["success"]:
        raise gr.Error(result["message"])

    data = result["data"]
    speakers = [item[1] for item in data]
    messages = [item[2] for item in data]

    df = pd.DataFrame(columns=["Speaker", "Sentiment Intensity", "Timestamp"])

    visitor_previous = {
        "Speaker": ["Visitor"],
        "Sentiment Intensity": [0],
        "Timestamp": [0],
    }
    agent_previous = {
        "Speaker": ["Agent"],
        "Sentiment Intensity": [0],
        "Timestamp": [0],
    }
    for i in range(len(messages)):
        new_row = {
            "Speaker": [speakers[i]],
            "Sentiment Intensity": [analyze_sentiment(messages[i])],
            "Timestamp": [i + 1],
        }
        df = pd.concat([df, pd.DataFrame(new_row)], ignore_index=True)

        # Duplicate the sentiment of the person that does not speak at this duration
        if speakers[i] == "Agent":
            # Agent speak this turn, duplicate the sentiment of the visitor
            visitor_previous["Timestamp"] = [i + 1]
            df = pd.concat([df, pd.DataFrame(visitor_previous)], ignore_index=True)
            agent_previous = new_row
        else:
            # Visitor speak, duplicate the sentiment of the agent
            agent_previous["Timestamp"] = [i + 1]
            df = pd.concat([df, pd.DataFrame(agent_previous)], ignore_index=True)
            visitor_previous = new_row

    return gr.LinePlot(
        df,
        label="Intensity Plot",
        x="Timestamp",
        y="Sentiment Intensity",
        color="Speaker",
        color_legend_position="right",
        title="Conversation Sentiment Intensity",
        tooltip=["Speaker", "Sentiment Intensity"],
        y_lim=[-1, 1],
        height=400,
        width=400,
    )


SENTIMENT_SCORE_TABLE = """
<div align="center" style='font-size: small;'>

|  Sentiment Score  |  Sentiment Label  |
| :---------------: | :---------------: |
|         3         | Acceptance, Openness |
|         2         | Interest, Curiosity |
|         1         | Informative  |
|         0         | Greeting  |
|        -1         | Uninterested |
|        -2         | Anxious, Confused, Annoyed, Remorse |
|        -3         | Disapproval, Accusatory, Denial, Obscene |

</div>
"""


def display_sentiment_score_table():
    gr.Markdown(
        "After submit the conversation and run the model to predict the sentiment labels from the above, we can generate the plot of the sentimen labels found in this conversation to see how the sentiment flow changes over time.\nThe sentiment labels are grouped into corresponding scores below:"
    )
    gr.Markdown(SENTIMENT_SCORE_TABLE)


EXAMPLE_CONVERSATIONS = {
    "Example 1": "Visitor: Hey Jess, available for outcall tonight?\n\nAgent: hey hey what you looking for luv?\n\nVisitor: Some company in El Segundo\n\nAgent: out with friends or just the two of us?\n\nVisitor: Just the two\n\nAgent: looking for some connection then...\n\nVisitor: Yes\n\nAgent: I get it. Looking for companionship, it's a very legit need. I'll level with you. We're a group of guys trying to show other men the harms of the sex trade industry, both to buyers and sellers. You up for a little conversation? No pressure... no judgement.\n\nVisitor: No, but interested in the harms in companionship?\n\nAgent: The truth is nearly all of the women on these sites are not in this career by choice. We work with survivors. Their money goes to pimps.. their lives are controlled.\n\nVisitor: Really?\n\nVisitor: Even when they say \"independent\"?\n\nAgent: Yes. It's all part of the human trafficking system... pimps find & recruit nearly all women who are doing this, they are very, very good at manipulation and control, and working in the background. Sadly, this modern day slavery will be with us as long as there are willing buyers. That's why we are trying to work this angle. The more men that know the truth, the fewer will want to exploit someone in this way.\n\nAgent: I need to move on, but just consider what I've told you tonight. Polarisproject.org is a great resource to learn more... it's nothing like what Hollywood shows ya. We work with the women & some guys who have escaped out of this lifestyle. It is modern-day slavery, straight up. Thanks for the chat. Good night!\n\nVisitor: Thanks, fyi it worked. Hung it up, I'm to old to me messin around in this crazy stuff take care\n\nAgent: I am so glad to hear this friend. You calling this number tonight was not just a coincidence. Take care also!!\n\n",
    "Example 2": "Visitor: What's up sexy\n\nAgent: Thank you! I'm hangin\n\nVisitor: Are you available at the moment\n\nAgent: what's your name\n\nVisitor: David ... Yours\n\nAgent: Dakota is not available right now, but my name is Jeff and I'm working with a bunch of guys trying to curb the demand for victims of human trafficking. Did you ever consider that Dakota could be a victim of trafficking?\n\nVisitor: Fuck no I just wanted to get some pussy .... Bye\n\nAgent: Just so you know your number is in our system now. Understand that next time you click on one of these ads you could be walking into an undercover cop and many of these girls are underage. That is a crime that will follow you the rest of your life!\n\nVisitor: . You think the predators your looking for going to fall into your lap? Lol your not a cop stop trying to be one ... You want to help?\n\nVisitor: Just so you know I don't give no fucks about you or no fucking cops... Threatening me will not get you shit from me ,no info no nothing you stupid cunt..\n\nAgent: Hey man, I did not want to threaten you. I want you to be aware of the risks. Have you heard that most women in prostitution do it against their will, and many are minors?\n\nVisitor: I never fuck with minors that's for sure cause you can tell if they are not of age\n\nAgent: Yeah, many people think it's harmless and these women are in complete control. But, the truth is, most of these women are enslaved, forced to sell themselves by pimps who take 100% of their money. I hope you can see these women in a different light.\n\nVisitor: I know most are blacks living off a woman cause they can't get no job ... I won't see a female if she has a guy around\n\nAgent: I'm curious, is there something missing inside of you, or are you angry about something not right in your life that causes you to seek out connection with these girls?\n\nVisitor: No actually I am on gps monitor and can't go out and meet no one so I got needs and feel sometimes hooking up with one of these girls here and there is what works for my situation....but I'm not disrespectful to them or be doing ugly things ... My ex wife cheated on me after 8yrs of marriage I don't really trust women like enough for a relationship\n\nVisitor: I'm being honest with you cause even tho I don't know u I believe in doing the right thing and your movement with alot of work is going to save lives so\n\nVisitor: that's good\n\nAgent: Hey David, I'm very sorry for your situation and the pain you must be in. Were you raised with any faith perspective?\n\nVisitor: Yes I actually was baptized for the first time in my life in 2018 and have made some drastic changes to my life.... I'm not perfect but I am working hard\n\nAgent: Okay, David. That's great news. We're all a work in progress. God is faithful to forgive us of our sins if we just ask him. Read 1 John 1:9. Can I direct you to some resources to help you overcome this urge to buy sex?\n\nVisitor: Honestly I'm not ready for that is all I can say at the moment.... Sorry but I don't like to lie\n\nAgent: Hey, no worries. When you're ready, check out www.celebraterecovery.com as they have a lot of resources that might help. I will be praying for you, David. I've had a lot of pain in my life and only in my relationship with Jesus have I found true peace that surpasses my understanding. I pray that for you too.\n\nVisitor: What kinda pain ? Do you mind sharing?\n\nVisitor: And what are y'all doing to put those pieces of shit in prison who are hurting these women\n\nAgent: I've had several people close to me pass away tragically in the past few years. Thanks for asking. I'm just trusting God each day trusting he's in control.\n\nAgent: Hey, honestly, the best way we can solve this problem of evil pieces of shit harming these girls is to dry up the demand for these girls by encouraging men like us to stop calling these ads. If the demand dries up, evil predatory scums' business with die.\n\nAgent: Can I get your commitment to do your part and stop engaging these girls on their escort ads?\n\nVisitor: Yes I will but I can promise you that it will not stop matter of fact it will get worse ... They will beat them more and make them do it more only way is\n\nVisitor: to put the pimps in prison for alot of years....\n\nAgent: Hey, I'll be praying for you. Feel free to call me any time if you need to talk. My number is (615) 628-7545. God bless you, brother.\n\nVisitor: Thank u\n\n",
    "Example 3": "Visitor: Heyyy\n\nVisitor: How are you this evening\n\nAgent: better now ;) call me\n\nVisitor: Iam at work for now ,be off around 10pm\n\nVisitor: Need some company\n\nVisitor: Are you independent honey\n\nAgent: well since you arent available at the moment ill just come out and say-these sites are bad news. did you know that most of the girls on here are here against their will? Most of them got dragged into this lifestyle by an abuser, oftentimes before they were of legal consenting age. isnt that sad?\n\nAgent: we are with some guys who are trying to spread awareness of the realities of this \"industry\".\n\nAgent: https://exoduscry.com/choice/\n\nVisitor: Thanks\n\nAgent: i encourage you to watch this video. it is jarring to think about how bad someone else's options must be to choose to be on these sites\n\nVisitor: Ooohhh\n\nAgent: selling their body to make ends meet or appease a pimp\n\nVisitor: That's really awful\n\nAgent: it is. you seem like the kind of guy who wouldnt wont to proliferate that kind of harmful lifestyle. am i right in thinking that?\n\nVisitor: Well iam just looking for attention\n\nVisitor: My marriage is not going well lol\n\nAgent: i know that it is hard to find ourselves lonely and without much alternative to meet that perceived need but its humbling to think that our needs can force someone else into such a dark place\n\nAgent: hey, thanks for sharing that my man. i know it can be hard\n\nAgent: marraige is the most humbling of relationships, isnt it?\n\nVisitor: She leaves with her friends n no time for me\n\nAgent: ive been there my guy. i know that it is alot easier to numb that loneliness for sure\n\nVisitor: I want to be faithful\n\nAgent: does your wife know how you feel when she chooses her friends instead of you?\n\nVisitor: I been drinking lately\n\nVisitor: Yes , she takes pills\n\nAgent: if so, i hope you are praying for her to realize the hurt she is causing and to seek change\n\nVisitor: She had surgery 4 yes ago n it's been hard for her n her addiction on pills\n\nVisitor: Yes for now iam looking for a female friend to talk n see what can we do for each other\n\nAgent: that is hard my man. physical pain is a huge obstacle in life for sure so i hear you\n\nVisitor: Well chat later .thanks\n\nAgent: have you considered pursuing other men who can encourage you instead of looking for the easy way out?\n\nAgent: what is your name my friend? i will be praying for you by name if you wouldnt mind sharing it\n\nAgent: well, i gotta run. watch that video i sent and i will definitely be praying for you. I hope you pray for yourself and for your wife - God can definitely intervene and cause complete change in the situation if He wills it. He is good and He hears you. You are loved by Him, brother. Good night\n\n",
}