app.py

import gradio as gr
from transformers import pipeline
from matplotlib.ticker import MaxNLocator
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

MODEL_NAMES = ["bert-base-uncased",
               "distilbert-base-uncased", "xlm-roberta-base"]

DECIMAL_PLACES = 1
EPS = 1e-5  # to avoid /0 errors

# Example date conts
DATE_SPLIT_KEY = "DATE"
START_YEAR = 1800
STOP_YEAR = 1999
NUM_PTS = 20
DATES = np.linspace(START_YEAR, STOP_YEAR, NUM_PTS).astype(int).tolist()
DATES = [f'{d}' for d in DATES]

# Example place conts
# https://www3.weforum.org/docs/WEF_GGGR_2021.pdf
# Bottom 10 and top 10 Global Gender Gap ranked countries.
PLACE_SPLIT_KEY = "PLACE"
PLACES = [
    "Afghanistan",
    "Yemen",
    "Iraq",
    "Pakistan",
    "Syria",
    "Democratic Republic of Congo",
    "Iran",
    "Mali",
    "Chad",
    "Saudi Arabia",
    "Switzerland",
    "Ireland",
    "Lithuania",
    "Rwanda",
    "Namibia",
    "Sweden",
    "New Zealand",
    "Norway",
    "Finland",
    "Iceland"]


# Example Reddit interest consts
# in order of increasing self-identified female participation.
# See http://bburky.com/subredditgenderratios/ , Minimum subreddit size: 400000
SUBREDDITS = [
    "GlobalOffensive",
    "pcmasterrace",
    "nfl",
    "sports",
    "The_Donald",
    "leagueoflegends",
    "Overwatch",
    "gonewild",
    "Futurology",
    "space",
    "technology",
    "gaming",
    "Jokes",
    "dataisbeautiful",
    "woahdude",
    "askscience",
    "wow",
    "anime",
    "BlackPeopleTwitter",
    "politics",
    "pokemon",
    "worldnews",
    "reddit.com",
    "interestingasfuck",
    "videos",
    "nottheonion",
    "television",
    "science",
    "atheism",
    "movies",
    "gifs",
    "Music",
    "trees",
    "EarthPorn",
    "GetMotivated",
    "pokemongo",
    "news",
    # removing below subreddit as most of the tokens are taken up by it:
    # ['ff', '##ff', '##ff', '##fu', '##u', '##u', '##u', '##u', '##u', '##u', '##u', '##u', '##u', '##u', '##u', ...]
    # "fffffffuuuuuuuuuuuu",
    "Fitness",
    "Showerthoughts",
    "OldSchoolCool",
    "explainlikeimfive",
    "todayilearned",
    "gameofthrones",
    "AdviceAnimals",
    "DIY",
    "WTF",
    "IAmA",
    "cringepics",
    "tifu",
    "mildlyinteresting",
    "funny",
    "pics",
    "LifeProTips",
    "creepy",
    "personalfinance",
    "food",
    "AskReddit",
    "books",
    "aww",
    "sex",
    "relationships",
]

GENDERED_LIST = [
    ['he', 'she'],
    ['him', 'her'],
    ['his', 'hers'],
    ["himself", "herself"],
    ['male', 'female'],
    ['man', 'woman'],
    ['men', 'women'],
    ["husband", "wife"],
    ['father', 'mother'],
    ['boyfriend', 'girlfriend'],
    ['brother', 'sister'],
    ["actor", "actress"],
]


# Fire up the models
# TODO: Make it so models can be added in the future
models_paths = dict()
models = dict()


# %%
for bert_like in MODEL_NAMES:
    models_paths[bert_like] = bert_like
    models[bert_like] = pipeline(
        "fill-mask", model=models_paths[bert_like])


def get_gendered_token_ids():
    male_gendered_tokens = [list[0] for list in GENDERED_LIST]
    female_gendered_tokens = [list[1] for list in GENDERED_LIST]

    return male_gendered_tokens, female_gendered_tokens


def prepare_text_for_masking(input_text, mask_token, gendered_tokens, split_key):
    text_w_masks_list = [
        mask_token if word in gendered_tokens else word for word in input_text.split()]
    num_masks = len([m for m in text_w_masks_list if m == mask_token])

    text_portions = ' '.join(text_w_masks_list).split(split_key)
    return text_portions, num_masks


def get_avg_prob_from_pipeline_outputs(mask_filled_text, gendered_token, num_preds):
    pronoun_preds = [sum([
        pronoun["score"] if pronoun["token_str"].lower(
        ) in gendered_token else 0.0
        for pronoun in top_preds])
        for top_preds in mask_filled_text
    ]
    return round(sum(pronoun_preds) / (EPS + num_preds) * 100, DECIMAL_PLACES)


def get_figure(df, gender, n_fit=1):
    df = df.set_index('x-axis')
    cols = df.columns
    xs = list(range(len(df)))
    ys = df[cols[0]]
    fig, ax = plt.subplots()

    # find stackoverflow reference
    p, C_p = np.polyfit(xs, ys, n_fit, cov=1)
    t = np.linspace(min(xs)-1, max(xs)+1,  10*len(xs))
    TT = np.vstack([t**(n_fit-i) for i in range(n_fit+1)]).T

    # matrix multiplication calculates the polynomial values
    yi = np.dot(TT, p)
    C_yi = np.dot(TT, np.dot(C_p, TT.T))  # C_y = TT*C_z*TT.T
    sig_yi = np.sqrt(np.diag(C_yi))  # Standard deviations are sqrt of diagonal

    ax.fill_between(t, yi+sig_yi, yi-sig_yi, alpha=.25)
    ax.plot(t, yi, '-')
    ax.plot(df, 'ro')
    ax.legend(list(df.columns))

    ax.axis('tight')

    # fig.canvas.draw()

    ax.set_xlabel("Value injected into input text")
    ax.set_title(
        f"Probability of predicting {gender} pronouns.")
    ax.set_ylabel(f"Softmax prob for pronouns")
    ax.xaxis.set_major_locator(MaxNLocator(6))
    ax.tick_params(axis='x', labelrotation=15)
    return fig


# %%
def predict_gender_pronouns(
    model_type,
    indie_vars,
    split_key,
    normalizing,
    n_fit,
    input_text,
):
    """Run inference on input_text for each model type, returning df and plots of precentage
    of gender pronouns predicted as female and male in each target text.
    """
    model = models[model_type]
    mask_token = model.tokenizer.mask_token

    indie_vars_list = indie_vars.split(',')

    male_gendered_tokens, female_gendered_tokens = get_gendered_token_ids()

    text_segments, num_preds = prepare_text_for_masking(
        input_text, mask_token, male_gendered_tokens + female_gendered_tokens, split_key)

    male_pronoun_preds = []
    female_pronoun_preds = []
    for indie_var in indie_vars_list:

        target_text = f"{indie_var}".join(text_segments)
        mask_filled_text = model(target_text)
        # Quick hack as realized return type based on how many MASKs in text.
        if type(mask_filled_text[0]) is not list:
            mask_filled_text = [mask_filled_text]

        female_pronoun_preds.append(get_avg_prob_from_pipeline_outputs(
            mask_filled_text,
            female_gendered_tokens,
            num_preds
        ))
        male_pronoun_preds.append(get_avg_prob_from_pipeline_outputs(
            mask_filled_text,
            male_gendered_tokens,
            num_preds
        ))

    if normalizing:
        total_gendered_probs = np.add(
            female_pronoun_preds, male_pronoun_preds)
        female_pronoun_preds = np.around(
            np.divide(female_pronoun_preds, total_gendered_probs+EPS)*100,
            decimals=DECIMAL_PLACES
        )
        male_pronoun_preds = np.around(
            np.divide(male_pronoun_preds, total_gendered_probs+EPS)*100,
            decimals=DECIMAL_PLACES
        )

    results_df = pd.DataFrame({'x-axis': indie_vars_list})
    results_df['female_pronouns'] = female_pronoun_preds
    results_df['male_pronouns'] = male_pronoun_preds
    female_fig = get_figure(results_df.drop(
        'male_pronouns', axis=1), 'female', n_fit)
    male_fig = get_figure(results_df.drop(
        'female_pronouns', axis=1), 'male', n_fit)

    return (
        target_text,
        female_fig,
        male_fig,
        results_df,
    )

# %%
title = "Causing Gender Pronouns"
description = """
## Intro 

"""

place_example = [
    MODEL_NAMES[0],
    ','.join(PLACES),
    'PLACE',
    "False",
    1,
    'Born in PLACE, she was a teacher.'
]

date_example = [
    MODEL_NAMES[0],
    ','.join(DATES),
    'DATE',
    "False",
    2,
    'Born in DATE, she was a doctor.'
]


subreddit_example = [
    MODEL_NAMES[2],
    ','.join(SUBREDDITS),
    'SUBREDDIT',
    "False",
    1,
    'I saw on r/SUBREDDIT that she is a hacker.'
]


def date_fn():
    return date_example
def place_fn():
    return place_example
def reddit_fn():
    return subreddit_example


# %%
demo = gr.Blocks()
with demo:
    gr.Markdown("## Hunt for spurious correlations in our LLMs.")
    gr.Markdown("Please see a better explanation in another [Space](https://huggingface.co/spaces/emilylearning/causing_gender_pronouns_two).")

    
    with gr.Row():
        x_axis = gr.Textbox(
            lines=5,
            label="Pick a spectrum of values for text injection and x-axis",
        )
    with gr.Row():
        model_name = gr.Radio(
            MODEL_NAMES,
            type="value",
            label="Pick a BERT-like model.",
        )
        place_holder = gr.Textbox(
            label="Special token used in input text that will be replaced with the above spectrum of values.",
            type="index",
        )
        to_normalize = gr.Dropdown(
            ["False", "True"],
            label="Normalize?",
            type="index",
        )
        n_fit = gr.Dropdown(
            list(range(1, 5)),
            label="Degree of polynomial fit for dose response trend",
            type="value",
        )
    with gr.Row():
        input_text = gr.Textbox(
            lines=5,
            label="Input Text: Sentence about a single person using some gendered pronouns to refer to them.",
        )
    with gr.Row():
        sample_text = gr.Textbox(
            type="auto", label="Output text: Sample of text fed to model")
    with gr.Row():
        female_fig = gr.Plot(
            type="auto", label="Plot of softmax probability pronouns predicted female.")
    with gr.Row():
        male_fig = gr.Plot(
            type="auto", label="Plot of softmax probability pronouns predicted male.")
    with gr.Row():
        df = gr.Dataframe(
            show_label=True,
            overflow_row_behaviour="show_ends",
            label="Table of softmax probability for pronouns predictions",
        )
    gr.Markdown("x-axis sorted by older to more recent dates:")
    place_gen = gr.Button('Populate fields with a location example')

    gr.Markdown("x-axis sorted by bottom 10 and top 10 Global Gender Gap ranked countries:")
    date_gen = gr.Button('Populate fields with a date example')

    gr.Markdown("x-axis sorted in order of increasing self-identified female participation (see [bburky demo](http://bburky.com/subredditgenderratios/)): ")
    subreddit_gen = gr.Button('Populate fields with a subreddit example')

    #https://github.com/gradio-app/gradio/issues/690#issuecomment-1118772919
    with gr.Row():
        date_gen.click(date_fn, inputs=[], outputs=[model_name,
                       x_axis, place_holder, to_normalize, n_fit, input_text])
        place_gen.click(place_fn, inputs=[], outputs=[
                        model_name, x_axis, place_holder, to_normalize, n_fit, input_text])
        subreddit_gen.click(reddit_fn, inputs=[], outputs=[
                            model_name, x_axis, place_holder, to_normalize, n_fit, input_text])
    with gr.Row():
        btn = gr.Button("Hit submit")
        btn.click(
            predict_gender_pronouns,
            inputs=[model_name, x_axis, place_holder,
                    to_normalize, n_fit, input_text],
            outputs=[sample_text, female_fig, male_fig, df])

demo.launch(debug=True)