<jupyter_start><jupyter_text>Analyze Survey Monkey Response files.Based upon project: https://github.com/RuoyunLin/code_snippets  Prerequisites- python 3.7- faker- matlab- numpy- pandas- seaborn<jupyter_code>!pip3 install faker
!pip3 install matlab
!pip3 install numpy
!pip3 install pandas
!pip3 install seaborn

import site
site.main()
# now with refreshed module path...
#import libraries
import pandas as pd
import numpy as np
import random
random.seed(1)

#set plot style
import seaborn as sns
import matplotlib.pyplot as plt
plt.style.use('bmh')

from IPython.display import Image, display
display(Image(filename='../images/analyze_surveymonkey_results_questionaire.png'))<jupyter_output><empty_output><jupyter_text>Step 1: Load the dataset and Generate codebook<jupyter_code># Import dataframe with multi index column names for generating the codebook
df1 = pd.read_csv('../data/analyze_surveymonkey_results.csv', header=[0, 1])

# Prepare the codebook
codebook = df1.columns.to_frame(index=0, name=['question', 'options'])
df1.head()<jupyter_output><empty_output><jupyter_text>Step 2: Merge the dataset that you download directly from Survey Monkey<jupyter_code># Prepare the merged version of dataframe, please replace the file names below
df_1 = pd.read_csv("../data/analyze_surveymonkey_results.csv", header=None, prefix="Q").iloc[2:]
df_2 = pd.read_csv("../data/analyze_surveymonkey_results.csv", header=None, prefix="Q").iloc[2:]


# Pass in group name
df_1["group"] = "group_1"
df_2["group"] = "group_2"

# Merge different datasets into one dataframe
df = df_1.append(df_2, ignore_index=True)
df.head()<jupyter_output><empty_output><jupyter_text>Step 3: Faker package is used here to generate a larger fake dataset (instead of using the manually generated fake data from Survey Monkey)<jupyter_code>from faker import Faker

fake = Faker()

# import a Provider
from faker.providers import BaseProvider

from datetime import datetime, timedelta

# create my new provider class
class MyProvider(BaseProvider):
    # define a method that return a random expense type
    def devDesktop(self):
        foo = ["Desktop computer", np.nan]
        return random.choice(foo)

    def devLaptop(self):
        foo = ["Laptop computer", np.nan]
        return random.choice(foo)

    def devTablet(self):
        foo = ["Tablet", np.nan]
        return random.choice(foo)

    def devPhone(self):
        foo = ["Smart phone", np.nan]
        return random.choice(foo)

    def satLevel(self):
        foo = [
            "Very satisfied",
            "Somewhat satisfied",
            "Neither satisfied nor dissatisfied",
            "Somewhat dissatisfied",
            "Very dissatisfied",
        ]
        return random.choice(foo)

    def agreeLevel(self):
        foo = ["Strongly agree", "Agree", "Undecided", "Disagree", "Strongly disagree"]
        return random.choice(foo)

    def group(self):
        foo = ["Group 1", "Group 2"]
        return random.choice(foo)


# then add my new provider to faker instance
fake.add_provider(MyProvider)
def createFakeDataset(num_examples):
    """Function that creates a fake dataset using the python module Faker

    Args:
        num_examples (int): number of fake examples to generate

    Returns:
        pandas.core.frame.DataFrame: the final dataframe

    """

    l_response_ID = []
    l_collector_ID = []
    l_start_dates = []
    l_end_dates = []
    l_devDesktop = []
    l_devLaptop = []
    l_devTablet = []
    l_devPhone = []
    l_devOther = []
    l_satLevel = []
    l_matrix1 = []
    l_matrix2 = []
    l_matrix3 = []
    l_matrix4 = []
    l_group = []
    l_missing = []

    for _ in range(num_examples):
        l_response_ID.append(fake.random_int(min=10000000000, max=99999999999))
        l_collector_ID.append(fake.random_int(min=100000000, max=999999999))
        # create a fake date within 30 days
        fake_date = fake.date_time_between(
            start_date="-30d", end_date="now", tzinfo=None
        )
        l_start_dates.append(fake_date)
        l_end_dates.append(fake_date + timedelta(minutes=random.randrange(90)))
        # create fake data for device question
        l_devDesktop.append(fake.devDesktop())
        l_devLaptop.append(fake.devLaptop())
        l_devTablet.append(fake.devTablet())
        l_devPhone.append(fake.devPhone())
        # create fake data for satisfaction question
        l_satLevel.append(fake.satLevel())
        # create fake data for the matrix question
        l_matrix1.append(fake.agreeLevel())
        l_matrix2.append(fake.agreeLevel())
        l_matrix3.append(fake.agreeLevel())
        l_matrix4.append(fake.agreeLevel())
        # create fake data for group
        l_group.append(fake.group())
        # create a list of missing values
        l_missing.append(np.nan)

    # create a dataframe
    df = pd.DataFrame(
        {
            "Q0": l_response_ID,
            "Q1": l_collector_ID,
            "Q2": l_start_dates,
            "Q3": l_end_dates,
            "Q4": l_missing,
            "Q5": l_missing,
            "Q6": l_missing,
            "Q7": l_missing,
            "Q8": l_missing,
            "Q9": l_devDesktop,
            "Q10": l_devLaptop,
            "Q11": l_devTablet,
            "Q12": l_devPhone,
            "Q13": l_missing,
            "Q14": l_satLevel,
            "Q15": l_matrix1,
            "Q16": l_matrix2,
            "Q17": l_matrix3,
            "Q18": l_matrix4,
            "group": l_group,
        }
    )

    return df
df = createFakeDataset(200)
df.head()
df.group.value_counts()<jupyter_output><empty_output><jupyter_text>Step 4: Visualize multi-answer question by group (Checkboxes)<jupyter_code># Prepare the summary table (please clean the data beforehand)
def prepare_table(data, column_range, group_column_name='group'):
    res = []
    col_range_index = list(column_range)
    group_index = data.columns.to_list().index(group_column_name)
    col_range_index.append(group_index)
    series = data.iloc[:, col_range_index].groupby([group_column_name]).count().unstack()
    for group in series.index.levels[1]:
        for var in series.index.levels[0]:
            res.append(
                [
                    (series[var][group] / data.loc[:,group_column_name].value_counts()[group] * 100).round(2),
                    codebook.iloc[int(var[1:]), 1],
                    group,
                ]
            )
    return pd.DataFrame(columns=["percentage(%)", "options", group_column_name], data=res)


# Generate the checkbox chart based on the summary table
def gen_chart_checkbox(data, column_range, group_column_name='group'):
    listOfGroup = list(data.loc[:,group_column_name].unique())
    listOfGroup.sort()
    table_sum = prepare_table(data, column_range, group_column_name)

    print("Number of answers in each group: ")
    print(data.loc[:,group_column_name].value_counts())

    fig, ax = plt.subplots(figsize=(10, 8))
    ax = sns.barplot(
        x="percentage(%)",
        y="options",
        hue=group_column_name,
        hue_order=listOfGroup,
        data=table_sum,
    )
    ax.set(xlim=(0, 100))
    ax.set_ylabel("")
    plt.title(codebook.iloc[column_range[0], 0], fontsize=15)
    plt.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.0, title="group")
    return plt.show()
gen_chart_checkbox(df, range(9,13))<jupyter_output>Number of answers in each group: 
Group 1    102
Group 2     98
Name: group, dtype: int64<jupyter_text>Step 5: Visualize single-answer question by group (Radio Button)<jupyter_code># Generate a chart to visualize single-answer questions
def gen_chart_radiobutton(data, question_name, index, group_column_name='group'):
    print("Number of answers in each group: ")
    print(data[[question_name, group_column_name]].groupby(group_column_name).count())
    i_counts = (
        data.groupby([group_column_name])[question_name]
        .value_counts(normalize=True)
        .rename("percentage(%)")
        .mul(100)
        .reset_index()
        .round(2)
    )

    listOfGroup = list(data.loc[:,group_column_name].unique())
    listOfGroup.sort()

    fig, ax = plt.subplots(figsize=(10, 8))

    fig = sns.barplot(
        x="percentage(%)",
        y=question_name,
        order=index,
        hue=group_column_name,
        hue_order=listOfGroup,
        data=i_counts,
    )

    plt.title(codebook.iloc[int(question_name[1:]), 0])
    plt.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.0, title="group")
    ax.set(xlim=(0, 100))
    ax.set_ylabel("")
    return plt.show()
index = [
    "Very satisfied",
    "Somewhat satisfied",
    "Neither satisfied nor dissatisfied",
    "Somewhat dissatisfied",
    "Very dissatisfied",
]
gen_chart_radiobutton(df, "Q14", index)<jupyter_output>Number of answers in each group: 
         Q14
group       
Group 1  102
Group 2   98<jupyter_text>Step 6: Visualize matrix question by group<jupyter_code># Generate summary data for each group
def gen_table(data, group_name, col_range, group_column_name='group'):
    data_sub = data[data[group_column_name] == group_name].iloc[:, col_range].dropna(how="all")
    for var in data_sub.columns:
        data_sub[var] = data_sub[var].map(
            {
                "Strongly agree": 5,
                "Agree": 4,
                "Undecided": 3,
                "Disagree": 2,
                "Strongly disagree": 1,
            }
        )
    table = data_sub.describe().loc[["mean", "std", "count"]].T

    index = []

    for var in table.index:
        i = int(var[1:])
        index.append(codebook.iloc[i, 1])

    table["item"] = index

    # table["item_n"] = range(len(index),0,-1)
    table["item_n"] = range(0, len(index))
    return table
gen_table(df,'Group 1', range(15,19))
# Generate a chart to compare the importance of missing features across two groups
def compare_importance(data, groups, col_range, group_column_name='group'):

    group_name_to_describe_data = {}

    #Create a dictionary to save the summary data for each group
    for i, group_name in enumerate(groups):
        group_name_to_describe_data[group_name] = "data_describe_%s" % i

    for group_name in groups:
        table = gen_table(data, group_name, col_range, group_column_name)
        group_name_to_describe_data[group_name] = table

    #Get the item list and index
    items = table.item.tolist()
    item_n = table.item_n.tolist()
    
    # Visualize the mean value with the 95% confidence interval
    # Change the figsize if you have more yticks
    plt.figure(num=None, figsize=(12, 6), dpi=90, facecolor="w", edgecolor="k")
    ax = plt.axes()

    for i, group_name in enumerate(groups):
        plt.errorbar(
            group_name_to_describe_data[group_name]["mean"].astype(float),
            group_name_to_describe_data[group_name]["item_n"] - 0.1 * i,
            xerr=1.96
            * (
                group_name_to_describe_data[group_name]["std"].astype(float)
                / (group_name_to_describe_data[group_name]["count"] ** 0.5)
            ),
            fmt="o",
            elinewidth=0.5,
            capsize=4,
            marker="o",
            ms=4,
            label=group_name,
        )

    ax.set_xlim(1, 5)
    
    ax.set_yticks(item_n)
    ax.set_yticklabels(labels=items, fontdict={'fontsize':16})

    plt.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.0, title="group")
    plt.title("Compare the mean values across groups (scale 1-5)", fontsize=16)

    return plt.show()
# Compare the Group 1 and 2
compare_importance(df, ['Group 1', 'Group 2'], range(15,19))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11886549204984959, 'fraction_numerical': 0.021143102707348517, 'mean_word_length': 2.9986254295532646, 'pattern_counts': {'":': 25, '<': 27, '<?xml version=': 0, '>': 27, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 4, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '422184', 'license': 'MIT', 'n_tokens_mistral': 4005, 'n_tokens_neox': 3766, 'n_words': 848, 'path': 'notebooks/analyze_surveymonkey_results.ipynb', 'repo_name': 'Stimson-Center/stimson-tutorials', 'size': '11635'}