<jupyter_start><jupyter_text>Something to keep in mind! **In Specialist csvs, one "quiz_task_uuid" can have multiple contributors. To uniquely identify each input, use "quiz_taskrun_uuid** **To construct reliability matrix, use contributor_uuid, quiz_task_uuid, answer_label**   Helper functions<jupyter_code># mapping for Triager tasks
triager_dic = {"Quoted Sources":1,
               "Arguments":2,
               "Assertions":3, 
               "Needs Fact-Check":4, 
               "Language":5, 
               "Reasoning":6, 
               "Probability":7, 
               "Evidence":8}<jupyter_output><empty_output><jupyter_text>The dictionary needs to be updated based on newer versions of schema files.<jupyter_code># dic for Specialists tasks
specialist_dic = {"Argument relevance": {"mc": {"categorical":[1], "ordinal":[2, 3]}, "tx":[4]},
                  "Evidence Specialist": {"mc":{"categorical":[6, 9], "ordinal":[4, 5, 8, 10, 11, 12, 13, 14, 15]}, "cl":[1, 2], "tx":[3, 7, 16]},
                  "Holistic Evaluation of Article": {"mc":{"categorical":[1, 5, 14], "ordinal":[7, 8, 11, 12, 13, 17, 18]}, "cl":[3, 6, 9, 10, 15], "tx":[2, 4, 16, 19]}, 
                  "Language Specialist V4": {"mc":{"categorical":[], "ordinal":[2, 3, 4, 5, 8, 9, 10, 11, 12, 13, 14, 15]}, "cl":[1, 6], "tx":[7, 16]},
                  "Probability Specialist": {"mc": {"categorical":[6], "ordinal":[1, 2, 4, 5, 7, 8, 9, 10, 11, 13, 14, 15]}, "cl":[12], "tx":[16]},
                  "Reasoning Specialist": {"mc": {"categorical":[4, 5], "ordinal":[7, 9, 10, 11]}, "cl":[1, 2, 3, 6], "tx":[8, 12]},
                  "Quote Source Relevance": {"mc": {"categorical":[3, 8], "ordinal":[2, 9, 10, 11]}, "cl":[1, 4, 5, 6, 7], "tx":[12]}}
# decompose question_label to get the int 
def decompose_question(question_label):
    # question_label - e.g. "T1.Q2"
    # return an int - e.g. 2 
    m = re.search(r'Q\d+', question_label)
    return int(m.group(0)[1:])
def get_question_type(topic_name, question_label):
    # topic_name - e.g. "Argument relevance"
    # question_label - e.g. 'T1.Q12'
    # return "mc"/"cl" or None if not found
    question_int = decompose_question(question_label)
    # key - "mc"/"cl"/"tx"
    for key in specialist_dic[topic_name].keys():
        if key == "mc":
            mc_mapping = specialist_dic[topic_name]["mc"]
            # mc_key = "categorical"/"ordinal"
            for mc_key in mc_mapping.keys():
                cur_list = mc_mapping[mc_key]
                if question_int in cur_list: 
                    return key
        elif key == "cl":
            cur_list = specialist_dic[topic_name]["cl"]
            if question_int in cur_list:
                return key
        elif key == "tx":
            cur_list = specialist_dic[topic_name]["tx"]
            if question_int in cur_list:
                return key
# assuming the question_type is "mc"
# return "ordinal"/"categorical"
def get_measurement(topic_name, question_label, mapping):
    # topic_name - e.g. "Argument relevance"
    # question_label - e.g. "T1.Q1"
    # mapping - e.g. specialist_dic, this might change
    question_int = decompose_question(question_label)
    dic = mapping[topic_name]["mc"]
    for key in dic.keys():
        cur_list = dic[key]
        if question_int in cur_list:
            if key == "categorical":
                return "nominal"
            if key == "ordinal":
                return "ordinal"
# check if a certain column contains any duplicate values 
# return true if all values in the column are unique
def check_duplicate(df, col):
    # df - any dataframe 
    # col - column label 
    return df[col].is_unique
# count how many Nan rows are there in a df 
# Nan rows are rows that have all missing values
def count_nan_rows(df):
    return df.shape[0] - df.dropna(axis=0, how='all').shape[0]
# count how many nan cols are there in a df
# nan cols are cols where all values are missing 
def count_nan_cols(df):
    return df.shape[1] - df.dropna(axis=1, how="all").shape[1]
# only select that value, useful for constructing pivot_table
def select(x):
    return list(x)[0]
# return the unique values sorted in a df's column
def unique(df, x):
    # df - dataframe 
    # x - the label of the column 
    a = sorted(df[x].unique())
    return a
# convert all strings in the array to int 
# e.g. "T1.Q1.A5" -> 5 
def to_int(array):
    # array - e.g. 2-d array where some entries are "T1.Q1.A3"
    for i in range(array.shape[0]):
        for j in range(array.shape[1]):
            cur = array[i][j]
            # if the current value is a string
            if not pd.isnull(cur):
                try:
                    array[i][j] = int(array[i][j][-1])
                except Exception as e:
                    print(i, j)
    df = pd.DataFrame(array)
    return array
# construct a reliability matrix (2d array) from Specialist csv 
def to_reliability(input_directory, fileName, question_type, question_label):
    # input_directory - e.g. "Specialists data"
    # fileName - e.g. "Covid_SourceRelevancev1-2020-10-04T1838-DataHunt.csv"
    # question_type - "mc"/"cl"
    # question_label - e.g. "Q2"
    name = os.path.join(input_directory, fileName)
    # read in csv 
    df = pd.read_csv(name)
    # select only that question_label 
    df = df[df["question_label"] == question_label].copy()
    # slice 
    cols = ["quiz_task_uuid", "contributor_uuid", "question_label", "answer_label"]
    df = df.reindex(columns=cols)
    if question_type == "mc":
        return to_reliability_mc(df)
    elif question_type == "cl":
        return to_reliability_cl(df)
# helper function, construct reliability matrix for mc question type
def to_reliability_mc(df):
    # construct a pivot_table 
    df = pd.pivot_table(df, values='answer_label', index='contributor_uuid', columns='quiz_task_uuid', aggfunc=select)
    # to numpy array
    array = df.to_numpy()
    # convert string to int 
    array = to_int(array)
    # return the array 
    return array.astype(dtype=float)
# helper function, construct reliability matrix for cl question type
def to_reliability_cl(df):
    # one-hot-encode answer_label
    df = pd.get_dummies(df, columns = ["answer_label"])
    # combine rows 
    df = df.groupby(["quiz_task_uuid", "contributor_uuid"]).agg(sum)
    # reset_index, so quiz_task_uuid and contributor_uuid become columns
    df = df.reset_index()
    # melt, prepare for pivot
    df = pd.melt(df, id_vars=["quiz_task_uuid", "contributor_uuid"], 
                 var_name = "choice", 
                 value_name = "value")
    df["choice"] = df["quiz_task_uuid"].str.slice(start=0, stop=6) + " " + df["choice"]
    pivot = pd.pivot_table(df, values='value', index='contributor_uuid', 
                        columns='choice', 
                        aggfunc=select)
    # to numpy array 
    array = pivot.to_numpy()
    # return the array 
    return array.astype(dtype=float)
# extract topic name (Argument/Holistic...) from df 
def get_topic(df):
    # df - e.g. df after reading in "Covid_Evidencev1-2020-10-04T1836-DataHunt.csv"
    # return the first one coz that's fast than unique()
    try: 
        return df["topic_name"][0]
    except Exception as e:
        print("This dataframe is empty")
# compute alpha values 
def compute_alpha(input_directory, fileName, question_type, question_label, measurement, decimal_places):
    # input_directory - e.g. "Specialist data"
    # fileName - e.g. "Covid_ArgumentRelevancev1.2-2020-10-04T1802-DataHunt.csv"
    # question_type - "mc"/"cl"
    # question _label - e.g. "T1.Q2"
    # measurement - "nominal", "ordinal", "interval"
    # decimal_plaes - number of decimal places to be rounded 
    array = to_reliability(input_directory, fileName, question_type, question_label)
    return np.round(alpha(reliability_data=array, level_of_measurement=measurement), decimal_places)
def get_question_types(repo):
    # repo - e.g. "Specialists DataHuntByCase"
    for file in os.listdir(repo):
        name = os.path.join(repo, file)
        df = pd.read_csv(name)
        topic = get_topic(df)
        print(file)
        for question in df["question_label"].unique():
            cur = get_question_type(topic, question)
            if cur == 'mc':
                measurement = get_measurement(topic, question)
            else:
                measurement = ''
            print(question, cur, measurement)
def calculate_alpha(repo, outrepo):
    # repo - e.g. "Specialists DataHuntByCase"
    for file in os.listdir(repo):
        questions_list = []
        question_type_list = []
        sub_type = []
        alphas = [] 
        pairable_list = []
        
        print()
        print(file)
        name = os.path.join(repo, file)
        df = pd.read_csv(name)
        try:
            topic = get_topic(df)
        # the dataframe might be empty
        except Exception as e: 
            print(file, "doesn't have any data")
            continue
        questions = df["question_label"].unique()
        # question - 'T1.Q1', 'T1.Q2'...
        for question in questions:
            # question_type - "mc"/"cl"/"tx"
            question_type = get_question_type(topic, question)
            if question_type == "mc":
                # "categorical"/"ordinal"
                try:
                    reliability_array = to_reliability(repo, 
                                                       file,
                                                       question_type, question)

                    measurement = get_measurement(topic, question)
                    pairs = count_pairable(reliability_array)
                    value = alpha(reliability_data=reliability_array, level_of_measurement=measurement)
                    print(question, question_type, measurement, 
                          'alpha: {0}'.format(value), '# of pairable values: {0}'.format(pairs))
                    # append in lists 
                    questions_list.append(question)
                    question_type_list.append(question_type)
                    sub_type.append(measurement)
                    alphas.append(value) 
                    pairable_list.append(pairs)
                except Exception as e: 
                    print(question, question_type, e)
                    questions_list.append(question)
                    question_type_list.append(question_type)
                    sub_type.append(measurement)
                    alphas.append(e) 
                    pairable_list.append(pairs)
            elif question_type == "cl":
                copy = df[df["question_label"] == question]
   
                cols = ["quiz_task_uuid", "contributor_uuid", "answer_label"]
                copy = copy.reindex(columns=cols)
                # how many answer_labels are there 
                # answers - 'answer_label_T1.Q1.A1'
                answers = ['answer_label_' + answer for answer in copy["answer_label"].unique()]
                # one-hot-encode
                copy = pd.get_dummies(copy, columns = ["answer_label"])
                # iterate through columns 
                for col in answers: 
                    cur_col = ["quiz_task_uuid", "contributor_uuid"] + [col]
                    df_slice = copy.reindex(columns=cur_col).copy()
                    try:
                        df_slice = pd.pivot_table(df_slice, values=col, index="contributor_uuid", 
                                                  columns='quiz_task_uuid', aggfunc=lambda x: list(x)[0])
                        array = df_slice.to_numpy()
                        pairs = count_pairable(array)
                        value = alpha(reliability_data=array, level_of_measurement="nominal")
                        print(question, question_type, col[13:], 
                              'alpha: {0}'.format(value), '# of pairable values: {0}'.format(pairs))
                        
                        
                        questions_list.append(question)
                        question_type_list.append(question_type)
                        sub_type.append(col[13:])
                        alphas.append(value) 
                        pairable_list.append(pairs)
                    except Exception as e:
                        print(question, question_type, col[13:], e)
                        
                        questions_list.append(question)
                        question_type_list.append(question_type)
                        sub_type.append(col[13:])
                        alphas.append(e) 
                        pairable_list.append(pairs)
        # write to csv 
        output = pd.DataFrame({"Question": questions_list, "Question_type": question_type_list, 
                               "Question_Subtype": sub_type, "Alpha": alphas, 
                               "Number of Pairable Elements": pairable_list})
        path = os.path.join(outrepo, 'out'+file)
        output.to_csv(path)
def count_pairable(a):
    count = 0
    for i in range(a.shape[1]):
        col = a[:, i]
        # number of non-zero values 
        true = ~pd.isnull(col)
        nozero = np.sum(true)
        count = count + nozero if nozero > 1 else count
    return count<jupyter_output><empty_output><jupyter_text>Triager to uAlpha<jupyter_code># mapping topic_name to integers 
def mapping(df, dic):
    # df - dataframe to be mapped 
    # dic - dictionary of mapping 
    df["topic_name"] = df["topic_name"].map(dic)
    return df 


# filter df and select only certain cols as input
def slice_input(df):
    cols = ["contributor_uuid", "topic_name", "start_pos",
            "end_pos", "article_text_length", "article_number", "created"]
    return df.reindex(columns=cols)


# filter df and select only certain cols as output
def slice_output(df):
    cols = ["contributor_uuid", "topic_name", "blank",  
            "start_pos_adjusted", "end_pos_adjusted"]
    return df.reindex(columns=cols)
    


# read in a directory that contains Triager csvs 
# split each Triager csv to 4 other csvs based on topic_name (argument, reasoning, etc)
# and write it as csvs in the output directory
def triager_split(input_directory, output_directory):
    # input_directory - e.g. "Triager data"
    # output_directory - e.g. "Triager output"
    for file in os.listdir(input_directory):
        triager_split_help(input_directory, file, output_directory)
    print("Triager Tranformation Done! Ready to be imported to uAlpha!")
# resolve overlapping issues by merging overlapping rows!!!
def resolve_overlapping(df):
    # df - a dataframe with 5 cols {0: index, 1: user_id, 2:topic_name(as int), 3:blank, 4:start_pos, 5:end_pos}
    # this function only works when start_pos are sorted for each user!!!
    created = df["created"].unique()
    # iterate through all unique user 
    for time in created:
        # start and end index of that user 
        start_i = df[df["created"] == time].index.values[0]
        end_i = df[df["created"] == time].index.values[-1]
        # iterate through the rows of that user 
        i = start_i
        while i < end_i:
            # if the start_pos of next row > end_pos of this row
            # no overlap
            # 5 - the col of end_pos
            # the 10th column is end_pos_adjusted
            end_pos = df.iat[i, 10]
            # the 9th column is start_pos_adjusted
            start_pos = df.iat[i+1, 9]
            if start_pos > end_pos:
                i += 1 
            else:
                # if the article_number is the same, doesn't really matter 
                # modify the end_pos of this row, merge 
                if df.iat[i, 10] < df.iat[i+1 , 10]:
                    df.iat[i, 10] = df.iat[i+1, 10].copy()
                # drop the next row 
                df.drop(i+1, inplace=True)
                # reindex
                df.reset_index(drop=True, inplace=True)
                # decrement end_i 
                end_i -= 1
    return df
# add adjusted start_pos and adjusted end_pos for df 
def adjust(df):
    # df - a dataframe with ["article_number", "article_text_length", "start_pos", "end_pos"] cols
    article_number = df["article_number"]
    article_text_length = df["article_text_length"]
    start_pos = df["start_pos"]
    end_pos = df["end_pos"]
    # the cumulative sum of article_text_length
    cumulative = []
    start_pos_adjusted = []
    end_pos_adjusted = []
    pre_total = 0
    total = article_text_length[0]
    cur_num = article_number[0]
    for i in np.arange(len(article_number)):
        if article_number[i] == cur_num:
            total = total + 0
            cumulative.append(total)
        else:
            pre_total = total
            cur_num = article_number[i]
            total = total + article_text_length[i]
            cumulative.append(total)
        start_pos_adjusted.append(start_pos[i] + pre_total)
        end_pos_adjusted.append(end_pos[i] + pre_total)
    df["cumulative"] = cumulative
    df["start_pos_adjusted"] = start_pos_adjusted
    df["end_pos_adjusted"] = end_pos_adjusted
    return df 
# take in a Triager csv and split it to 4 different csvs 
# based on topic_name 
def triager_split_help(input_directory, fileName, output_directory):
    # input_directory - e.g. "Triager data"
    # fileName - e.g. "Covid_Form1.0.adjudicated-2020-10-04T2314-Tags.csv"
    # output_directory - e.g. "Triager output"
    name = os.path.join(input_directory, fileName)
    df = pd.read_csv(name)
    # different topic_names 
    topic_names = df["topic_name"].unique()
    for topic in topic_names: 
        # select all rows for that specific topic name 
        df_topic = df.loc[df["topic_name"] == topic]
        # select only these columns 
        filtered = slice_input(df_topic)
        # slice contributor_uuid with the first 6 chars 
        filtered["contributor_uuid"] = filtered["contributor_uuid"].str[:6]
        # map topic_name to integers
        mapping(filtered, triager_dic)
        # add a blank column 
        filtered.insert(loc=3, column="blank", value="")
        # sort
        filtered = filtered.sort_values(by=["article_number", "contributor_uuid", "start_pos"])
        # re-index such that its index starts from 0 again
        filtered.reset_index(drop=True, inplace=True)
        # add modified columns
        filtered = adjust(filtered)
        # resolve overlapping issues 
        filtered = resolve_overlapping(filtered)
        # filter and select only certain columns 
        output = slice_output(filtered)
        # sort again by contributor
        output = output.sort_values(by=["contributor_uuid", "start_pos_adjusted"])
        # add one column of u + str(row) and set it as index 
        index = 'u' + pd.Series(filtered.index).astype(str)
        output.set_index(keys=index, inplace=True)
        # the batch_name, e.g. "Covid"
        batch_name = re.split(r'_', fileName)[0]
        name = '{0}-Triager-{1}.csv'.format(batch_name, topic)
        path = os.path.join(output_directory, name)
        # write to csvs
        output.to_csv(path, header=False)
    return None
# debugging function - find potential overlapping issue 
def detect_overlapping(input_directory, fileName):
    # input directory - e.g. "Triager output"
    # fileName - e.g. "Covid-Triager-Arguments.csv"
    name = os.path.join(input_directory, fileName)
    df = pd.read_csv(name, header=None)
    # all the unique user 
    users = df.iloc[:, 1].unique()
    # iterate through all unique user 
    for user in users:
        # first only select the rows of that user 
        # use copy to deal with the wanring
        piece = df.loc[df[1] == user].copy()
        # sort the 4th column
        piece.sort_values(by=4, inplace=True)
        # find or not
        found = False 
        start_pos = np.array(piece[4])
        end_pos = np.array(piece[5])
        for i in np.arange(1, len(start_pos)):
            if start_pos[i] > end_pos[i-1]:
                continue
            else:
                found = True
                print('user {0}, start_pos {1}, end_pos {2} begin overlapping!'.format(user, start_pos[i], end_pos[i]))
                break 
    return None<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1117368109069354, 'fraction_numerical': 0.015115589804386485, 'mean_word_length': 2.2252668472200097, 'pattern_counts': {'":': 63, '<': 15, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '443071', 'license': 'MIT', 'n_tokens_mistral': 6502, 'n_tokens_neox': 6029, 'n_words': 1597, 'path': '.ipynb_checkpoints/Validation-checkpoint.ipynb', 'repo_name': 'Hegelim/PE-Validation-Fall-20', 'size': '20244'}