<jupyter_start><jupyter_text>Analysis of test set human predictions. 3 human reviewers labeled all videos in the test set, for every 10 seconds of each video, for audio, video, and audio+video inputs.<jupyter_code>import pandas as pd
from matplotlib import pyplot as plt
import matplotlib
import os
import numpy as np
import networkx as nx
from sklearn.metrics import cohen_kappa_score
# Humans make predictions every 10 seconds on test set.
INTERVAL = 10
# Load information about each video
video_info = pd.read_csv('/datasets/cgn/EGOCOM/video_info.csv')
print('Total hours of video in EgoCom dataset: {:.1f}'.format(
    np.sum(video_info['duration_seconds']) / 3600))
pd.read_csv('/datasets/cgn/EGOCOM/ground_truth_transcriptions.csv').head()
# Load ground truth labels. This takes a few minutes.
data_dir = "/datasets/cgn/EGOCOM/human_predictions/"
egocom_loc = "/datasets/cgn/EGOCOM/egocom_features/no_audio/"
cols = ['video_id', 'clip_id', 'video_speaker_id', 'is_speaking', 'multiclass_speaker_label']
egocom_features = pd.read_csv(egocom_loc + "egocom_features_history_10sec.csv.gz")[cols]<jupyter_output><empty_output><jupyter_text>Notes about 'egocom_features_history_NNsec.csv.gz' files. the clip_id is not a universal id across files. The clip_id starts at 00:10 for the 10sec history and 00:05 for 5sec history and so forth. So for 'egocom_features_history_10sec.csv.gz':* clip_id = 0 refers to the 1 second clip from 00:10-00:11* clip_id = 1 refers to the 1 second clip from 00:11-00:12* clip_id = 2 refers to the 1 second clip from 00:12-00:13* and so forth... And for 'egocom_features_history_5sec.csv.gz':* clip_id = 0 refers to the 1 second clip from 00:05-00:06* clip_id = 1 refers to the 1 second clip from 00:06-00:07* clip_id = 2 refers to the 1 second clip from 00:07-00:08* and so forth...  compute baselines<jupyter_code>from sklearn.metrics import confusion_matrix

print('Test set\n--------\n')

print("Baselines for anyone speaking")
test_set_is_speaking = egocom_features[
    egocom_features.video_id.isin(video_info[video_info['test']].video_id)
]['is_speaking']

print('Baseline prob(not speaking):', round((len(test_set_is_speaking) - sum(test_set_is_speaking)) / len(test_set_is_speaking), 2))

for future in [1, 5, 10]:
    # We can compute the prob(speaking in future | speaking now) in two ways
    # Method 1
    cmat = confusion_matrix(
        test_set_is_speaking[:-future], test_set_is_speaking[future:])
    conditional_prob = cmat[0][0] / (cmat[0][1] + cmat[0][0])
    # Method 2
    conditional_prob2 = sum((test_set_is_speaking[:-future].values == 0) &\
                      (test_set_is_speaking[future:].values == 0)) /\
          sum(test_set_is_speaking[:-future].values == 0)
    assert(conditional_prob == conditional_prob2)
    print('Future:', future, 'second(s)\t| prob(not speaking in future | not speaking now):', 
          round(conditional_prob, 2))
    
for future in [1, 5, 10]:
    # We can compute the prob(speaking in future | speaking now) in two ways
    # Method 1
    cmat = confusion_matrix(
        test_set_is_speaking[:-future], test_set_is_speaking[future:])
    conditional_prob = cmat[1][1] / (cmat[1][0] + cmat[1][1])
    # Method 2
    conditional_prob2 = sum((test_set_is_speaking[:-future].values == 1) &\
                      (test_set_is_speaking[future:].values == 1)) /\
          sum(test_set_is_speaking[:-future].values == 1)
    assert(conditional_prob == conditional_prob2)
    print('Future:', future, 'second(s)\t| prob(speaking in future | speaking now):', 
          round(conditional_prob, 2))
    
for future in [1, 5, 10]:
    cmat = confusion_matrix(
        test_set_is_speaking[:-future], test_set_is_speaking[future:])
    conditional_prob = (cmat[0][0] + cmat[1][1]) / sum(sum(cmat))
    print('Future:', future, 'second(s)\t| prob(same, i.e. still speaking or not):', 
          round(conditional_prob, 2))
    
    
print("\nBaselines for host speaking")
test_set_host_is_speaking = egocom_features[
    egocom_features.video_id.isin(video_info[video_info['test'] & video_info['speaker_is_host']].video_id)
]['is_speaking']

for future in [1, 5, 10]:
    # We can compute the prob(speaking in future | speaking now) in two ways
    # Method 1
    cmat = confusion_matrix(
        test_set_host_is_speaking[:-future], test_set_host_is_speaking[future:])
    conditional_prob = cmat[0][0] / (cmat[0][1] + cmat[0][0])
    # Method 2
    conditional_prob2 = sum((test_set_host_is_speaking[:-future].values == 0) &\
                      (test_set_host_is_speaking[future:].values == 0)) /\
          sum(test_set_host_is_speaking[:-future].values == 0)
    assert(conditional_prob == conditional_prob2)
    print('Future:', future, 'second(s)\t| prob(host not speaking in future | host not speaking now):', 
          round(conditional_prob, 2))
    
for future in [1, 5, 10]:
    # We can compute the prob(speaking in future | speaking now) in two ways
    # Method 1
    cmat = confusion_matrix(
        test_set_host_is_speaking[:-future], test_set_host_is_speaking[future:])
    conditional_prob = cmat[1][1] / (cmat[1][0] + cmat[1][1])
    # Method 2
    conditional_prob2 = sum((test_set_host_is_speaking[:-future].values == 1) &\
                      (test_set_host_is_speaking[future:].values == 1)) /\
          sum(test_set_host_is_speaking[:-future].values == 1)
    assert(conditional_prob == conditional_prob2)
    print('Future:', future, 'second(s)\t| prob(host speaking in future | host speaking now):', 
          round(conditional_prob, 2))
    
for future in [1, 5, 10]:
    cmat = confusion_matrix(
        test_set_host_is_speaking[:-future], test_set_host_is_speaking[future:])
    conditional_prob = (cmat[0][0] + cmat[1][1]) / sum(sum(cmat))
    print('Future:', future, 'second(s)\t| prob(host still speaking or not):', 
          round(conditional_prob, 2))    


print("\nBaselines for multi-speaker")
test_set_label = egocom_features[
    egocom_features.video_id.isin(video_info[video_info['test']].video_id)
]['multiclass_speaker_label']
    
for future in [1, 5, 10]:
    conditional_prob = sum(test_set_label[:-future].values == test_set_label[future:].values) / len(test_set_label)
    print('Future:', future, 'second(s)\t| prob(same person is speaking):', 
          round(conditional_prob, 2))

# create map that relabels participants 2 and 3 to 2 (non-host),
# 1 to 1 (host), and 0 to 0 (no one is peaking)
map32 = {-1:0, 0:0, 1:1, 2:2, 3:2}
fontsize = 24
labels = {0: 'No speech', 1: 'Host', 2: 'Participants*',}
# Compute multi-speaker transition matrix
for future in [1, 5, 10]:
    cmat = confusion_matrix(
        test_set_label[:-future].map(map32),  # now (rows)
        test_set_label[future:].map(map32),  # future (columns)
    )
    conditional_matrix = (cmat.T / cmat.sum(axis=1)).T
    print('Future:', future, 'second(s)\t| prob(same person is speaking):\n', conditional_matrix)
    
    # Generate transition graph
    G = nx.from_numpy_matrix(conditional_matrix.round(2), create_using=nx.DiGraph)
    # Set node labels
    nx.set_node_attributes(G, {k: {'label': labels[k]} for k in labels.keys()})
    nx.set_edge_attributes(G, {(e[0], e[1]): {'label': e[2]['weight']} for e in G.edges(data=True)})
    D = nx.drawing.nx_agraph.to_agraph(G)
    # Modify node fillcolor and edge color.
    D.node_attr.update(color='#406ca7', style='filled', fillcolor='#58d631', fontsize=fontsize)
    D.edge_attr.update(color='#406ca7', arrowsize=1.5, fontsize=fontsize)
    pos = D.layout('circo')
    D.draw('transition_graph_test_{}{}.pdf'.format('0' if future < 10 else '', future))
    

from sklearn.metrics import confusion_matrix

output_english = False  # Set to false to output latex
draw_graphs = True  # Turn on to draw graphs
print_matrices = True  # Turn on to print conditional matrices that go with the graphs

print('Train set\n--------\n')

print("Baselines for anyone speaking")
train_set_is_speaking = egocom_features[
    egocom_features.video_id.isin(video_info[video_info['train']].video_id)
]['is_speaking']

print('Baseline prob(not speaking):', round((len(train_set_is_speaking) - sum(train_set_is_speaking)) / len(train_set_is_speaking), 2))

for future in [1, 5, 10]:
    # We can compute the prob(speaking in future | speaking now) in two ways
    # Method 1
    cmat = confusion_matrix(
        train_set_is_speaking[:-future], train_set_is_speaking[future:])
    conditional_prob = cmat[0][0] / (cmat[0][1] + cmat[0][0])
    # Method 2
    conditional_prob2 = sum((train_set_is_speaking[:-future].values == 0) &\
                      (train_set_is_speaking[future:].values == 0)) /\
          sum(train_set_is_speaking[:-future].values == 0)
    assert(conditional_prob == conditional_prob2)
    if output_english:
        print('Future:', future, 'second(s)\t| prob(not speaking in future | not speaking now):', 
          round(conditional_prob, 2))
    else:
        print(r'$p(\neg s_{{t = {f}}} \vert  \neg s_{{t = 0}}) = {p:.2f}$'.format(f=future,p=conditional_prob))
        
    
for future in [1, 5, 10]:
    # We can compute the prob(speaking in future | speaking now) in two ways
    # Method 1
    cmat = confusion_matrix(
        train_set_is_speaking[:-future], train_set_is_speaking[future:])
    conditional_prob = cmat[1][1] / (cmat[1][0] + cmat[1][1])
    # Method 2
    conditional_prob2 = sum((train_set_is_speaking[:-future].values == 1) &\
                      (train_set_is_speaking[future:].values == 1)) /\
          sum(train_set_is_speaking[:-future].values == 1)
    assert(conditional_prob == conditional_prob2)
    if output_english:
        print('Future:', future, 'second(s)\t| prob(speaking in future | speaking now):', 
          round(conditional_prob, 2))
    else:
        print(r'$p(s_{{t = {f}}} \vert  s_{{t = 0}}) = {p:.2f}$'.format(f=future,p=conditional_prob))
    
for future in [1, 5, 10]:
    cmat = confusion_matrix(
        train_set_is_speaking[:-future], train_set_is_speaking[future:])
    conditional_prob = (cmat[0][0] + cmat[1][1]) / sum(sum(cmat))
    if output_english:
        print('Future:', future, 'second(s)\t| prob(same, i.e. still speaking or not):', 
          round(conditional_prob, 2))
    else:
        print(r'$p(s_{{t = {f}}} = s_{{t = 0}}) = {p:.2f}$'.format(f=future,p=conditional_prob))

# create map that relabels participants 2 and 3 to 2 (non-host),
# 1 to 1 (host), and 0 to 0 (no one is peaking)
fontsize = 24
labels = {0: 'Speaking', 1: 'Not speaking',}
# Compute multi-speaker transition matrix
for future in [1, 5, 10]:
    if print_matrices:
        cmat = confusion_matrix(
            train_set_is_speaking[:-future],  # now (rows)
            train_set_is_speaking[future:],  # future (columns)
        )
        conditional_matrix = (cmat.T / cmat.sum(axis=1)).T
        print('Future:', future, 'second(s)\t| prob(same person is speaking):\n', conditional_matrix)
    
    if draw_graphs:
        # Generate transition graph
        G = nx.from_numpy_matrix(conditional_matrix.round(2), create_using=nx.DiGraph)
        # Set node labels
        nx.set_node_attributes(G, {k: {'label': labels[k]} for k in labels.keys()})
        nx.set_edge_attributes(G, {(e[0], e[1]): {'label': e[2]['weight']} for e in G.edges(data=True)})
        D = nx.drawing.nx_agraph.to_agraph(G)
        # Modify node fillcolor and edge color.
        D.node_attr.update(color='#406ca7', style='filled', fillcolor='#58d631', fontsize=fontsize)
        D.edge_attr.update(color='#406ca7', arrowsize=1.5, fontsize=fontsize)
        pos = D.layout('dot')
        D.draw('transition_graph_train_any_speaking_{}{}.pdf'.format('0' if future < 10 else '', future)) 



print("\nBaselines for host speaking")


train_set_host_is_speaking = egocom_features[
    egocom_features.video_id.isin(video_info[video_info['train'] & video_info['speaker_is_host']].video_id)
]['is_speaking']

for future in [1, 5, 10]:
    # We can compute the prob(speaking in future | speaking now) in two ways
    # Method 1
    cmat = confusion_matrix(
        train_set_host_is_speaking[:-future], train_set_host_is_speaking[future:])
    conditional_prob = cmat[0][0] / (cmat[0][1] + cmat[0][0])
    # Method 2
    conditional_prob2 = sum((train_set_host_is_speaking[:-future].values == 0) &\
                      (train_set_host_is_speaking[future:].values == 0)) /\
          sum(train_set_host_is_speaking[:-future].values == 0)
    assert(conditional_prob == conditional_prob2)
    if output_english:
        print('Future:', future, 'second(s)\t| prob(host not speaking in future | host not speaking now):', 
          round(conditional_prob, 2))
    else:
        print(r'$p(\neg h_{{t = {f}}} \vert  \neg h_{{t = 0}}) = {p:.2f}$'.format(f=future,p=conditional_prob))
    
for future in [1, 5, 10]:
    # We can compute the prob(speaking in future | speaking now) in two ways
    # Method 1
    cmat = confusion_matrix(
        train_set_host_is_speaking[:-future], train_set_host_is_speaking[future:])
    conditional_prob = cmat[1][1] / (cmat[1][0] + cmat[1][1])
    # Method 2
    conditional_prob2 = sum((train_set_host_is_speaking[:-future].values == 1) &\
                      (train_set_host_is_speaking[future:].values == 1)) /\
          sum(train_set_host_is_speaking[:-future].values == 1)
    assert(conditional_prob == conditional_prob2)
    if output_english:
        print('Future:', future, 'second(s)\t| prob(host speaking in future | host speaking now):', 
          round(conditional_prob, 2))
    else:
        print(r'$p(h_{{t = {f}}} \vert  h_{{t = 0}}) = {p:.2f}$'.format(f=future,p=conditional_prob))
    
for future in [1, 5, 10]:
    cmat = confusion_matrix(
        train_set_host_is_speaking[:-future], train_set_host_is_speaking[future:])
    conditional_prob = (cmat[0][0] + cmat[1][1]) / sum(sum(cmat))
    if output_english:
        print('Future:', future, 'second(s)\t| prob(host still speaking or not):', 
          round(conditional_prob, 2))
    else:
        print(r'$p(h_{{t = {f}}} = h_{{t = 0}}) = {p:.2f}$'.format(f=future,p=conditional_prob))

# create map that relabels participants 2 and 3 to 2 (non-host),
# 1 to 1 (host), and 0 to 0 (no one is peaking)
fontsize = 24
labels = {0: 'Host not speaking', 1: 'Host speaking',}
# Compute multi-speaker transition matrix
for future in [1, 5, 10]:
    if print_matrices:
        cmat = confusion_matrix(
            train_set_host_is_speaking[:-future],  # now (rows)
            train_set_host_is_speaking[future:],  # future (columns)
        )
        conditional_matrix = (cmat.T / cmat.sum(axis=1)).T
        print('Future:', future, 'second(s)\t| prob(same person is speaking):\n', conditional_matrix)
    
    if draw_graphs:
        # Generate transition graph
        G = nx.from_numpy_matrix(conditional_matrix.round(2), create_using=nx.DiGraph)
        # Set node labels
        nx.set_node_attributes(G, {k: {'label': labels[k]} for k in labels.keys()})
        nx.set_edge_attributes(G, {(e[0], e[1]): {'label': e[2]['weight']} for e in G.edges(data=True)})
        D = nx.drawing.nx_agraph.to_agraph(G)
        # Modify node fillcolor and edge color.
        D.node_attr.update(color='#406ca7', style='filled', fillcolor='#58d631', fontsize=fontsize)
        D.edge_attr.update(color='#406ca7', arrowsize=1.5, fontsize=fontsize)
        pos = D.layout('dot')
        D.draw('transition_graph_train_host_speaking_{}{}.pdf'.format('0' if future < 10 else '', future)) 



print("\nBaselines for multi-speaker")


train_set_label = egocom_features[
    egocom_features.video_id.isin(video_info[video_info['train']].video_id)
]['multiclass_speaker_label']

for future in [1, 5, 10]:
    conditional_prob = sum(train_set_label[:-future].values == train_set_label[future:].values) / len(train_set_label)
    if output_english:
        print('Future:', future, 'second(s)\t| prob(same person is speaking):', 
              round(conditional_prob, 2))
    else:
        print(r'$p(m_{{t = {f}}} = m_{{t = 0}}) = {p:.2f}$'.format(f=future,p=conditional_prob))

# create map that relabels participants 2 and 3 to 2 (non-host),
# 1 to 1 (host), and 0 to 0 (no one is peaking)
map32 = {-1:0, 0:0, 1:1, 2:2, 3:2}
fontsize = 24
labels = {0: 'No speech', 1: 'Host', 2: 'Participants*',}
# Compute multi-speaker transition matrix
for future in [1, 5, 10]:
    if print_matrices:
        cmat = confusion_matrix(
            train_set_label[:-future].map(map32),  # now (rows)
            train_set_label[future:].map(map32),  # future (columns)
        )
        conditional_matrix = (cmat.T / cmat.sum(axis=1)).T
        print('Future:', future, 'second(s)\t| prob(same person is speaking):\n', conditional_matrix)
    
    if draw_graphs:
        # Generate transition graph
        G = nx.from_numpy_matrix(conditional_matrix.round(2), create_using=nx.DiGraph)
        # Set node labels
        nx.set_node_attributes(G, {k: {'label': labels[k]} for k in labels.keys()})
        nx.set_edge_attributes(G, {(e[0], e[1]): {'label': e[2]['weight']} for e in G.edges(data=True)})
        D = nx.drawing.nx_agraph.to_agraph(G)
        # Modify node fillcolor and edge color.
        D.node_attr.update(color='#406ca7', style='filled', fillcolor='#58d631', fontsize=fontsize)
        D.edge_attr.update(color='#406ca7', arrowsize=1.5, fontsize=fontsize)
        pos = D.layout('circo')
        D.draw('transition_graph_train_{}{}.pdf'.format('0' if future < 10 else '', future))
# Example conversation transition map.

# Get conversation with longest duration in train set.
example_conversation_label = egocom_features[
    egocom_features.video_id.isin(video_info[video_info['conversation_id'] == 'day_6__con_3'].video_id)
]['multiclass_speaker_label']

map32 = {-1:0, 0:0, 1:1, 2:2, 3:3}
fontsize = 24
labels = {0: 'No speech', 1: 'Host', 2: 'Participant 1', 3: 'Participant 2'}
# Compute multi-speaker transition matrix
for future in [1, 5, 10]:
    cmat = confusion_matrix(
        example_conversation_label[:-future],  # now (rows)
        example_conversation_label[future:],  # future (columns)
    )
    conditional_matrix = (cmat.T / cmat.sum(axis=1)).T
    print('Future:', future, 'second(s)\t| prob(same person is speaking):\n', conditional_matrix)
    
    # Generate transition graph
    G = nx.from_numpy_matrix(conditional_matrix.round(2), create_using=nx.DiGraph)
    # Set node labels
    nx.set_node_attributes(G, {k: {'label': labels[k]} for k in labels.keys()})
    nx.set_edge_attributes(G, {(e[0], e[1]): {'label': e[2]['weight']} for e in G.edges(data=True)})
    D = nx.drawing.nx_agraph.to_agraph(G)
    # Modify node fillcolor and edge color.
    D.node_attr.update(color='#406ca7', style='filled', fillcolor='#58d631', fontsize=fontsize)
    D.edge_attr.update(color='#406ca7', arrowsize=1.5, fontsize=fontsize)
    pos = D.layout('circo')
    D.draw('transition_graph_day6con3_{}{}.pdf'.format('0' if future < 10 else '', future))
print('Accuracy just predicting host for multiclass:', round(sum(train_set_label == 1) / len(train_set_label), 3))<jupyter_output>Accuracy just predicting host for multiclass: 0.486<jupyter_text>Compute human prediction analysis<jupyter_code># Load ground truth labels. This takes a few minutes.
data_dir = "/datasets/cgn/EGOCOM/human_predictions/"
egocom_loc = "/datasets/cgn/EGOCOM/egocom_features/no_audio/"
cols = ['video_id', 'clip_id', 'video_speaker_id', 'is_speaking', 'multiclass_speaker_label']
egocom_features = pd.read_csv(egocom_loc + "egocom_features_history_10sec.csv.gz")[cols]
# Humans make predictions every 10 seconds on test set.
INTERVAL = 10
files = [
    'test_set/egocom_test_set_human_predictions_curtis.csv',
    'test_set/egocom_test_set_human_predictions_cindy.csv',
    'test_set/egocom_test_set_human_predictions_lisa.csv',
]
# Print out the total number of labels done by humans
for file in files:
    df = pd.read_csv(data_dir + file, skiprows=8).set_index('Unnamed: 0')
    # Remove columns that just contain times
    df = df[[x for x in df.columns if 'Unnamed' not in x]]
    print(file, df.count().sum())
def read_human_labeling_data(filename):
    # Read in data
    df = pd.read_csv(filename, skiprows=5)
    # Remove duplicate columns with times
    df = df.T[['CONTENT LINK (240p)' not in x or i == 0 for i, x in enumerate(df.columns)]].T
    # Remove 480p video link row
    df.drop(0, inplace=True)

    # Prepare column and indices
    avmap = {
        'AUDIO ONLY': 'a',
        'VIDEO ONLY': 'v',
        'AUDIO + VI': 'av',    
    }
    modalities = [avmap[x[:10]] for x in df.iloc[0].values.tolist() if type(x) is str  for i in range(3)]
    videos = [x[:-4] for x in df.columns if 'vid' in x for i in range(3)]
    futures = df.iloc[1].values[1:].astype(int).tolist()

    # Remove top rows used to get the above column information
    df.drop([1, 2], inplace=True)

    # Rename video frame column
    df['video_frame'] = df['CONTENT LINK (240p)']
    del df['CONTENT LINK (240p)']

    # df.reset_index(drop=True, inplace=True)
    df.set_index('video_frame', inplace=True)

    # Name columns
    dft = df.T
    dft.index = [pd.Series(videos, name='video'), pd.Series(modalities, name='modality'), pd.Series(futures, name='future'),]
    df = dft.T

    # Convert strings to floats
    df = df.astype(float)
    
    # Replace 2's with 1's
    df.replace(2., 1., inplace=True)
    
    return df
# Parse human prediction experiment data.
results = []
ratings = {}
total_ratings = 0
for file in files:  # Each human label has their own file. read it.
    filename = data_dir + file
    df = read_human_labeling_data(filename)
    print(filename, df.count().sum())
    total_ratings += df.count().sum()
    for key in df.columns:
        video, modality, future = key
        # Get video id (vid)
        vid = video_info[video_info['video_name'] == video]['video_id']
        assert(len(vid) == 1)
        vid = vid.values[0]
        # Compute accuracy
        shift_amount = future - 1
        truth = egocom_features[egocom_features['video_id'] == vid]['is_speaking'][shift_amount::INTERVAL].values
        human_pred = df[key].values
        length = min(len(truth), len(human_pred))
        acc = np.mean(truth[:length] == human_pred[:length])
#         if verbose:
#             print('Accuracy {}s in future: {:.2}'.format(shift_amount + 1, acc))
        results.append({
            'file': file,
            'video': video,
            'modality': modality,
            'future': future,
            'acc': acc,
            'rater': file.split('_')[-1][:-4],
        })
        ratings[(file, key)] = human_pred
results = pd.DataFrame(results)

# Compute inter-rater reliabilities
irrs = []
for key in df.columns:
    for file1, file2 in [(files[0], files[1]), (files[1], files[2]), (files[2], files[0])]:
        r1 = ratings[(file1, key)][np.isfinite(ratings[(file1, key)])].astype(int)
        r2 = ratings[(file2, key)][np.isfinite(ratings[(file2, key)])].astype(int)
        length = min(len(r1), len(r2))
        irr = cohen_kappa_score(
            r1[:length],
            r2[:length],
        )
        irrs.append({
            'filename': key[0],
            'human1': file1.split("_")[-1][:-4],
            'human2': file2.split("_")[-1][:-4],
            'modality': key[1],
            'future': key[2],
            'irr': irr,
        })
#                 (key, file1.split("_")[-1], file2.split("_")[-1])] = irr
irrs = pd.DataFrame(irrs)
irrs['video'] = irrs['filename']
del irrs['filename']

# Combine inter-rater cohen cappas scores with human rating accuracies
human = pd.merge(
    results,
    irrs,
    on=['video', 'future', 'modality'],
).sort_values(by=['video', 'future', 'modality'])

# Remove rows where the irr has nothing to do with the rater
human = human[(human['rater'] == human['human1']) | (human['rater'] == human['human2'])]
print('Number of unique videos and perspectives in test set:', len(video_info[video_info['test']]['video_id']))
print('Total number of cohen-kappas pairwise scores | len(irrs):', len(irrs))
print('Total number of video/modality/future/raters combos | len(results):', len(results))
print('Total turn-taking predictions labeled by humans:', total_ratings)
irr_threshold = 0.3

print("This threshold (irr > {}) was chosen because it had no significant".format(irr_threshold),
      "effect on overall avearge prediction accuracy or standard deviation.")
print('both huamns and machine are given a past of 10 seconsd')
print('Accuracy without thresholds:')
print(human.groupby(['future', 'modality']).mean()[['acc']].T.round(2))
print()
print("*"*50)
frac_tossed = (len(results) - len(human[human['irr'] > irr_threshold][['future', 'modality', 'acc', 'video', 'rater']].drop_duplicates())) / len(results)
print('Fraction of ratings we throw out:', round(frac_tossed, 2))
print("*"*50)
print()
# First get only the unique accuracy rows
tmp1 = human[human['irr'] > irr_threshold][['future', 'modality', 'acc', 'video', 'rater']].drop_duplicates()
tmp1 = tmp1.groupby(['future', 'modality']).agg(['count','mean', 'std']).round(2).T
tmp2 = human[human['irr'] > irr_threshold][['human1', 'human2', 'future', 'video', 'modality', 'irr']].drop_duplicates()
tmp2 = tmp2.groupby(['future', 'modality']).mean().round(2).T
df = pd.concat([tmp1, tmp2], axis=0)
df['average'] = df.mean(numeric_only=True, axis=1).round(2)
print(df.to_latex())
df
irrs[irrs.irr > .3].groupby('video')['irr'].mean().head()<jupyter_output><empty_output><jupyter_text>Looking at individual reviewers<jupyter_code>for future in [1, 5, 10]:
    plt.figure()
    for modality in ['a', 'v', 'av']:
        tmpdf = results[(results['future'] == future) & (results['modality'] == modality)].groupby(['video']).mean()['acc']
        tmpdf = tmpdf.reindex([x for i, x in enumerate(videos) if i %3 == 0],).dropna()
        tmpdf.plot(label=modality)
    plt.legend(title='Modality')
    plt.title('Future:' + str(future) + 's')
    plt.ylabel('Accuracy')
    plt.xticks([1, len(tmpdf) // 2, len(tmpdf)], [1, len(tmpdf) // 2, len(tmpdf)])
    plt.show()

results.groupby(['rater', 'future']).mean()['acc']
results.groupby(['modality', 'future']).mean()['acc'].round(2)
results.groupby('rater')['acc'].agg(['mean', 'std'])<jupyter_output><empty_output><jupyter_text>Making figure for paper<jupyter_code>df.T['acc', 'mean']
df.T['acc', 'std']
# # 5 second history
# ml_results = {
#     'v' : [0.65, 0.65, 0.65],
#     'a' : [0.72, 0.65, 0.65],
#     'av' : [0.71, 0.65, 0.65],   
# }

# 5 second history (with prior because humans also use the prior)
ml_results = {
    'v' : [0.73, 0.65, 0.65],
    'a' : [0.75, 0.65, 0.65],
    'av' : [0.75, 0.66, 0.65],   
} 

# # 5 second history -- uses logreg
# old_lr_ml_results = {
#     'v' : [0.67, 0.65, 0.65],
#     'a' : [0.72, 0.69, 0.67],
#     'av' : [0.71, 0.69, 0.67],   
# }

# # 10 second history
# ml_results = {
#     'v' : [0.65, 0.65, 0.65],
#     'a' : [0.71, 0.69, 0.67],
#     'av' : [0.71, 0.68, 0.67],   
# }

savefig = True
fontsize = 30

# Make sure fontsize is set
matplotlib.rcParams.update({'font.size': 3*fontsize})
plt.figure(figsize=(fontsize,fontsize/2))

# Create legend labels.
plt.plot([],marker="", ls="", label='Machine (MLP)')
plt.errorbar([],[],marker="", ls="", label='Human')
plt.gca().set_prop_cycle(None)

# Create markers.
markers = ['s', 'o', '^']

for i, modality in enumerate(['a','v','av']):
    if modality == 'av':
        name = 'audio+video'
    else:
        name = 'audio' if modality == 'a' else 'video'
    x = np.array([1,5,10]) + 0.025 * i
    y = df.T['acc', 'mean'].reset_index()[df.T['acc', 'mean'].reset_index()['modality'] == modality]['acc', 'mean']
    std = df.T['acc', 'std'].reset_index()[df.T['acc', 'std'].reset_index()['modality'] == modality]['acc', 'std'] / 2 / np.sqrt(3)
    p = plt.plot(x, ml_results[modality], marker=markers[i], 
        markersize=fontsize*0.8, linestyle=':',
        linewidth=fontsize / 3,label=name, alpha = 1)
    color = p[0].get_color()
    plt.errorbar(x, y, yerr=std, marker=markers[i], markersize=fontsize*0.8, 
        linewidth=fontsize / 3,label=name, alpha = 0.8, color=color)
plt.ylabel('')
plt.xticks([1,5,10])
plt.yticks([0.6, 0.7, 0.8])

ax = plt.gca()
        
# Format y-axis values
ax.set_yticklabels([str(int(round(z*100, 0)))+"%" for z in ax.get_yticks()])

# Hide the right and top spines
ax.spines['right'].set_visible(False)
ax.spines['top'].set_visible(False)

# Only show ticks on the left and bottom spines
ax.yaxis.set_ticks_position('left')
ax.xaxis.set_ticks_position('bottom')

legend_title = ''# 'Modality'
legend = plt.legend(title=legend_title, fontsize=1.3*fontsize, ncol=2, loc=1)

plt.axhline(1 - 0.35, xmin=.04, xmax=.96, color='black', ls='--', linewidth = fontsize/4)
plt.text(1,1 - 0.35 - 0.02,'Baseline: 65% = 1 - prob(anyone is speaking) = 1 - 0.35', fontsize=40)

# plt.axhline(0.5, xmin=.04, xmax=.96, color='black', ls='--', linewidth = fontsize/4)
# plt.text(1,0.51,'Random guessing', fontsize=40)

# plt.axhline(1 - 0.48, xmin=.04, xmax=.96, color='red', linestyle='--', linewidth = fontsize/4)
# plt.text(0.8,1 - 0.48 + 0.02,'1 - prob(host is speaking) = 1 - 0.48', fontsize=40)

# Make legend fontsize larger
_ = plt.setp(legend.get_title(),fontsize=2*fontsize)

_ = plt.xlabel("Seconds in the future", fontsize = 2.2*fontsize )
# _ = plt.title("Comparison of Machine vs. Human Baselines", fontsize = 2.2*fontsize )
    
if savefig:
    plt.savefig("test_set_human_predictions.pdf", dpi=600, pad_inches=0, bbox_inches = 'tight')
plt.show()<jupyter_output><empty_output><jupyter_text>540 * 6<jupyter_code>results.head()
# In a small-scale human validation experiment, two human reviewers analyzed 5 minute segments across six videos. 
# The human reviewers. 
# Each video was reviewed once for only one modality to mitigate multiple exposure bias.<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16137663467005586, 'fraction_numerical': 0.029666544031572963, 'mean_word_length': 3.7847655624899983, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '657239', 'license': 'MIT', 'n_tokens_mistral': 11176, 'n_tokens_neox': 10361, 'n_words': 2410, 'path': 'paper_experiments_work_log/turntaking/human_predictions/analyze_human_predictions.ipynb', 'repo_name': 'cgnorthcutt/EgoCom-Dataset', 'size': '29899'}