<jupyter_start><jupyter_text>Simple Bandit<jupyter_code>mac = pickle.load(open(os.path.join('output', 'mac_expectations.pkl'), 'rb'))
win = pickle.load(open(os.path.join('output', 'win_expectations.pkl'), 'rb'))
sgm = pickle.load(open(os.path.join('output', 'sgm_expectations.pkl'), 'rb'))
rh = pickle.load(open(os.path.join('output', 'rh_expectations.pkl'), 'rb'))
dar = pickle.load(open(os.path.join('output', 'dar_expectations.pkl'), 'rb'))
responses = pd.read_csv('simulated_data.csv')
test = responses[responses['set']=='test']
test.head()
test = test[['arm', 'reward']]
test['user'] = [i for i in range(test.shape[0])]
test.reset_index(inplace=True, drop=True)
mac_df = pd.DataFrame(mac)
win_df = pd.DataFrame(win)
sgm_df = pd.DataFrame(sgm)
rh_df = pd.DataFrame(rh)
dar_df = pd.DataFrame(dar)

mac_df = mac_df.merge(test, how='left', left_index=True, right_index=True)
win_df = win_df.merge(test, how='left', left_index=True, right_index=True)
sgm_df = sgm_df.merge(test, how='left', left_index=True, right_index=True)
rh_df = rh_df.merge(test, how='left', left_index=True, right_index=True)
dar_df = dar_df.merge(test, how='left', left_index=True, right_index=True)

mac_df.head()
mac_df = mac_df.melt(id_vars=['user', 'arm', 'reward'], var_name='item_id', value_name='raw_score')
win_df = win_df.melt(id_vars=['user', 'arm', 'reward'], var_name='item_id', value_name='raw_score')
sgm_df = sgm_df.melt(id_vars=['user', 'arm', 'reward'], var_name='item_id', value_name='raw_score')
rh_df = rh_df.melt(id_vars=['user', 'arm', 'reward'], var_name='item_id', value_name='raw_score')
dar_df = dar_df.melt(id_vars=['user', 'arm', 'reward'], var_name='item_id', value_name='raw_score')
mac_df.head()
mac_df['probability'] = mac_df.apply(lambda x: 1/(1 + np.exp(-x['raw_score'])), axis=1)
mac_df.head()
mac_df['probability'].describe()
win_df['probability'] = win_df.apply(lambda x: 1/(1 + np.exp(-x['raw_score'])), axis=1)
sgm_df['probability'] = sgm_df.apply(lambda x: 1/(1 + np.exp(-x['raw_score'])), axis=1)
rh_df['probability'] = rh_df.apply(lambda x: 1/(1 + np.exp(-x['raw_score'])), axis=1)
dar_df['probability'] = dar_df.apply(lambda x: 1/(1 + np.exp(-x['raw_score'])), axis=1)
mac_df['pred'] = mac_df['probability'] >= 0.5
win_df['pred'] = win_df['probability'] >= 0.5
sgm_df['pred'] = sgm_df['probability'] >= 0.5
rh_df['pred'] = rh_df['probability'] >= 0.5
dar_df['pred'] = dar_df['probability'] >= 0.5
mac_df.head()<jupyter_output><empty_output><jupyter_text>Table 6, Clicks Column<jupyter_code>mac_df['pred'].sum()
win_df['pred'].sum()
sgm_df['pred'].sum()
rh_df['pred'].sum()
dar_df['pred'].sum()
mac_df.shape[0]<jupyter_output><empty_output><jupyter_text>Table 6, Accuracy, Precision, Recall, and AUC Columns<jupyter_code>scores = [('Mac', mac_df), ('Windows', win_df), ('SageMaker', sgm_df), ('Red Hat', rh_df), ('Darwin', dar_df)]
for name, df in scores:

    temp_idx = df['arm'] == df['item_id']
    temp = df[temp_idx]
    acc = BinaryClassificationMetrics.Accuracy().get_score(temp['pred'].tolist(), temp['reward'].tolist())
    f1 = BinaryClassificationMetrics.F1().get_score(temp['pred'].tolist(), temp['reward'].tolist())
    precision = BinaryClassificationMetrics.Precision().get_score(temp['pred'].tolist(), temp['reward'].tolist())
    recall = BinaryClassificationMetrics.Recall().get_score(temp['pred'].tolist(), temp['reward'].tolist())
    auc = BinaryClassificationMetrics.AUC().get_score(temp['pred'].tolist(), temp['reward'].tolist())
    print(name, ':', acc, f1, precision, recall, auc)
df_full = None

for name, df in scores:
    df_temp = df[['user', 'item_id', 'raw_score', 'probability', 'pred']]
    df_temp.columns = [name + c if c not in ['user', 'item_id'] else c for c in df_temp.columns]
    
    if df_full is None:
        df_full = df_temp
    else:
        df_full = df_full.merge(df_temp, how='left', on=['user', 'item_id'])
df_full.head()<jupyter_output><empty_output><jupyter_text>Table 5, Score Comparisons<jupyter_code>results = []
env = ['Mac', 'Windows', 'SageMaker', 'Red Hat', 'Darwin']

for i, name_i in enumerate(env):
    a = name_i + 'raw_score'
    for j, name_j in enumerate(env):
        if j > i:
            b = name_j + 'raw_score'
            similarity = np.isclose(df_full[a], df_full[b])
            results.append((name_i, name_j, similarity))
for a, b, c in results:
    print(a, b, c.shape, str(c.sum()))<jupyter_output>Mac Windows (480,) 129
Mac SageMaker (480,) 130
Mac Red Hat (480,) 480
Mac Darwin (480,) 64
Windows SageMaker (480,) 214
Windows Red Hat (480,) 129
Windows Darwin (480,) 64
SageMaker Red Hat (480,) 130
SageMaker Darwin (480,) 64
Red Hat Darwin (480,) 64<jupyter_text>Table 5, Probability Comparisons<jupyter_code>results2 = []
env = ['Mac', 'Windows', 'SageMaker', 'Red Hat', 'Darwin']

for i, name_i in enumerate(env):
    a = name_i + 'probability'
    for j, name_j in enumerate(env):
        if j > i:
            b = name_j + 'probability'
            similarity = np.isclose(df_full[a], df_full[b])
            results2.append((name_i, name_j, similarity))
            
for a, b, c in results2:
    print(a, b, c.shape, str(c.sum()))<jupyter_output>Mac Windows (480,) 162
Mac SageMaker (480,) 168
Mac Red Hat (480,) 480
Mac Darwin (480,) 69
Windows SageMaker (480,) 238
Windows Red Hat (480,) 162
Windows Darwin (480,) 69
SageMaker Red Hat (480,) 168
SageMaker Darwin (480,) 69
Red Hat Darwin (480,) 69<jupyter_text>Table 6, CTR, NDCG@1, NDCG@4 Columns<jupyter_code>for name, df in scores:
    actual = df[['user', 'arm', 'reward']].drop_duplicates()
    predicted = df[['user', 'item_id', 'probability']]
    predicted.columns = actual.columns
    ctr = BinaryRecoMetrics.CTR(user_id_column='user', item_id_column='arm', click_column="reward", k=1).get_score(actual, predicted)
    ndcg = RankingRecoMetrics.NDCG(user_id_column='user', item_id_column='arm', click_column="reward", k=4).get_score(actual, predicted)
    precision = RankingRecoMetrics.Precision(user_id_column='user', item_id_column='arm', click_column="reward", k=1).get_score(actual, predicted)
    recall = RankingRecoMetrics.Recall(user_id_column='user', item_id_column='arm', click_column="reward", k=1).get_score(actual, predicted)
    ndcg1 = RankingRecoMetrics.NDCG(user_id_column='user', item_id_column='arm', click_column="reward", k=1).get_score(actual, predicted)
    ctr4 = BinaryRecoMetrics.CTR(user_id_column='user', item_id_column='arm', click_column="reward", k=4).get_score(actual, predicted)

    print(name, ':', ctr, ndcg, precision, recall, ndcg1, ctr4)
temp = mac_df[['user', 'arm', 'reward']].drop_duplicates()
temp['reward'].mean()<jupyter_output><empty_output><jupyter_text>Table 5, Prediction Comparisons<jupyter_code>results3 = []
env = ['Mac', 'Windows', 'SageMaker', 'Red Hat', 'Darwin']

for i, name_i in enumerate(env):
    a = name_i + 'pred'
    for j, name_j in enumerate(env):
        if j > i:
            b = name_j + 'pred'
            predictions = df_full[a] == df_full[b]
            results3.append((name_i, name_j, predictions))
            
for a, b, c in results3:
    print(a, b, c.shape, str(c.sum()))<jupyter_output>Mac Windows (480,) 476
Mac SageMaker (480,) 480
Mac Red Hat (480,) 480
Mac Darwin (480,) 441
Windows SageMaker (480,) 476
Windows Red Hat (480,) 476
Windows Darwin (480,) 437
SageMaker Red Hat (480,) 480
SageMaker Darwin (480,) 441
Red Hat Darwin (480,) 441<jupyter_text>Cholesky<jupyter_code>mac = pickle.load(open(os.path.join('output', 'mac_ch_expectations.pkl'), 'rb'))
win = pickle.load(open(os.path.join('output', 'win_ch_expectations.pkl'), 'rb'))
sgm = pickle.load(open(os.path.join('output', 'sgm_ch_expectations.pkl'), 'rb'))
rh = pickle.load(open(os.path.join('output', 'rh_ch_expectations.pkl'), 'rb'))
dar = pickle.load(open(os.path.join('output', 'dar_ch_expectations.pkl'), 'rb'))
responses = pd.read_csv('simulated_data.csv')
test = responses[responses['set']=='test']
test.head()
test = test[['arm', 'reward']]
test['user'] = [i for i in range(test.shape[0])]
test.reset_index(inplace=True, drop=True)
mac_df = pd.DataFrame(mac)
win_df = pd.DataFrame(win)
sgm_df = pd.DataFrame(sgm)
rh_df = pd.DataFrame(rh)
dar_df = pd.DataFrame(dar)

mac_df = mac_df.merge(test, how='left', left_index=True, right_index=True)
win_df = win_df.merge(test, how='left', left_index=True, right_index=True)
sgm_df = sgm_df.merge(test, how='left', left_index=True, right_index=True)
rh_df = rh_df.merge(test, how='left', left_index=True, right_index=True)
dar_df = dar_df.merge(test, how='left', left_index=True, right_index=True)

mac_df.head()
mac_df = mac_df.melt(id_vars=['user', 'arm', 'reward'], var_name='item_id', value_name='raw_score')
win_df = win_df.melt(id_vars=['user', 'arm', 'reward'], var_name='item_id', value_name='raw_score')
sgm_df = sgm_df.melt(id_vars=['user', 'arm', 'reward'], var_name='item_id', value_name='raw_score')
rh_df = rh_df.melt(id_vars=['user', 'arm', 'reward'], var_name='item_id', value_name='raw_score')
dar_df = dar_df.melt(id_vars=['user', 'arm', 'reward'], var_name='item_id', value_name='raw_score')
mac_df['probability'] = mac_df.apply(lambda x: 1/(1 + np.exp(-x['raw_score'])), axis=1)
win_df['probability'] = win_df.apply(lambda x: 1/(1 + np.exp(-x['raw_score'])), axis=1)
sgm_df['probability'] = sgm_df.apply(lambda x: 1/(1 + np.exp(-x['raw_score'])), axis=1)
rh_df['probability'] = rh_df.apply(lambda x: 1/(1 + np.exp(-x['raw_score'])), axis=1)
dar_df['probability'] = dar_df.apply(lambda x: 1/(1 + np.exp(-x['raw_score'])), axis=1)
mac_df['pred'] = mac_df['probability'] >= 0.5
win_df['pred'] = win_df['probability'] >= 0.5
sgm_df['pred'] = sgm_df['probability'] >= 0.5
rh_df['pred'] = rh_df['probability'] >= 0.5
dar_df['pred'] = dar_df['probability'] >= 0.5<jupyter_output><empty_output><jupyter_text>Table 6, Clicks Column<jupyter_code>mac_df['pred'].sum()
win_df['pred'].sum()
sgm_df['pred'].sum()
rh_df['pred'].sum()
dar_df['pred'].sum()
scores = [('Mac', mac_df), ('Windows', win_df), ('SageMaker', sgm_df), ('Red Hat', rh_df), ('Darwin', dar_df)]

df_full = None

for name, df in scores:
    df_temp = df[['user', 'item_id', 'raw_score', 'probability', 'pred']]
    df_temp.columns = [name + c if c not in ['user', 'item_id'] else c for c in df_temp.columns]
    
    if df_full is None:
        df_full = df_temp
    else:
        df_full = df_full.merge(df_temp, how='left', on=['user', 'item_id'])
df_full.head()
results = []
env = ['Mac', 'Windows', 'SageMaker', 'Red Hat', 'Darwin']

for i, name_i in enumerate(env):
    a = name_i + 'raw_score'
    for j, name_j in enumerate(env):
        if j > i:
            b = name_j + 'raw_score'
            similarity = np.isclose(df_full[a], df_full[b])
            results.append((name_i, name_j, similarity))
            
            
for a, b, c in results:
    print(a, b, c.shape, str(c.sum()))<jupyter_output>Mac Windows (480,) 480
Mac SageMaker (480,) 480
Mac Red Hat (480,) 480
Mac Darwin (480,) 480
Windows SageMaker (480,) 480
Windows Red Hat (480,) 480
Windows Darwin (480,) 480
SageMaker Red Hat (480,) 480
SageMaker Darwin (480,) 480
Red Hat Darwin (480,) 480<jupyter_text>Table 6, Cholesky Row, CTR, NDCG@1, NDCG@4 Columns<jupyter_code>for name, df in scores:
    actual = df[['user', 'arm', 'reward']].drop_duplicates()
    predicted = df[['user', 'item_id', 'probability']]
    predicted.columns = actual.columns
    ctr = BinaryRecoMetrics.CTR(user_id_column='user', item_id_column='arm', click_column="reward", k=1).get_score(actual, predicted)
    ndcg = RankingRecoMetrics.NDCG(user_id_column='user', item_id_column='arm', click_column="reward", k=4).get_score(actual, predicted)
    precision = RankingRecoMetrics.Precision(user_id_column='user', item_id_column='arm', click_column="reward", k=1).get_score(actual, predicted)
    recall = RankingRecoMetrics.Recall(user_id_column='user', item_id_column='arm', click_column="reward", k=1).get_score(actual, predicted)
    ndcg1 = RankingRecoMetrics.NDCG(user_id_column='user', item_id_column='arm', click_column="reward", k=1).get_score(actual, predicted)
    ctr4 = BinaryRecoMetrics.CTR(user_id_column='user', item_id_column='arm', click_column="reward", k=4).get_score(actual, predicted)

    print(name, ':', ctr, ndcg, precision, recall, ndcg1, ctr4)<jupyter_output>Mac : 0.3888888888888889 0.6321075028884455 0.2413793103448276 0.2413793103448276 0.2413793103448276 0.48333333333333334
Windows : 0.3888888888888889 0.6321075028884455 0.2413793103448276 0.2413793103448276 0.2413793103448276 0.48333333333333334
SageMaker : 0.3888888888888889 0.6321075028884455 0.2413793103448276 0.2413793103448276 0.2413793103448276 0.48333333333333334
Red Hat : 0.3888888888888889 0.6321075028884455 0.2413793103448276 0.2413793103448276 0.2413793103448276 0.48333333333333334
Darwin : 0.3888888888888889 0.6321075028884455 0.2413793103448276 0.2413793103448276 0.2413793103448276 0.48333333333333334<jupyter_text>Table 6, Cholesky Row, Accuracy, Precision, Recall, and AUC Columns<jupyter_code>scores = [('Mac', mac_df), ('Windows', win_df), ('SageMaker', sgm_df), ('Red Hat', rh_df), ('Darwin', dar_df)]
for name, df in scores:

    temp_idx = df['arm'] == df['item_id']
    temp = df[temp_idx]
    acc = BinaryClassificationMetrics.Accuracy().get_score(temp['pred'].tolist(), temp['reward'].tolist())
    f1 = BinaryClassificationMetrics.F1().get_score(temp['pred'].tolist(), temp['reward'].tolist())
    precision = BinaryClassificationMetrics.Precision().get_score(temp['pred'].tolist(), temp['reward'].tolist())
    recall = BinaryClassificationMetrics.Recall().get_score(temp['pred'].tolist(), temp['reward'].tolist())
    auc = BinaryClassificationMetrics.AUC().get_score(temp['pred'].tolist(), temp['reward'].tolist())
    print(name, ':', acc, f1, precision, recall, auc)<jupyter_output>Mac : 0.425 0.448 0.4827586206896552 0.417910447761194 0.4259363559560687
Windows : 0.425 0.448 0.4827586206896552 0.417910447761194 0.4259363559560687
SageMaker : 0.425 0.448 0.4827586206896552 0.417910447761194 0.4259363559560687
Red Hat : 0.425 0.448 0.4827586206896552 0.417910447761194 0.4259363559560687
Darwin : 0.425 0.448 0.4827586206896552 0.417910447761194 0.4259363559560687
{'fraction_non_alphanumeric': 0.17870239774330043, 'fraction_numerical': 0.08004231311706629, 'mean_word_length': 5.92431640625, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 53, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 10, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '644849', 'license': 'Apache-2.0', 'n_tokens_mistral': 6302, 'n_tokens_neox': 5475, 'n_words': 1053, 'path': 'examples/lints_reproducibility/additional_experiments/synthetic_data/Analysis.ipynb', 'repo_name': 'ncilfone/mabwiser', 'size': '14180'}