<jupyter_start><jupyter_text>Sanity Check: When Force_left = 0, contrastive FQI and FQI should perform the same<jupyter_code>bg_successes = []
fg_successes = []
for i in range(2):
    print(str(i))
    printed_bg, printed_fg, performance, nfq_agent = run(verbose=True, is_contrastive=True, evaluations=1, force_left=0)
    bg_successes.append(success_bg)
    fg_successes.append(success_fg)
    print("BG Succeeded: " + str(printed_bg))
    print("FG Succeeded: " + str(printed_fg))
    
plt.title("Comparing BG and FG success for CFQI when force_left=0")
sns.stripplot(x=bg_successes, label='Background', color='blue')
sns.stripplot(x=fg_successes, label='Foreground', color='red')
plt.xlabel("# of successful runs (out of 10)")
plt.legend()
plt.show()
nfq_agent._nfq_net.layers_fg[2].weight
plt.title("Comparing BG and FG success for CFQI when force_left=0")
sns.stripplot(x=bg_successes, label='Background', color='blue')
sns.stripplot(x=fg_successes, label='Foreground', color='red')
plt.xlabel("# of successful runs (out of 10)")
plt.legend()
plt.show()
cfqi_success = []
fqi_success = []
for i in range(10):
    print(str(i))
    printed_bg, printed_fg, performance, nfq_agent = run(verbose=False, is_contrastive=True, evaluations=10, force_left=0)
    cfqi_success.extend(performance)
    printed_bg, printed_fg, performance, nfq_agent = run(verbose=False, is_contrastive=False, evaluations=10, force_left=0)
    fqi_success.extend(performance)
sns.stripplot(x=cfqi_success, label='CFQI', color='blue')
sns.stripplot(x=fqi_success, label='FQI', color='red')
plt.title("Force left = 0, CFQI and FQI performance")
plt.xlabel("Number of steps eval survived (out of 1000)")
plt.legend()<jupyter_output><empty_output><jupyter_text>Force_left vs success<jupyter_code>import numpy as np
import scipy.stats


def mean_confidence_interval(data, confidence=0.95):
    a = 1.0 * np.array(data)
    n = len(a)
    m, se = np.mean(a), scipy.stats.sem(a)
    h = se * scipy.stats.t.ppf((1 + confidence) / 2., n-1)
    return m, h
# Mean of success and confidence intervals
c_success = []
f_success = []
c_errs = []
f_errs = []
# Looping over force left
for i in range(10):
    cfqi_success = []
    fqi_success = []
    # Iterations at this force left
    for j in range(10):
        print(str(i) + " " + str(j))
        printed_bg, printed_fg, performance, nfq_agent = run(verbose=False, is_contrastive=True, evaluations=2, force_left=i)
        cfqi_success.extend(performance)
        printed_bg, printed_fg, performance, nfq_agent = run(verbose=False, is_contrastive=False, evaluations=2, force_left=i)
        fqi_success.extend(performance)

    c_success.append(np.mean(cfqi_success))
    f_success.append(np.mean(fqi_success))
    m, h = mean_confidence_interval(cfqi_success)
    c_errs.append(h)
    m, h = mean_confidence_interval(fqi_success)
    f_errs.append(h)
    
    x = [k for k in range(i+1)]
    sns.scatterplot(x, c_success, label='CFQI')
    plt.errorbar(x, c_success ,yerr=c_errs, linestyle="None")
    sns.scatterplot(x, f_success, label='FQI')
    plt.errorbar(x, f_success ,yerr=f_errs, linestyle="None")
    plt.title("Performance of CFQI and FQI when force on cart is modified")
    plt.xlabel("Force Left")
    plt.ylabel("Average steps the cartpole environment runs for")
    plt.show()<jupyter_output><empty_output><jupyter_text>Interpretability<jupyter_code># Foreground group
heatmap = []
for pos in range(-10, 11, 1):
    pos /= 10
    pos_values = []
    vel = 0
    ang = -4
    p_vel = 0
    state = np.asarray([pos, vel, ang, p_vel])
    best_action = nfq_agent.get_best_action(state, np.array([0, 1]), 1)
    pos_values.append(best_action)
    
    heatmap.append(pos_values)
    
heatmap = np.asarray(heatmap)
xticklabels = [x/10 for x in range(-10, 10, 1)]
yticklabels = [-4]
ax = plt.figure(figsize=(10, 2))
sns.heatmap(heatmap.T, xticklabels=xticklabels, yticklabels=yticklabels)
plt.title("Foreground")
plt.xlabel("Cart Position")
plt.ylabel("Pole Angle")
# Background group
heatmap = []
for pos in range(-10, 11, 1):
    pos /= 10
    pos_values = []
    vel = 0
    ang = -4
    p_vel = 0
    state = np.asarray([pos, vel, ang, p_vel])
    best_action = nfq_agent.get_best_action(state, np.array([0, 1]), 0)
    pos_values.append(best_action)
    
    heatmap.append(pos_values)
heatmap = np.asarray(heatmap)
xticklabels = [x/10 for x in range(-10, 10, 1)]
yticklabels = [-4]
ax = plt.figure(figsize=(10, 2))
sns.heatmap(heatmap.T, xticklabels=xticklabels, yticklabels=yticklabels)
plt.title("Background")
plt.xlabel("Cart Position")
plt.ylabel("Pole Angle")
nfq_agent._nfq_net.layers_fg
nc_success = []
c_success = []
for i in range(10):
    print(str(i))
    success, total = run(verbose=False, is_contrastive=False, evaluations=5)
    nc_success.append(success/total)
    
    success, total = run(verbose=False, is_contrastive=True, evaluations=5)
    c_success.append(success/total)
ax = sns.boxplot(x=nc_success)
sns.swarmplot(nc_success, label='FQI', color='.25')
ax = sns.boxplot(x=c_success)
sns.swarmplot(c_success, label='CFQI', color='.25')
plt.ylabel("Percentage of evaluations that were successful")
plt.boxplot(x=[nc_success, c_success], labels=['FQI', 'CFQI'])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1402973694243233, 'fraction_numerical': 0.018109035455585207, 'mean_word_length': 4.8364849833147945, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '104158', 'license': 'MIT', 'n_tokens_mistral': 2089, 'n_tokens_neox': 1943, 'n_words': 405, 'path': 'simulated_fqi/notebooks/asa_experiments.ipynb', 'repo_name': 'bee-hive/nested-policy-rl', 'size': '5246'}