<jupyter_start><jupyter_text>Policy heatmap, epsilon = 0.5<jupyter_code>qdf = pd.concat([pd.DataFrame.from_dict(qouts9[0], orient='index', columns=['random', 'probs1','probs2', 'smart', 'exclude']).sort_index(),
           pd.DataFrame.from_dict(qouts9[1], orient='index', columns=['random', 'probs1','probs2', 'smart', 'exclude']).sort_index(),
           pd.DataFrame.from_dict(qouts9[2], orient='index', columns=['random', 'probs1','probs2', 'smart', 'exclude']).sort_index()]).groupby(level=0).mean()
qdf
qdft = pd.DataFrame(MinMaxScaler().fit_transform(qdf.T).T, columns=qdf.columns, index=qdf.index)
qdft.drop(qdft.tail(1).index,inplace=True)

plt.rcParams['figure.figsize'] = 5,10
sns.set(font_scale=1.4)
ax = sns.heatmap(qdft, cmap='viridis')
#plt.title('Optimal Policy (Normalized)', fontsize=20)
plt.savefig('plots/policy_heatmap_0p5e.svg')<jupyter_output><empty_output><jupyter_text>Policy heatmap, epsilon = 0.02<jupyter_code>qdf = pd.concat([pd.DataFrame.from_dict(qouts8[0], orient='index', columns=['random', 'probs1','probs2', 'smart', 'exclude']).sort_index(),
           pd.DataFrame.from_dict(qouts8[1], orient='index', columns=['random', 'probs1','probs2', 'smart', 'exclude']).sort_index(),
           pd.DataFrame.from_dict(qouts8[2], orient='index', columns=['random', 'probs1','probs2', 'smart', 'exclude']).sort_index()]).groupby(level=0).mean()

qdft = pd.DataFrame(MinMaxScaler().fit_transform(qdf.T).T, columns=qdf.columns, index=qdf.index)
qdft.drop(qdft.tail(1).index,inplace=True)

plt.rcParams['figure.figsize'] = 5,10
sns.set(font_scale=1.4)
ax = sns.heatmap(qdft, cmap='viridis')
#plt.title('Optimal Policy (Normalized)', fontsize=20)
plt.savefig('plots/policy_heatmap_0p02e.svg')
#policy for 10k iter, total state
pd.concat([pd.DataFrame.from_dict(qouts5[0], orient='index', columns=['random', 'probs', 'smart', 'exclude']).sort_index(),
           pd.DataFrame.from_dict(qouts5[1], orient='index', columns=['random', 'probs', 'smart', 'exclude']).sort_index(),
           pd.DataFrame.from_dict(qouts5[2], orient='index', columns=['random', 'probs', 'smart', 'exclude']).sort_index()]).groupby(level=0).mean()
## count of times hitting on first or second guess
print('num wins on first try:', len(np.where(souts8[0].episode_lengths+1==1)[0]))
print('num wins on second try:', len(np.where(souts8[0].episode_lengths+1==2)[0]))
print('num wins on third try:', len(np.where(souts8[0].episode_lengths+1==3)[0]))

import seaborn as sns
plt.rcParams['figure.figsize']=7,5
plt.style.use('seaborn-white')
#plt.rcParams['axes.facecolor']='white'
#plt.rcParams["axes.edgecolor"] = "black"
#plt.rcParams["axes.linewidth"] = 2
labels, counts = np.unique(souts8[0].episode_lengths+1, return_counts=True)
plt.bar(labels, counts, align='center', color='steelblue')

#plt.hist(s1.episode_lengths+1, bins=10, align='mid')
#plt.title('10,000 trials', fontsize=24)
plt.xlabel('game length', fontsize=24)
plt.ylabel('counts', fontsize=24)
plt.gca().set_xticks(labels)
plt.xticks(fontsize=24)
plt.yticks(fontsize=24)

plt.savefig('plots/game_lengths_10k_v2.svg', bbox_inches='tight')
tmphist = plt.hist(souts8[0].episode_rewards, color='steelblue')
plt.xticks(fontsize=24)
plt.yticks(fontsize=24)
plt.ylabel('count', fontsize=24)
plt.xlabel('reward', fontsize=24)
#plt.title('10,000 trials', fontsize=24)
plt.savefig('plots/rewards-distro_10k_v2.svg', bbox_inches='tight')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.20690664325431665, 'fraction_numerical': 0.036289142522680715, 'mean_word_length': 7.994736842105263, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '263274', 'license': 'MIT', 'n_tokens_mistral': 1426, 'n_tokens_neox': 1337, 'n_words': 176, 'path': 'notebooks/wordl-reinforcement-learning-v3.ipynb', 'repo_name': 'benton-anderson/wordle-opt', 'size': '3417'}