<jupyter_start><jupyter_text>Bonus Demonstration: Cliff WalkingInspired by Gaskett et al. (2003), but not ultimately reported in the manuscript.<jupyter_code>import os
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.gridspec as gridspec
import seaborn as sns
from sisyphus.envs import CliffWalking
from mpl_toolkits.axes_grid1 import make_axes_locatable
sns.set_style('white')
sns.set_context('notebook', font_scale=1.5)
img_dir = os.path.join('..','figures')
%matplotlib inline<jupyter_output><empty_output><jupyter_text>Value Iteration<jupyter_code>from sisyphus.mdp import ValueIteration
from matplotlib.colors import LinearSegmentedColormap
from palettable.cmocean.sequential import Amp_20_r as palette

#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
### Define parameters.
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~#

## Simulation parameters.
weights = [1.0, 0.8, 0.6]

#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
### Value iteration.
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~#

## Initialize environment.
gym = CliffWalking()

## Preallocate space.
V = np.zeros((3,*gym.shape))
pi = []

for i, w in enumerate(weights):
    
    ## Initialize value iteration.
    qvi = ValueIteration(policy='pessimism', gamma=1, w=w)
    
    ## Solve for Q-values.
    qvi = qvi.fit(gym, verbose=False)
    V[i] = qvi.V.reshape(gym.shape)
    
    ## Solve for policy.
    pi.append( qvi.pi )
    
## Fill in terminal states.
V[:,-1, 1:] = -100
V[:,-1, -1] = 0

#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
### Plotting.
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
    
## Initialize figure.
fig = plt.figure(figsize=(16,4))
colors = palette.hex_colors
colors.insert(0,'k')
cmap = LinearSegmentedColormap.from_list('Amp_21_r', colors)

## Initialize plots and divider.
gs = gridspec.GridSpec(1,4)
gs.update(left=0.01, right=0.94, bottom=0.04, top=0.9, hspace=0, wspace=0.02)

## Plot environment.
annot_kws = annot_kws=dict(fontsize=18,fontweight='semibold',color='k')
ax = gym.plot_cliff(annot_kws=annot_kws, ax=plt.subplot(gs[0]))
ax.text(0,0,'A',ha='left',va='bottom',fontsize=28,fontweight='bold')

## Scale values for clarity.
Vscale = -np.log(np.abs(V)+1)

## Iteratively plot.
for i, (w, title) in enumerate(zip(weights, ['B','C','D'])):
    
    ## Initialize canvas.
    ax = plt.subplot(gs[i+1])
    
    if i < 2: 
        
        ## Plot heatmap.
        sns.heatmap(Vscale[i], cmap=cmap, vmin=-4.61, vmax=0, cbar=False, ax=ax)
        
    else:
        
        ## Divide canvas for colorbar.
        divider = make_axes_locatable(ax)
        cax = divider.append_axes("right", size="5%", pad=0.05)
        
        ## Plot heatmap.
        ax = sns.heatmap(Vscale[i], vmin=-4.61, vmax=0, cmap=cmap, cbar=True, ax=ax, cbar_ax=cax)
        
        ## Update heatmap ticklabels.
        cbar = ax.collections[0].colorbar
        cbar.set_ticks(-np.linspace(0,4.61,5))
        ticklabels = -np.exp(np.linspace(0,4.61,5)).astype(int)
        ticklabels[0] = 0
        cbar.set_ticklabels(ticklabels)
    
    ## Plot policy.
    ax = gym.plot_policy(ax, pi[i])
    
    ## Clean up image.
    ax.set(xticklabels=[], yticklabels=[])
    ax.text(0,0,title,ha='left',va='bottom',fontsize=28,fontweight='bold')
    
## Update colobar.
cbar = ax.collections[0].colorbar
cbar.set_label(r'$V(s)$', fontsize=20)
    
plt.savefig(os.path.join(img_dir, 'XX_cliff.png'), dpi=180)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.21978984238178634, 'fraction_numerical': 0.02714535901926445, 'mean_word_length': 4.1073025335320414, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '246028', 'license': 'MIT', 'n_tokens_mistral': 1323, 'n_tokens_neox': 1252, 'n_words': 229, 'path': 'notebooks/06_CliffWalking.ipynb', 'repo_name': 'ndawlab/seqanx', 'size': '3426'}