<jupyter_start><jupyter_text>Consistent Greedy solved 1173 out of 3000 instances  Generate DF with [greedy_actions]<jupyter_code>filename = 'data_consistent_greedy_paper.csv'
data = pd.read_csv(filename)
data = data.loc[data.instance_id.isin(selected_instances)][['instance_id', 'step', 'strtype', 'action']].copy()

count = 0
for instanceid in data.instance_id.unique():
    select_ = data.loc[data.instance_id == instanceid].sort('step').reset_index(drop = True).copy()   
    if count == 0:
        new_data = select_.copy()
    else: 
        new_data = new_data.append(select_)
   
    count = count + 1
new_data = new_data.reset_index(drop=True)
new_data.loc[new_data.step==8]
len(new_data.instance_id.unique())<jupyter_output><empty_output><jupyter_text>now look at entropy measure<jupyter_code>import scipy.stats as sc

# Input a pandas series 
def ent(data):
    p_data= data.value_counts()/len(data) # calculates the probabilities
    entropy=sc.entropy(p_data)  # input probabilities to get the entropy 
    return entropy

records = []
for type_ in new_data.strtype.unique():
    for step in new_data.step.unique():
        slice_ = new_data.loc[new_data.strtype == type_].loc[new_data.step == step].copy()
        actions_greedy = slice_.action
        ent_greedy = ent(actions_greedy)

        records.append({'type_': type_, 'step': step, 'ent_greedy': ent_greedy})
ent_data = pd.DataFrame.from_records(records)
ent_data['composition'] = ent_data.apply(lambda row: row.type_.replace('14','SLT4').replace('16', 'SLT6').replace('24','VIRM4').replace('26','VIRM6'), axis =1 )

ent_data.head(100)
import seaborn as sns
import matplotlib.pyplot as plt

from matplotlib import rcParams
rcParams.update({'figure.autolayout': True})

%matplotlib inline  

max_ = 2.20 #max(max(ent_data.ent_ej), max(ent_data.ent_roel), max(ent_data.ent_random))
min_ = 0 #min(min(ent_data.ent_ej), min(ent_data.ent_roel), min(ent_data.ent_random))

#entropies Greedy
plt.figure()
sns.heatmap(ent_data.pivot(index='step', columns='composition', values='ent_greedy'), vmin=min_, vmax=max_).set_title('Entropy Greedy')
plt.xticks(rotation=45, ha='right')
sns.plt.savefig('entropy_consistent_greedy.eps')<jupyter_output>/Users/evertjanpeer/anaconda/lib/python3.5/site-packages/matplotlib/figure.py:1742: UserWarning: This figure includes Axes that are not compatible with tight_layout, so its results might be incorrect.
  warnings.warn("This figure includes Axes that are not "<jupyter_text>entropies on train type level<jupyter_code># Input a pandas series 
def ent(data):
    p_data= data.value_counts()/len(data) # calculates the probabilities
    entropy=sc.entropy(p_data)  # input probabilities to get the entropy 
    return entropy

records = []
for type_ in new_data.strtype.unique():
    actions_greedy = new_data.loc[new_data.strtype == type_].action.copy()
    ent_greedy = ent(actions_greedy)

    records.append({'type_': type_, 'algo': 'Greedy', 'ent': ent_greedy})
    
ent_data = pd.DataFrame.from_records(records)
ent_data['composition'] = ent_data.apply(lambda row: row.type_.replace('14','SLT4').replace('16', 'SLT6').replace('24','VIRM4').replace('26','VIRM6'), axis =1 )
ent_data.head()
ent_data
result_table = ent_data.pivot(index='algo', columns='composition', values='ent').round(2)
result_table.to_latex()
result_table<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1407319250223148, 'fraction_numerical': 0.01606664683130021, 'mean_word_length': 5.724, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '609492', 'license': 'MIT', 'n_tokens_mistral': 1239, 'n_tokens_neox': 1174, 'n_words': 225, 'path': 'train_algo_no_reloc/analysis/paper_test_compute_consistent_greedy_entropies.ipynb', 'repo_name': 'hsinyic/tusp-archive', 'size': '3361'}