<jupyter_start><jupyter_text>Instances features statistics<jupyter_code>import numpy as np
from tqdm.notebook import tqdm
import pickle
%load_ext autoreload
%autoreload 2
import sys
# insert at 1, 0 is the script path (or '' in REPL)
sys.path.insert(1, '../models')
sys.path.insert(0, '../')

from batch_env_rl import BatchEnvRL
from generator.op.generator_utils import get_generated_seeds
gen_seeds = get_generated_seeds()
#to consider only up to seed 4000 for all n_nodes
gen_seeds[20] = np.array([s for s in gen_seeds.get(20) if s<=4000]) 
feature_list = ['x_coordinate',
                'y_coordinate',
                'tw_low',
                'tw_high',
                'prize',
                'tmax',
                'tw_delta',
                'prize_tw_delta_ratio',
                'tw_high_tmax_delta',
                'tw_low_tmax_delta',
                'prize_max_return_time_ratio']
# placeholders
psum   = np.zeros(len(feature_list))
psumsq = np.zeros(len(feature_list))

# loop through instances
count = 0
for n_nodes in tqdm(range(10, 211)):
    for seed in tqdm(gen_seeds[n_nodes], leave=False):

        env = BatchEnvRL(n_envs=1, from_file=True, 
            x_path=f'../data/generated/n_nodes_{n_nodes}/instances/instance_nnodes_{n_nodes}_seed_{seed}.csv', 
            adj_path=f'../data/generated/n_nodes_{n_nodes}/adjs/adj-instance_nnodes_{n_nodes}_seed_{seed}.csv',
            noise_on=False)

        inst_features, _ = env.get_our_features(feature_list)

        psum   += inst_features[0].sum(axis=0)
        psumsq += (inst_features[0] ** 2).sum(axis=0)
        count += inst_features[0].shape[0]
####### FINAL CALCULATIONS

# mean and std
total_mean = psum / count
total_var  = (psumsq / count) - (total_mean ** 2)
total_std  = np.sqrt(total_var)

# output
print('mean: '  + str(total_mean))
print('std:  '  + str(total_std))
normalizations_stats = {}
for k, (mu, std) in enumerate (zip(total_mean,total_std)):
    normalizations_stats[feature_list[k]]= {'mean': mu , 'std': std}
normalizations_stats
normalizations_stats_file = open("../data/generated/normalizations_stats.pkl", "wb")
pickle.dump(normalizations_stats, normalizations_stats_file)
normalizations_stats_file.close()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1275078020508248, 'fraction_numerical': 0.014266607222469906, 'mean_word_length': 3.524193548387097, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '467184', 'license': 'MIT', 'n_tokens_mistral': 821, 'n_tokens_neox': 775, 'n_words': 147, 'path': 'notebooks/1.0.0-rg-instances-stats.ipynb', 'repo_name': 'mustelideos/td-opswtw-competition-rl', 'size': '2243'}