<jupyter_start><jupyter_text>Setup parameters for weight adaption session (results are only optimal for the used dct_dim)<jupyter_code>#There are the relevant hyper-parameters for weight adaption session (must be the same as in create_training_paths):
dct_dim = 32
min_img_dims = 128
#Experimental switches:
use_pil_rz = False #uses PIL LANCZOS for downsampling instead of opencv INTER_AREA  
num_threads = 16 #use multi-core procedures with this many cores where possible
import numpy as np
#see create_training_paths for creation of the precalculated dcts; 
#due to copyright these may present an issue when uploaded/clash with the open license -> you have to repeat the script yourself
precalc_dcts = np.load('ordered_dct_balanced.npz')
orig_dct, all_bu = precalc_dcts['dcts'], precalc_dcts['paths']
from naphash_py import naphash_obj as nhcpp, rot_inv_type
orig_dct1 = orig_dct[:]
init_weights = np.ones(324,np.float32)
h_crop0 = nhcpp(dct_dim=dct_dim, rot_inv_mode=rot_inv_type.none, apply_center_crop=False, is_rgb=False)
h_crop0.get_norm(init_weights)
print(init_weights)
from tqdm.notebook import tqdm
#order indices roughly according to entropy/likelyhood to change under minor transformations
def pack_hash_ids(max_l):
    all_mixes = []
    for l in range(max_l):
        for j in range((l+1)//2 -1,-1,-1):
            all_mixes+=[(j,l-j),(l-j,j)]
        if l % 4 == 3:
            all_mixes+=[(l//2,l//2),(l//2+1,l//2+1)]
    return all_mixes
def pack_hash_ids2(max_l):
    all_mixes = []
    for l in range(max_l):
        for j in range((l+1)//2 -1,-1,-1):
            all_mixes+=[(j,l-j),(l-j,j)]
    return all_mixes

sym_idx128 = pack_hash_ids2(23)
nap_hash_ids24 = pack_hash_ids(24)
nap_hash_ids25 = pack_hash_ids(25)

def concat_rot_n1(dct, normi):
    dctb = [dct[i,j]*normi[idx] for idx, (i,j) in enumerate(nap_hash_ids25)]
    return np.array(dctb)
res_norm, nums_norm = [], []

for inp_vers in [orig_dct]:#, sqr_dct1]:
    thr_f = 206/256 # or 230/256 for NAPHASH
    result_weights = init_weights[:]
    #result_weights = unify_wheights(np.array(norm_sqeq0),sym_idx128) # for NAPHASH necessary; skip for NPHASH
    c0 = result_weights
    for _ in tqdm(range(2)):
        nums_ro2=[]
        for d in inp_vers:
            c = concat_rot_n1(np.abs(d.reshape(32,32)),result_weights)
            nums_ro2.append((c > thr_f*np.mean(c[:64]))*1.0) #206/256
        nums_ro2 = np.vstack(np.array(nums_ro2))
        c0 = np.reciprocal(np.mean(nums_ro2,axis=0))*0.5
        #c0 = unify_wheights(c0,sym_idx128) # for NAPHASH necessary; skip for NPHASH
        result_weights = np.power(c0,0.75)*result_weights
        #result_weights[274:] = np.power(c0[274:],0.75)*result_weights[274:] #idea3: limiter to fixed indices
        print(c0[16], result_weights[16], np.mean(np.mean(nums_ro2,axis=0)[:64]), np.mean(np.mean(nums_ro2,axis=0)[-64:]))
    res_norm.append(result_weights)
    nums_norm.append(nums_ro2)
init_weights
res_norm[-1].astype(np.float32)
h_crop0.set_norm(res_norm[-1].astype(np.float32), True)
check_weights = np.ones(324,np.float32)
h_crop0.get_norm(check_weights)
#check difference from changing weights
check_weights-init_weights
len(orig_dct), len(all_bu)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12383612662942271, 'fraction_numerical': 0.0446927374301676, 'mean_word_length': 4.5, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '39520', 'license': 'MIT', 'n_tokens_mistral': 1337, 'n_tokens_neox': 1246, 'n_words': 262, 'path': 'experiments/adapt_weights.ipynb', 'repo_name': 'ozendelait/naphash', 'size': '3222'}