<jupyter_start><jupyter_text>Gradually trying bigger and bigger kernels<jupyter_code>def is_prod_kernel(kernel):
    return type(kernel) == GPy.kern.src.prod.Prod

def is_add_kernel(kernel):
    return type(kernel) == GPy.kern.src.add.Add

def kernel_to_string(kernel, ident = 0):
    if kernel is None:
        return ""
    s = ""
    tab = ident * " "
    if is_prod_kernel(kernel) or is_add_kernel(kernel):
        op = "*" if is_prod_kernel(kernel) else "+"
        sub_kernels = []
        for sub_kernel in kernel.parameters:
            sub_kernels.append(kernel_to_string(sub_kernel, ident = ident + 1))
        s = "(" + op + "\n" + "\n".join(sub_kernels) + "\n" + tab + ")"
    else:
        # get name of kernel without "'>" characters
        name = str(type(kernel)).split(".")[-1]
        name = name[:-2]

        values = ",".join(["{:5f}".format(x) for x in kernel])
        s = name + "(" + values + ")"
    return " " * ident + s

def get_total_variance(kernel):
    ans = 0
    if is_prod_kernel(kernel) or is_add_kernel(kernel):
        for sub_kernel in kernel.parameters:
            ans += get_total_variance(sub_kernel)
    else:
        try:
            if hasattr(kernel, "variance"):
                ans += kernel.variance
            elif hasattr(kernel, "variances"):
                ans += kernel.variances
        except:
            # some kernels don't have variance as parameter
            ans += 0
            
    return ans
        
# function to get a measure of performance of kernel
# marginal_log_likelihood - alpha * var
def get_score(emu, alpha = 5):
    marg_log_likelihood = emu.get_log_marginal_likelihood()
    kern = emu.get_trained_kern()
    var = get_total_variance(kern)
    return marg_log_likelihood - alpha * var
n_parameters = emu.n_parameters()

base_kerns = [GPy.kern.RBF(n_parameters), 
              GPy.kern.RatQuad(n_parameters),
              GPy.kern.Linear(n_parameters),
              GPy.kern.PeriodicExponential(1),
              GPy.kern.PeriodicMatern52(1),
             ]

max_depth = 10
prev_kern = None
#time_limit?
#objective 
optimizer = 'lfgs'
#perform initial kernel selection
max_score = -1000
max_kern = None

for kern in base_kerns:
    emu.set_parameters(kernel = kern)
    emu.fit(normalizer = True, messages = False)
    score = get_score(emu)
    
    if score > max_score:
        print('here')
        max_score = score
        max_kern = emu.get_trained_kern()
        
    print(emu.get_trained_kern())
    print('Marginal log likelihood - alpha * var: ', score)

emu.set_parameters(kernel = max_kern)
emu.fit(normalizer = True)
emu.get_gp()
depth = 1
prev_max_kern = max_kern
while depth < max_depth:
    print("-" * 10 + "Depth " + str(depth) + '-'*10)
    
    max_score = -1000
    for op in [operator.add, operator.mul]:
        for kern in base_kerns:
            current_kern = op(prev_max_kern, kern)
            emu.set_parameters(kernel = current_kern)
            emu.fit(normalizer = True, messages = False)
            score = get_score(emu)
    
            if score > max_score:
                max_kern = emu.get_trained_kern()
                max_score = score
            
    prev_max_kern = max_kern
    print(max_kern)
    print(max_score)
    
    depth += 1
get_total_variance(prev_max_kern)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11184407796101949, 'fraction_numerical': 0.008395802098950524, 'mean_word_length': 2.3936927772126144, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '583419', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 1123, 'n_tokens_neox': 1084, 'n_words': 241, 'path': 'studies/legacy/automatic-kernel-selection.ipynb', 'repo_name': 'MaksymPetyak/emupints', 'size': '3335'}