<jupyter_start><jupyter_text>Genetic search approach<jupyter_code>import numpy as np
np.random.seed(420)
train_X = np.load('../data/train_X.npy')
train_y = np.load('../data/train_y.npy')
valid_X = np.load('../data/valid_X.npy')
valid_y = np.load('../data/valid_y.npy')
N = 1500  # dataset size
T = 16  # number of datasets per generation
M = 0.2  # fraction of dataset samples dropped during mutation phase
from typing import List, Tuple
from dataclasses import dataclass

@dataclass
class DataSet(object):
    X: np.array
    y: np.array
    ids: np.array
    
    @classmethod
    def combine(cls, ds1, ds2):
        assert(ds1.X.shape == ds2.X.shape)
        intersection = np.intersect1d(ds1.ids, ds2.ids)
        other = np.random.choice(
            np.setxor1d(ds1.ids, ds2.ids), len(ds1.ids)-len(intersection)
        )
        new_ids = np.concatenate([intersection, other])
        take_from_left = np.isin(ds1.ids, new_ids)
        take_from_right = np.isin(ds2.ids, new_ids)
        # new ids need to be re-ordered to match the order 
        # in which X and y are selected for the new DataSet:
        new_ids = np.select(
            [take_from_left, take_from_right],
            [ds1.ids, ds2.ids]
        )
        new_X = np.select(
            [
                np.repeat(take_from_left, ds1.X.shape[1]).reshape(ds1.X.shape),
                np.repeat(take_from_right, ds2.X.shape[1]).reshape(ds2.X.shape)
            ],
            [ds1.X, ds2.X]
        )
        new_y = np.select(
            [take_from_left, take_from_right],
            [ds1.y, ds2.y]
        )
        return cls(new_X, new_y, new_ids)
        

def sample_datasets(
        n_samples: int=N, 
        n_datasets: int=T, 
        n_validation_samples: int=10*N, 
        source_X: np.array=train_X, 
        source_y: np.array=train_y
) -> Tuple[List[DataSet], DataSet]:
    ids = [
        np.random.choice(
            len(source_X), 
            n_samples, 
            replace=False
        ) for i in range(n_datasets)
    ]
    rest_ids = np.array(list(set(range(len(source_X))) - set(list(np.concatenate(ids)))))
    validation_ids = np.random.choice(rest_ids, n_validation_samples)
    return [DataSet(source_X[i], source_y[i], i) for i in ids], DataSet(source_X[validation_ids], source_y[validation_ids], validation_ids)
from sklearn.svm import SVR
from sklearn.model_selection import ParameterSampler
from sklearn.metrics import r2_score

params_dict = {
    'kernel': ['rbf'],
    'gamma': [1 / i for i in range(80, 130, 10)],
    'C': [0.9, 1.0, 1.1],
    'epsilon': [1e-3, 3e-3, 1e-2, 3e-2, 1e-1, 3e-1],
    'shrinking': [True]
}

def train_svr(train_set: DataSet, param_validation_set: DataSet, params_dict: dict=params_dict, n_iter: int=25):
    ps = ParameterSampler(n_iter=n_iter, param_distributions=params_dict)
    scores = np.zeros(n_iter)
    models = list()
    for idx, params in enumerate(ps):
        svr = SVR(**params)
        svr.fit(train_set.X, train_set.y)
        preds = svr.predict(param_validation_set.X)
        scores[idx] = r2_score(param_validation_set.y, preds)
        models.append(svr)
    return models[np.argmax(scores)]

def _train_one(tupl):
    ds, param_validation_set = tupl
    return train_svr(ds, param_validation_set)
def make_probabilities(weighted_sample_scores):
    # must modify sample scores to be nonnegative
    modified_sample_scores = np.exp(weighted_sample_scores)
    inversed_sample_scores = 1 / modified_sample_scores
    normalised_sample_scores = inversed_sample_scores / inversed_sample_scores.sum()
    # cut useless datasamples
    sorted_ids = np.argsort(normalised_sample_scores)
    cum_sum = np.cumsum(normalised_sample_scores[sorted_ids])
    to_zero_out_ids = sorted_ids[cum_sum < 0.5]
    normalised_sample_scores[to_zero_out_ids] = 0
    
    renormalised_sample_scores = normalised_sample_scores / normalised_sample_scores.sum()
    return renormalised_sample_scores
from tqdm import tqdm
from itertools import combinations
from math import ceil
from multiprocessing import Pool, Array

class VerboseMixin(object):
    def _progress(self, iterator, total):
        if self.verbose:
            return tqdm(iterator, total=total, desc=self.__class__.__name__)
        else:
            return iterator

    def _log(self, message):
        if self.verbose:
            print(f"[{self.__class__.__name__}] {message}")

class Evolution(VerboseMixin):
    def __init__(
            self, 
            T=T, 
            M=M,
            train_X=train_X, 
            train_y=train_y, 
            valid_X=valid_X, 
            valid_y=valid_y, 
            n_generations: int=100,
            n_workers: int=4,
            verbose: bool=False
    ):
        self.training_sets, self.param_validation_set = sample_datasets(n_datasets=T)
        self.entire_dataset = DataSet(train_X, train_y, np.arange(len(train_X)))
        self.model_validation_dataset = DataSet(valid_X, valid_y, None)
        self.n_generations = n_generations
        self.verbose = verbose
        self.M = M
        self.n_workers = n_workers

    def _train(self):
        
        sample_scores = np.zeros(
            (len(self.training_sets), len(self.entire_dataset.X))
        )  # sample_scores[model_id][sample] = model's uncertainty of this sample
        model_validation_scores = np.zeros(len(self.training_sets))
        model_params = list()
        

        print("Multiprocess training...")
        with Pool(processes=4) as pool:
            models = pool.map(_train_one, zip(self.training_sets, [self.param_validation_set] * len(self.training_sets)))
            
        print("Synchronization...")    
        for model_id, (ds, model) in self._progress(
                enumerate(zip(self.training_sets, models)), total=len(self.training_sets)
        ):
#             model = train_svr(ds, self.param_validation_set)
            preds = model.predict(self.entire_dataset.X)
            sample_scores[model_id] = np.abs(self.entire_dataset.y - preds)
            preds = model.predict(self.model_validation_dataset.X)
            model_validation_scores[model_id] = r2_score(self.model_validation_dataset.y, preds)
            model_params.append(model.get_params())
        return np.mean(sample_scores * model_validation_scores.reshape((-1,1)), axis=0), model_params, model_validation_scores
    
    def _select_sets(self, model_validation_scores: np.array):
        normalized_scores = model_validation_scores / np.sum(model_validation_scores)
        sorted_order = np.argsort(-normalized_scores)  # sort by DESCENDING SCORE
        self.training_sets = list(np.array(self.training_sets)[sorted_order])
        cum_scores = np.cumsum(normalized_scores[sorted_order])
        fitness_threshold = 0.
        while fitness_threshold < cum_scores[1]:
            fitness_threshold = np.random.random()
        fit_datasets = [
            ds for ds, is_fit 
            in zip(self.training_sets, cum_scores < fitness_threshold) 
            if is_fit
        ]
        return fit_datasets  # sorted by model score, descending
    
    def _crossover(self, fit_datasets: List[np.array]):
        new_datasets = []
        while len(new_datasets) < len(self.training_sets):
            # if there is a really small number of fit_datasets, 
            # we want to resample T new datasets from what we have
            for ds1, ds2 in combinations(fit_datasets, 2):
                new_datasets.append(DataSet.combine(ds1, ds2))
                if len(new_datasets) >= len(self.training_sets):
                    break
        self.training_sets = new_datasets
        
    def _mutate(self, sample_scores):
        new_datasets = []
        for ds in self.training_sets:
            ids = np.random.permutation(len(ds.X))
            num_of_dumped = ceil(len(ds.X) * self.M)
            chosen = ids[:-num_of_dumped]
            supplied = ds.ids[chosen]
            while len(np.intersect1d(ds.ids[chosen], supplied)) != 0:
                supplied = np.random.choice(np.arange(len(self.entire_dataset.X)), size=num_of_dumped, replace=False, p=sample_scores)
            new = DataSet(
                np.concatenate([ds.X[chosen], self.entire_dataset.X[supplied]]),
                np.concatenate([ds.y[chosen], self.entire_dataset.y[supplied]]),
                np.concatenate([ds.ids[chosen], supplied])
            )
            new_datasets.append(new)
        self.training_sets = new_datasets
            
                
    def __iter__(self):
        self.generation = 0
        return self
    
    def __next__(self):
        if self.generation >= self.n_generations:
            raise StopIteration()
        weighted_sample_scores, model_params, model_validation_scores = self._train()
        fit_datasets = self._select_sets(model_validation_scores)
        
        self._crossover(fit_datasets)
        probs = make_probabilities(weighted_sample_scores)
        print("Data probabilities variance:", probs[probs != 0].var())
        self._mutate(probs)
        self.generation += 1
        return model_validation_scores, probs
ev = Evolution(n_generations=20, verbose=True)
history = []
data = []
for scores, sample_probs in ev:
    print("Number of samples chosen to resample datasets:", len(sample_probs[sample_probs != 0]))
    history.append(scores)
    data.append(sample_probs)
    print("Mean model score:", scores.mean())
import matplotlib.pyplot as plt

probs = data[0][data[0] != 0]
print(len(probs))
plt.hist(probs, bins=100, log=True)
import json
with open("scores_first_ok.json", 'w') as f:
    json.dump([float(s) for s in scores], f)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10704312752094322, 'fraction_numerical': 0.010549177784672665, 'mean_word_length': 2.657337367624811, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '57211', 'license': 'BSD-2-Clause', 'n_tokens_mistral': 3282, 'n_tokens_neox': 3142, 'n_words': 610, 'path': 'notebooks/genetic.ipynb', 'repo_name': 'tmiskow/clash-royale-challenge', 'size': '9669'}