<jupyter_start><jupyter_text>Method- Generate balanced classes with a greater temperature granularity (say about 8-10)- Select the same number of species & same number of genes in all classes<jupyter_code>y = dataset_df['temperature'].values.astype(np.float64)
f, ax = plt.subplots(1, 1, figsize=(12, 6))
ax.hist(y, bins=len(np.unique(y)));
ax.set_xlabel('Optimal Growth Temperature (°C)');
ax.set_ylabel('Number of records');
def bin_data(y, bins):
    output = []
    counts = collections.defaultdict(int)
    bucket_to_species = collections.defaultdict(set)
    specie_to_bin = dict()
    for i, yi in enumerate(y):
        for j, bin_ in enumerate(bins):
            if yi >= bin_[0] and yi < bin_[1]:
                output.append(j)
                counts[j] += 1
                specie_name = dataset_df.iloc[i]['specie_name']
                if specie_name in specie_to_bin and specie_to_bin[specie_name] != j:
                    print(specie_name, j, specie_to_bin[specie_name])
                    raise ValueError()
                specie_to_bin[specie_name] = j
                bucket_to_species[j].add(specie_name)
                break
                
    for i in range(len(bins)):
        print(bins[i], counts[i])
        
    return np.array(output), bucket_to_species
bins = [
    [4, 25],
    [25, 30],
    [30, 35],
    [35, 40],
    [40, 80],
]
y_binned, bucket_to_species = bin_data(y, bins)
y_binned.shape, len(bins)
f, ax = plt.subplots(1, 1, figsize=(12, 6))
ax.hist(y_binned, bins=len(np.unique(y_binned)));
ax.set_xlabel('Temperature class');
ax.set_ylabel('Number of records');
for k in sorted(bucket_to_species.keys()):
    print(k, len(bucket_to_species[k]))
def balanced_species_per_temperature_buckets(species_per_temp_bucket, seed):
    rs = np.random.RandomState(seed)
    
    n_species = np.inf
    buckets = sorted(species_per_temp_bucket.keys())
    for k in buckets:
        n = len(species_per_temp_bucket[k])
        if n < n_species:
            n_species = n
    
    print(f'Species per bucket in balance dataset: {n_species}')
    assert n_species < np.inf
    
    species = []
    for k in buckets:
        species_in_bucket = sorted(species_per_temp_bucket[k])
        
        if len(species_in_bucket) <= n_species:
            species += species_in_bucket
        else:
            species_ = rs.choice(species_in_bucket, size=n_species, replace=False)
            species += sorted(species_.tolist())
            
    assert len(species) == len(buckets) * n_species
    assert len(species) == len(set(species))

    return species
selected_species = balanced_species_per_temperature_buckets(bucket_to_species, seed=444)<jupyter_output>Species per bucket in balance dataset: 238<jupyter_text>Split train test sets<jupyter_code>def split_train_test_sets(selected_species, test_ratio, seed):
    assert test_ratio > 0 and test_ratio < 1
    
    rs = np.random.RandomState(seed)
    
    n_species = len(selected_species)
    
    test_idx = rs.choice(range(n_species), size=int(test_ratio * n_species), replace=False)
    test_idx_set = set(test_idx.tolist())
    train_idx = np.array([idx for idx in range(n_species) if idx not in test_idx_set])
    
    selected_species_ndarray = np.array(selected_species)
    
    train_set = selected_species_ndarray[train_idx]
    test_set = selected_species_ndarray[test_idx]
    
    assert len(train_set) == len(train_idx)
    assert len(test_set) == len(test_idx)
    assert len(train_set) + len(test_set) == n_species
    
    return train_set, test_set, train_idx, test_idx
train_set, test_set, train_idx, test_idx = split_train_test_sets(selected_species, test_ratio=0.2, seed=444)
len(dataset_df[dataset_df['specie_name'].isin(test_set)])
len(dataset_df[dataset_df['specie_name'].isin(train_set)])
output_train_path = os.path.join(os.getcwd(), 'data/dataset_balanced_train.csv')
output_test_path = os.path.join(os.getcwd(), 'data/dataset_balanced_test.csv')
def export_train_test(dataset_df, train_set, test_set, output_train_path, output_test_path):
    df_train = dataset_df[dataset_df['specie_name'].isin(train_set)].reset_index(drop=True)
    df_test = dataset_df[dataset_df['specie_name'].isin(test_set)].reset_index(drop=True)
    
    df_train.to_csv(output_train_path, index=False)
    df_test.to_csv(output_test_path, index=False)
# export_train_test(dataset_df, train_set, test_set, output_train_path, output_test_path)
df_train = dataset_df[dataset_df['specie_name'].isin(train_set)].reset_index(drop=True)
df_test = dataset_df[dataset_df['specie_name'].isin(test_set)].reset_index(drop=True)
y_train = df_train['temperature'].values.astype(np.float64)
y_test = df_test['temperature'].values.astype(np.float64)
f, ax = plt.subplots(1, 1, figsize=(12, 6))
ax.hist(y_train, bins=len(np.unique(y_train)));
ax.set_xlabel('Optimal Growth Temperature (°C)');
ax.set_ylabel('Number of records');
ax.set_title('Train set');
f, ax = plt.subplots(1, 1, figsize=(12, 6))
ax.hist(y_test, bins=len(np.unique(y_test)));
ax.set_xlabel('Optimal Growth Temperature (°C)');
ax.set_ylabel('Number of records');
ax.set_title('Test set');<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12424888544291529, 'fraction_numerical': 0.012405504942818375, 'mean_word_length': 4.113974231912785, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '476847', 'license': 'MIT', 'n_tokens_mistral': 1927, 'n_tokens_neox': 1791, 'n_words': 331, 'path': 'notebook/archive/Prepare dataset (more temperature classes).ipynb', 'repo_name': 'srom/rna_learn', 'size': '5159'}