<jupyter_start><jupyter_text>MODNet 'matbench_phonons' benchmarking<jupyter_code>from collections import defaultdict
import itertools
import os
import pandas as pd
import matplotlib.pyplot as plt 
import numpy as np
from IPython.display import Markdown
from matminer.datasets import load_dataset
from pymatgen.core import Composition

from modnet.preprocessing import MODData
from modnet.models import MODNetModel
from modnet.featurizers import MODFeaturizer
from modnet.featurizers.presets import DeBreuck2020Featurizer

os.environ["CUDA_VISIBLE_DEVICES"] = "3"
Markdown(filename="./README.md")<jupyter_output><empty_output><jupyter_text>Data exploration<jupyter_code>df = load_dataset("matbench_phonons")
df.columns<jupyter_output><empty_output><jupyter_text>Target space<jupyter_code>df.describe()
fig, ax = plt.subplots(facecolor="w")
ax.hist(df["last phdos peak"], bins=100, density=True);
ax.set_ylabel("Frequency")
ax.set_xlabel("Last PhDOS peak")<jupyter_output><empty_output><jupyter_text>Featurization and feature selection First, we define some convenience classes that pass wraps composition data in a fake structure containe, and we define a composition only featurizer preset based on `DeBreuck2020Featurizer`.<jupyter_code>PRECOMPUTED_MODDATA = "./precomputed/phonon_benchmark_moddata.pkl.gz"

if os.path.isfile(PRECOMPUTED_MODDATA):
    data = MODData.load(PRECOMPUTED_MODDATA)
else:
    data = MODData(
        structures=df["structure"].tolist(), 
        targets=df["last phdos peak"].tolist(), 
        target_names=["last phdos peak"],
        featurizer=DeBreuck2020Featurizer(n_jobs=8)
    )
    data.featurize()
    data.feature_selection(n=-1)
    data.save(PRECOMPUTED_MODDATA)
#data.optimal_features=None
#data.cross_nmi = None
#data.num_classes = {"w":0}
#data.feature_selection(n=-1)
#data.save("./precomputed/phonon_benchmark_moddata_MPCNMI.pkl.gz")<jupyter_output><empty_output><jupyter_text>Training<jupyter_code>try:
    plot_benchmark
except:
    import sys
    sys.path.append('..')
    from modnet_matbench.utils import *
from sklearn.model_selection import KFold
from modnet.models import MODNetModel
from tensorflow.keras.callbacks import ReduceLROnPlateau, EarlyStopping

data.df_targets.rename(columns={data.target_names[0]: "w"}, inplace=True)

# [[512], [128], [32], [16]]
best_settings = {
    "increase_bs":True,
    "num_neurons": [[512], [128], [64], [64]],
    "n_feat": 280,
    "lr": 0.005,
    "epochs": 800,
    "act": "elu",
    "batch_size": 64,
    "loss": "mae",
}

results = matbench_benchmark(data, [[["w"]]], {"w": 1}, best_settings,save_folds=True)
np.mean(results['scores'])
best_settings = {
    "increase_bs":True,
    "num_neurons": [[512], [128], [64], [64]],
    "n_feat": 280,
    "lr": 0.005,
    "epochs": 800,
    "act": "elu",
    "batch_size": 64,
    "loss": "mae",
}
fig, ax = plt.subplots()
sns.scatterplot(data=reg_df, x="targets", y="predictions", hue="split", palette="Dark2", ax=ax, alpha=0.5)
sns.regplot(data=reg_df, x="targets", y="predictions", ax=ax, scatter=False)
plt.xlabel("True")
plt.ylabel("Pred.")
g = sns.jointplot(data=reg_df, x="errors", y="predictions", hue="split", palette="Dark2", alpha=0.0, marginal_kws={"shade": False})
g.plot_joint(sns.scatterplot, hue=None, c="black", s=5, alpha=0.8)
g.plot_joint(sns.kdeplot, color="split", zorder=0, levels=5, alpha=0.5)
sns.kdeplot(data=reg_df, x="targets", y="predictions", hue="split", shade=False, levels=3, palette="Dark2", alpha=0.5, )<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15104018238814476, 'fraction_numerical': 0.026503277286976348, 'mean_word_length': 6.237113402061856, 'pattern_counts': {'":': 19, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '495482', 'license': 'MIT', 'n_tokens_mistral': 1375, 'n_tokens_neox': 1290, 'n_words': 243, 'path': 'matbench_phonons/phonons_benchmark.ipynb', 'repo_name': 'ppdebreuck/modnet-matbench', 'size': '3509'}