<jupyter_start><jupyter_text>MODNet 'matbench_log_gvrh' and 'matbench_log_kvrh' benchmarking Multi-target benchmarking of the ~11,000 entry elastic properties datasets provided by matminer.<jupyter_code>from collections import defaultdict
import itertools
import os
import pandas as pd
import matplotlib.pyplot as plt 
import numpy as np
from matminer.datasets import load_dataset
from pymatgen.core import Composition

from IPython.display import Markdown
from modnet.preprocessing import MODData
from modnet.models import MODNetModel
from modnet.featurizers import MODFeaturizer
from modnet.featurizers.presets import DeBreuck2020Featurizer

os.environ["CUDA_VISIBLE_DEVICES"] = ""
Markdown(filename="./README.md")<jupyter_output><empty_output><jupyter_text>Data exploration<jupyter_code>df_gvrh = load_dataset("matbench_log_gvrh")
df_kvrh = load_dataset("matbench_log_kvrh")
df_combined = pd.DataFrame(df_gvrh)
df_combined["logK"] = df_kvrh["log10(K_VRH)"]
df_combined["logG"] = df_gvrh["log10(G_VRH)"]
df_temp = df_combined[['structure']]
df_temp['structure_dict'] = df_temp['structure'].apply(lambda x: x.as_dict())
df_temp = df_temp.drop('structure',axis=1)
df_temp.to_csv('df_GK_struc.csv')<jupyter_output><empty_output><jupyter_text>Target space<jupyter_code>df_combined.describe()
fig, ax = plt.subplots(facecolor="w")
ax.hist(df_combined["log10(G_VRH)"], bins=100, density=True);
ax.set_ylabel("Frequency")
ax.set_xlabel("log10(G_VRH)")
fig, ax = plt.subplots(facecolor="w")
ax.hist(df_combined["log10(K_VRH)"], bins=100, density=True);
ax.set_ylabel("Frequency")
ax.set_xlabel("log10(K_VRH)")
fig, ax = plt.subplots(facecolor="w")
ax.scatter(df_combined["log10(G_VRH)"], df_combined["log10(K_VRH)"], alpha=0.1)
ax.set_xlabel("log10(G_VRH)")
ax.set_ylabel("log10(K_VRH)")<jupyter_output><empty_output><jupyter_text>Featurization and feature selection<jupyter_code>PRECOMPUTED_MODDATA = "./precomputed/elastic_benchmark_moddata_MPCNMI.pkl.gz"

if os.path.isfile(PRECOMPUTED_MODDATA):
    data = MODData.load(PRECOMPUTED_MODDATA)
else:
    data = MODData(
        structures=df_combined["structure"].tolist(), 
        targets=df_combined[["log10(G_VRH)", "log10(K_VRH)"]],
        target_names=["log10(G_VRH)", "log10(K_VRH)"],
        featurizer=DeBreuck2020Featurizer(n_jobs=8)
    )
    data.featurize()
    # As this is a small data/feature set, order all features 
    data.feature_selection(n=-1)
data.df_targets.rename(columns={"log10(G_VRH)": "logG", "log10(K_VRH)": "logK"}, inplace=True)
#data.num_classes = {"log10(G_VRH)":0, "log10(K_VRH)":0}
#data.feature_selection(n=-1)
#data.save("./precomputed/elastic_benchmark_moddata_MPCNMI.pkl.gz")
#data.save("./precomputed/elastic_benchmark_moddata_MPCNMI.pkl.gz")
data.df_targets
data.<jupyter_output><empty_output><jupyter_text>Training This is a very small dataset, so we must make judicious use of what data there is. First, let's generate test folds according to [matbench's suggestions](https://hackingmaterials.lbl.gov/automatminer/datasets.htmlbenchmarking-and-reporting-your-algorithm):<jupyter_code>#data.df_targets.columns=['G','K']
from modnet.matbench.benchmark import matbench_benchmark

best_settings = {
    "increase_bs":False,
    "num_neurons": [[256], [64], [64], [32]],
    "n_feat": 350,
    "lr": 0.005,
    "epochs": 200,
    "verbose": 0,
    "act": "elu",
    "batch_size": 64,
    "loss": "mae",
    
}

results = matbench_benchmark(data, [[["logG"],["logK"]]],
                             {"logG": 1, "logK":1}, best_settings,multi_target=True,save_folds=True)
np.mean(np.array(results["scores"])[[0,1,2,4]],axis=0)
data.optimal_features_by_target["logG"] = data.optimal_features_by_target["log10(G_VRH)"]
data.optimal_features_by_target["logK"] = data.optimal_features_by_target["log10(K_VRH)"]
errors_K = []
errors_G = []
for f in range(5):
    df = pd.read_csv('folds/test_f{}.csv'.format(f+1),index_col=0)
    if f==3:
        print(df['K_error'].abs().nlargest(n=5))
        df = df.drop('id6951')
    print(df['K_error'].abs().describe())
    errors_K.append(df['K_error'].abs().mean())
    errors_G.append(df['G_error'].abs().mean())
print(np.mean(errors_K))
print(np.mean(errors_G))
data.df_structure.loc['id6951']['structure'].formula
for i,c in zip(range(5),['b','k','r','g','y']):
    plt.plot(results["models"][i].history.history["loss"][50:],c=c)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1644992014601871, 'fraction_numerical': 0.025553274013232947, 'mean_word_length': 6.870736086175943, 'pattern_counts': {'":': 15, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '584291', 'license': 'MIT', 'n_tokens_mistral': 1789, 'n_tokens_neox': 1697, 'n_words': 239, 'path': 'matbench_elastic/matbench_elastic.ipynb', 'repo_name': 'ppdebreuck/modnet-matbench', 'size': '4383'}