<commit_before>#%% IMPORTING THE FULL DATAFRAME
import pandas as pd  ## provides a fast spreadsheet object (DataFrame)
from pathlib import Path  ## for os-agnostic paths
import matplotlib.pyplot as plt  ## standard python plotting tool

HERE = Path(__file__).parent
MATERIALS_PROJECT_PKL = str(HERE/"materials_project.pkl")
SUBSET_IDS_CSV = str(HERE/"subset_ids.csv")
CITRINE_MP_PKL = str(HERE/"citrine_mp.pkl")
PLOTS_DIR = HERE/"plots"

mpdf = pd.read_pickle(MATERIALS_PROJECT_PKL)


print("DOWNLOADED:")
print(f"loaded {len(mpdf)} materials")

print("\n\nSUBSET:")
subset_ids = pd.read_csv(SUBSET_IDS_CSV, names=["material_id"])
print(len(subset_ids), "material_ids")
subset_mpdf = mpdf[mpdf["material_id"].isin(subset_ids["material_id"])]
print(len(subset_mpdf), "materials selected")
print(len(subset_mpdf)-len(subset_ids), "missing ids")
print(len(mpdf)-len(subset_mpdf), "missing from downloaded")

print("\navailable columns")
for name in list(mpdf.columns): print(" ",name)

print("\n\nCITRINE:")
citrine_mpdf = pd.read_pickle(CITRINE_MP_PKL)
print(f"loaded {len(citrine_mpdf)} materials")
print("\navailable columns:")
for name in list(citrine_mpdf.columns): print(" ",name)


#%% IS CITRINE VERY DIFFERENT FROM UPDATED DATA ?
mpdf_in_citrine = mpdf[mpdf["material_id"].isin(citrine_mpdf["material_id"])]
print(f"{len(mpdf_in_citrine)} updated/{len(citrine_mpdf)} originals, {len(citrine_mpdf)-len(mpdf_in_citrine)} missing") 

citrine_in_mpdf = 



#%% WHAT IS THE SUBSET FROM CITRINE ?

have_icsd = mpdf[mpdf['icsd_ids'].astype(bool)]
print(f"\n{len(have_icsd)} entries have ICSD IDs")
stable_icsd = mpdf[ (mpdf['icsd_ids'].astype(bool)) & (mpdf["e_above_hull"] <= 0)]
print(f"{len(stable_icsd)} entries have ICSD IDs and are stable")

no_warnings = mpdf[(~mpdf['warnings'].astype(bool)) & (mpdf["e_above_hull"] <= 0)]
print(f"{len(no_warnings)} entries have no warnings and are stable")





#%% ARE THE STRUCRURES VERY DIFFERENT IN OQMD, ICSD, MP?




<commit_msg>find out wht is CITRINE<commit_after>#%% IMPORTING THE FULL DATAFRAME
import pandas as pd  ## provides a fast spreadsheet object (DataFrame)
from pathlib import Path  ## for os-agnostic paths
import matplotlib.pyplot as plt  ## standard python plotting tool

HERE = Path(__file__).parent
MATERIALS_PROJECT_PKL = str(HERE/"materials_project.pkl")
SUBSET_IDS_CSV = str(HERE/"subset_ids.csv")
CITRINE_MP_PKL = str(HERE/"citrine_mp.pkl")
PLOTS_DIR = HERE/"plots"

mpdf = pd.read_pickle(MATERIALS_PROJECT_PKL)


print("DOWNLOADED:")
print(f"loaded {len(mpdf)} materials")

print("\n\nSUBSET:")
subset_ids = pd.read_csv(SUBSET_IDS_CSV, names=["material_id"])
print(len(subset_ids), "material_ids")
subset_df = mpdf[mpdf["material_id"].isin(subset_ids["material_id"])]
print(len(subset_df), "materials selected")
print(len(subset_df)-len(subset_ids), "missing ids")
print(len(mpdf)-len(subset_df), "missing from downloaded")

print("\navailable columns")
for name in list(mpdf.columns): print(" ",name)

print("\n\nCITRINE:")
citrine_df = pd.read_pickle(CITRINE_MP_PKL)
print(f"loaded {len(citrine_df)} materials")
print("\navailable columns:")
for name in list(citrine_df.columns): print(" ",name)


#%% IS CITRINE VERY DIFFERENT FROM UPDATED DATA ?
mpdf_in_citrine = mpdf[mpdf["material_id"].isin(citrine_df["material_id"])]
print(f"{len(mpdf_in_citrine)} updated/{len(citrine_df)} originals, {len(citrine_df)-len(mpdf_in_citrine)} missing") 
citrine_in_mpdf = citrine_df[citrine_df["material_id"].isin(mpdf_in_citrine["material_id"])]
citrine_not_in_mpdf = citrine_df[~citrine_df["material_id"].isin(mpdf_in_citrine["material_id"])]

print("common columnds")
for name in list(citrine_in_mpdf.columns):
    if name in list(mpdf_in_citrine.columns):
        print(" ",name)

X = citrine_in_mpdf.sort_values("material_id", ignore_index=True)
Y = mpdf_in_citrine.sort_values("material_id", ignore_index=True)
X["material_id"]==Y["material_id"]

#%% PLOT OLD vs NEW
column = "e_above_hull"
x = X[column].rename(f"updated_{column}")
y = Y[column].rename(f"citrine_{column}")
plot_df = pd.concat([x, y], axis=1)
plot_df.plot.scatter(f"citrine_{column}",f"updated_{column}")
plt.title(f"{column}\nonly {len(X[X[column]==Y[column]])}/{len(X)} are the same")
plt.savefig(PLOTS_DIR/f"updated_citrine_{column}")





#%% WHAT IS THE SUBSET FROM CITRINE ?
print(f"IN CITRINE (total {len(citrine_df)}) / IN LAST MP {len(mpdf)}")
mc = citrine_df
print(f"{len(mc[mc['icsd_ids'].astype(bool)])}/{len(mpdf[mpdf['icsd_ids'].astype(bool)])} have ICSD ids {len(mc[mc['icsd_ids'].astype(bool)])-len(mpdf[mpdf['icsd_ids'].astype(bool)])} have no more icsd_ids")


#%% ICSD? Stable?
have_icsd = mpdf[mpdf['icsd_ids'].astype(bool)]
print(f"\n{len(have_icsd)} entries have ICSD IDs")
stable_icsd = mpdf[ (mpdf['icsd_ids'].astype(bool)) & (mpdf["e_above_hull"] <= 0)]
print(f"{len(stable_icsd)} entries have ICSD IDs and are stable")

no_warnings = mpdf[(~mpdf['warnings'].astype(bool)) & (mpdf["e_above_hull"] <= 0)]
print(f"{len(no_warnings)} entries have no warnings and are stable")





#%% ARE THE STRUCRURES VERY DIFFERENT IN OQMD, ICSD, MP?





{'fraction_non_alphanumeric': 0.16375759952931948, 'fraction_numerical': 0.0009805844283192783, 'mean_word_length': 7.68824531516184, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2710314', 'n_tokens_mistral': 2081, 'n_tokens_neox': 1964, 'n_words': 364}