<commit_before>#%% IMPORTING THE FULL DATAFRAME
from pathlib import Path  ## for os-agnostic paths
import pandas as pd  ## provides a fast spreadsheet object (DataFrame)
import matplotlib.pyplot as plt  ## standard python plotting tool

HERE = Path(__file__).parent
PARENT = Path(__file__).parent.parent
PLOTS_DIR = HERE/"plots"

MATERIALS_PROJECT_PKL = str(HERE/"mp"/"materials_project.pkl")
SUBSET_MPIDS_CSV = str(HERE/"mp"/"subset_ids.csv")
CITRINE_MP_PKL = str(HERE/"mp"/"citrine_mp.pkl")

ICSD_AUG_PKL = str(HERE/"icsd"/"all_icsd_cifs_augmented.pkl")

OQMD_PKL = str(HERE/"oqmd"/"oqmd.pkl")
OQMD_CSV = str(HERE/"oqmd"/"oqmd.csv")

mpdf = pd.read_pickle(MATERIALS_PROJECT_PKL)
icsdf = pd.read_pickle(ICSD_AUG_PKL)
oqmdf = pd.read_pickle(OQMD_PKL)

icsdf = icsdf.set_index(icsdf['_database_code_ICSD'].astype(int))  ## rows are now indexed by icsd_id
mpdf = mpdf.set_index(mpdf['material_id'])  ## rows are now indexed by material_id


#%% MERGE THE MP and ICSD DATAFRAME (didn't find a pandas way to do it)
combined_rows = []
used_icsd_ids = []
wrong_icsd_ids = []
mp_rows = mpdf.to_dict('records')
for mp_row in mp_rows:
    if mp_row['icsd_ids']:
        for icsd_id in mp_row['icsd_ids']:
            try: ## sometimes the icsd_id doesn't exist
                icsd_row = icsdf.loc[icsd_id].to_dict()
                used_icsd_ids.append(icsd_id)
            except KeyError as e:
                wrong_icsd_ids.append(e)
                continue
            tmp_dict = {}
            for k,v in mp_row.items():
                tmp_dict['mp_'+k] = v
            for k,v in icsd_row.items():
                tmp_dict['icsd_'+k] = v
            combined_rows.append(tmp_dict)
    else:
        tmp_dict = {}
        for k,v in mp_row.items():
            tmp_dict['mp_'+k] = v
        combined_rows.append(tmp_dict)

remaining_icsd_rows = icsdf.loc[~icsdf.index.isin(used_icsd_ids)].to_dict('records')
for icsd_row in remaining_icsd_rows:
    tmp_dict = {}
    for k,v in icsd_row.items():
        tmp_dict['icsd_'+k] = v
    combined_rows.append(tmp_dict)

super_df = pd.DataFrame(combined_rows)


#%% PRINT BASIC INFO ABOUT THE DATASETS
for name, dataset in {'MP':mpdf, 'OQMD':oqmdf, 'ICSD':icsdf, 'super':super_df}.items():
    print(f"\nloaded {len(dataset)} materials from {name}")
    print("available columns")
    for column in list(dataset.columns): 
        print(" ",column)













#%%
icsd_mask = ~super_df['icsd__database_code_ICSD'].isna()
mp_mask = ~super_df['mp_material_id'].isna()
grouped_icsd = super_df.loc[icsd_mask & mp_mask].groupby('icsd__database_code_ICSD')
grouped_mp = super_df.loc[icsd_mask & mp_mask].groupby('mp_material_id')

#%%
print(f"{len(wrong_icsd_ids)} bad icsd_ids in the icsd_ids list in MP")
print(f"{len(grouped_mp)} materials from MP map to ICSD entries")
print(f"with maximum {grouped_mp['mp_material_id'].agg('count').max()} ICSD entries for one MP entry")
print(f"{len(grouped_icsd)} materials from ICSD map to MP entries")
print(f"with maximum {grouped_icsd['mp_material_id'].agg('count').max()} MP entries for one ICSD entry")














#%% ICSD WITH INTEGER FORMULA DOPING ONLY
def fraction_composition(s):<commit_msg>umap working<commit_after>#%% IMPORTING THE FULL DATAFRAME
from pathlib import Path  ## for os-agnostic paths
import pandas as pd  ## provides a fast spreadsheet object (DataFrame)
import matplotlib.pyplot as plt  ## standard python plotting tool

HERE = Path(__file__).parent
PARENT = Path(__file__).parent.parent
PLOTS_DIR = HERE/"plots"

MATERIALS_PROJECT_PKL = str(HERE/"mp"/"materials_project.pkl")
SUBSET_MPIDS_CSV = str(HERE/"mp"/"subset_ids.csv")
CITRINE_MP_PKL = str(HERE/"mp"/"citrine_mp.pkl")

ICSD_AUG_PKL = str(HERE/"icsd"/"all_icsd_cifs_augmented.pkl")

OQMD_PKL = str(HERE/"oqmd"/"oqmd.pkl")
OQMD_CSV = str(HERE/"oqmd"/"oqmd.csv")

mpdf = pd.read_pickle(MATERIALS_PROJECT_PKL)
icsdf = pd.read_pickle(ICSD_AUG_PKL)
oqmdf = pd.read_pickle(OQMD_PKL)

#%%
icsdf = icsdf.set_index(icsdf['_database_code_ICSD'].astype(int))  ## rows are now indexed by icsd_id
mpdf = mpdf.set_index(mpdf['material_id'])  ## rows are now indexed by material_id


#%% MERGE THE MP and ICSD DATAFRAME (didn't find a pandas way to do it)
combined_rows = []
used_icsd_ids = []
wrong_icsd_ids = []
mp_rows = mpdf.to_dict('records')
for mp_row in mp_rows:
    if mp_row['icsd_ids']:
        for icsd_id in mp_row['icsd_ids']:
            try: ## sometimes the icsd_id doesn't exist
                icsd_row = icsdf.loc[icsd_id].to_dict()
                used_icsd_ids.append(icsd_id)
            except KeyError as e:
                wrong_icsd_ids.append(e)
                continue
            tmp_dict = {}
            for k,v in mp_row.items():
                tmp_dict['mp_'+k] = v
            for k,v in icsd_row.items():
                tmp_dict['icsd_'+k] = v
            combined_rows.append(tmp_dict)
    else:
        tmp_dict = {}
        for k,v in mp_row.items():
            tmp_dict['mp_'+k] = v
        combined_rows.append(tmp_dict)

remaining_icsd_rows = icsdf.loc[~icsdf.index.isin(used_icsd_ids)].to_dict('records')
for icsd_row in remaining_icsd_rows:
    tmp_dict = {}
    for k,v in icsd_row.items():
        tmp_dict['icsd_'+k] = v
    combined_rows.append(tmp_dict)

super_df = pd.DataFrame(combined_rows)


#%% USING UMAP
import seaborn as sns

umap_mpdf = mpdf[[
    "band_gap",
    "density",
    "e_above_hull",
    # "efermi",
    # "encut",
    # "energy",
    "energy_per_atom",
    # "final_energy",
    # "final_energy_per_atom",
    "formation_energy_per_atom",
    # "nkpts",
    # "nsites",
    "total_magnetization",
    "volume",
    # "exchange_symmetry",
    # "num_unique_magnetic_sites",
    # "total_magnetization_normalized_vol",
    # "total_magnetization_normalized_formula_units",
    # "num_magnetic_sites",
    # "true_total_magnetization",
]].dropna()

# sns.pairplot(umap_mpdf, hue='total_magnetization')

 ## %% UMAP
from sklearn.preprocessing import StandardScaler
import umap
import matplotlib.pyplot as plt
reducer = umap.UMAP()
umap_data = umap_mpdf[[
    "band_gap",
    "density",
    "e_above_hull",
    "energy_per_atom",
    # "total_magnetization",
    "volume",
]].values
scaled_data = StandardScaler().fit_transform(umap_data)
embedding = reducer.fit_transform(scaled_data)
embedding.shape

plt.scatter(
    embedding[:, 0],
    embedding[:, 1],
    s=0.8,
    c=umap_mpdf.total_magnetization
)
plt.gca().set_aspect('equal', 'datalim')
plt.title('UMAP projection', fontsize=24)

#%% PRINT BASIC INFO ABOUT THE DATASETS
for name, dataset in {'MP':mpdf, 'OQMD':oqmdf, 'ICSD':icsdf, 'super':super_df}.items():
    print(f"\nloaded {len(dataset)} materials from {name}")
    print("available columns")
    for column in list(dataset.columns): 
        print(" ",column)


#%%
icsd_mask = ~super_df['icsd__database_code_ICSD'].isna()
mp_mask = ~super_df['mp_material_id'].isna()
grouped_icsd = super_df.loc[icsd_mask & mp_mask].groupby('icsd__database_code_ICSD')
grouped_mp = super_df.loc[icsd_mask & mp_mask].groupby('mp_material_id')


#%% HOW MANY ICSD HAVE 4 MP ASSOCIATED


#%%
print(f"{len(wrong_icsd_ids)} bad icsd_ids in the icsd_ids list in MP")
print(f"{len(grouped_mp)} materials from MP map to ICSD entries")
print(f"with maximum {grouped_mp['mp_material_id'].agg('count').max()} ICSD entries for one MP entry")
print(f"{len(grouped_icsd)} materials from ICSD map to MP entries")
print(f"with maximum {grouped_icsd['mp_material_id'].agg('count').max()} MP entries for one ICSD entry")


#%% ICSD WITH INTEGER FORMULA DOPING ONLY
def fraction_composition(s):
{'fraction_non_alphanumeric': 0.1329122252928017, 'fraction_numerical': 0.000921173838662982, 'mean_word_length': 3.935064935064935, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3815742', 'n_tokens_mistral': 2942, 'n_tokens_neox': 2880, 'n_words': 559}