<jupyter_start><jupyter_text>Match adsorbents by synonyms<jupyter_code>import pandas as pd
import pickle 
from tqdm import tqdm
from collections import Counter
ADSORBENTS_BLACKLISTED = [ # ambiguous names: see Supp. Info.
    "mil-53",
    "mil-100",
    "mil-101",
    "mil-102",
    "mof-74",
    "mil-68",
    "mil-88b",
]

with open("data/nistdb.pickle", 'rb') as f:
    nistdb = pickle.load(f)
# Compare mismatches for low/upper case
df = pd.read_csv("data/step-02.csv")
for i, csd_synonyms in enumerate(df['synonyms']): # Takes ca. 5 secs
    for csd_syn in eval(csd_synonyms):
        for mat in nistdb['Adsorbents']:
            all_nist_names = mat['synonyms'] + [mat['name']]
            if csd_syn.lower() in [ x.lower() for x in all_nist_names ] and csd_syn not in all_nist_names:
                print(csd_syn, all_nist_names)
# Use case INsensitive
df = pd.read_csv("data/step-02.csv")
for i, csd_synonyms in enumerate(tqdm(df['synonyms'])): # Takes ca. 1 min
    if df.at[i,"note"]=="Excluding more than 3 same MOFs from the same DOI":
        #print(df.at[i,"publication_doi"])
        continue
    for csd_syn in eval(csd_synonyms):
        csd_syn = csd_syn.lower()
        
        for mat in nistdb['Adsorbents']:
            nist_syns = [ x.lower() for x in [ mat['name']] + mat['synonyms'] ]
            
            if nist_syns[0] in ADSORBENTS_BLACKLISTED: # ambiguous names skipped
                continue
            
            if csd_syn in nist_syns :
                df.at[i,"nist_mat"] = mat['name'] # NOTE: adsorbent name will be always the main one for NIST, not a synonym (e.g., no HKUST-1, only CuBTC)
                df.at[i,"note"] = "Matched by synonym"
                break
                
            # Try to perform some further matches: "MOF"=="MOFa", "MOF"=="MOF'", "MOF-n"=="MOFn"=="MOF n" **
            for nist_syn in nist_syns:
                if any([
                    csd_syn+'a'==nist_syn,
                    csd_syn==nist_syn+'a',
                    csd_syn+"'"==nist_syn,
                    csd_syn==nist_syn+"'",
                    csd_syn.replace('-',"")==nist_syn,
                    csd_syn==nist_syn.lower().replace('-',""),
                    csd_syn.replace('-'," ")==nist_syn,
                    csd_syn==nist_syn.replace('-'," ")
                ]):
                    df.at[i,"nist_mat"] = mat['name']
                    df.at[i,"note"] = "Matched by synonym (corrected)"
                    break                      
            
df.to_csv("data/step-03.csv", index=False)
# step-03.csv adds "note: Matched by synonym / Matched by synonym (corrected)" and the nist_name, if matched
df = pd.read_csv("data/step-03.csv")
df[df['note'].str.startswith("Matched")].reset_index().head(10)<jupyter_output><empty_output><jupyter_text>Get Statistics<jupyter_code>df = pd.read_csv("data/step-03.csv")
print(f"(CSD matched thanks to correction: {len(df[df['note']=='Matched by synonym (corrected)'])}, see ** previous cell)")
df_matches_found = df[df['note'].str.startswith('Matched by synonym')] # can also be "Matched by synonym (corrected)"

print(f"Matches found CSD:  {df_matches_found['nist_mat'].count()} ... but they could be duplicates in the same paper!")
      
# Exclude those in the same paper
df_matches_found_differentpapers = df_matches_found.copy()
df_matches_found_differentpapers = df_matches_found_differentpapers[~df_matches_found_differentpapers.duplicated(subset=["nist_mat", "publication_doi"],keep='first')] 

print(f"Matches found CSD:  {df_matches_found_differentpapers['nist_mat'].count()} ... from different papers!")
      
# set() is taking too long!
unique = {} # considering unique if same CSD
for x in df_matches_found["nist_mat"]:
    if x not in unique:
        unique[x] = 1
    else:
        unique[x] += 1
        
unique_dp = {} # considering unique if same paper
for x in df_matches_found_differentpapers["nist_mat"]:
    if x not in unique_dp:
        unique_dp[x] = 1
    else:
        unique_dp[x] += 1
    
print()
print(f"Matches found NIST: {len(unique)} (by unique CSD enty)")
print(f"Matches found NIST: {len(unique_dp)} (by unique paper, should match with the previous!)")
# (continues from previous cell)
# Count CSD entries and NIST-ISODB adsorbents coming from the same DOI
count_unique = dict(Counter(unique.values()))
count_unique = {k: v for k, v in sorted(count_unique.items(), key=lambda x: x[0])}
for k, v in count_unique.items():
    print(f"{v}\t NIST-ISODB adsorbents associated to {k} CSD entries")

print()
unique = {k: v for k, v in sorted(unique.items(), key=lambda x: x[1])}   
for k, v in unique.items():
    if v>=4:
        print(v, k)
        
print("\nNow considering the paper: more CSD entries can be associated to the same paper!")
print("NOTE: this statistic is more interesting, because we care if the same MOF has been used for different studies (papers)\n")

count_unique = dict(Counter(unique_dp.values()))
count_unique = {k: v for k, v in sorted(count_unique.items(), key=lambda x: x[0])}
for k, v in count_unique.items():
    print(f"{v}\t NIST-ISODB adsorbents associated to {k} paper")

print()
unique_dp = {k: v for k, v in sorted(unique_dp.items(), key=lambda x: x[1])}   
for k, v in unique_dp.items():
    if v>=4:
        print(v, k)
# Exclude all the DOIs that contain already a match by synonim, or this can create problems when considering the number of structures/isotherms in the paper!
df = pd.read_csv("data/step-03.csv")
dois_already_matched = list(df[df['note'].str.startswith("Matched by synonym")]['publication_doi'])

for row in tqdm(df.itertuples(), total=df.shape[0]):
    if row.note=='-' \
       and not pd.isnull(row.publication_doi) \
       and row.publication_doi in dois_already_matched:
        df.at[row.Index, 'note'] = "Exclude DOI already matched by synonym"

df.to_csv("data/step-04.csv", index=False) 
# step-04.csv: add note "Exclude" if material from same DOI already matched

print("CSD entries excluded:", len(df[df['note']=="Exclude DOI already matched by synonym"]))<jupyter_output>100%|██████████| 105922/105922 [00:00<00:00, 165327.45it/s]
{'fraction_non_alphanumeric': 0.1401107852720756, 'fraction_numerical': 0.013685239491691105, 'mean_word_length': 3.100868403473614, 'pattern_counts': {'":': 1, '<': 9, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '132541', 'license': 'MIT', 'n_tokens_mistral': 2243, 'n_tokens_neox': 2108, 'n_words': 556, 'path': '03_match_by_synonym.ipynb', 'repo_name': 'danieleongari/matching_isodb_csd', 'size': '6138'}