<jupyter_start><jupyter_text>From Sigma Fragrance and Flavor Catalog (2014)<jupyter_code># Create a new file with CIDs and store here in `cids` dictionary
file_path = DATA / 'sigma_2014' / 'sigma.csv'
df = pd.read_csv(file_path)
cids['sigma-2014'] = set(df['CID']) - {0}<jupyter_output><empty_output><jupyter_text>Dravnieks Atlas of Odor Character<jupyter_code># Create a new file with CIDs and store here in `cids` dictionary
file_path = DATA / 'dravnieks_1985' / 'dravnieks.csv'
df = pd.read_csv(file_path)
cids['dravnieks-1985'] = set(df['CID']) - {0}<jupyter_output><empty_output><jupyter_text>Abraham et al, 2013<jupyter_code>file_path = DATA / 'abraham_2011' / 'abraham-2011-with-CIDs.csv'
df = pd.read_csv(file_path)
cids['abraham-2013'] = set(df['CID']) - {0}<jupyter_output><empty_output><jupyter_text>Bushdid et al, 2014<jupyter_code># Create a new file with CIDs and store here in `cids` dictionary
file_path = DATA / 'bushdid_2014' / 'bushdid.csv'
df = pd.read_csv(file_path)
cids['bushdid-2014'] = set(df['CID']) - {0}<jupyter_output><empty_output><jupyter_text>Chae et al, 2019<jupyter_code># Create a new file with CIDs and store here in `cids` dictionary
file_path = DATA / 'chae_2019' / 'odorants.csv'
df = pd.read_csv(file_path)
cids['chae-2019'] = set(df['CID']) - {0}<jupyter_output><empty_output><jupyter_text>Prestwick<jupyter_code>file_path = DATA / 'prestwick' / 'prestwick.csv'
df = pd.read_csv(file_path)
cids['prestwick'] = set(df['CID']) - {0}<jupyter_output><empty_output><jupyter_text>GRAS<jupyter_code>file_path = DATA / 'GRAS' / 'gras.csv'
df = pd.read_csv(file_path)
cids['gras'] = set(df['CID']) - {0}<jupyter_output><empty_output><jupyter_text>Sobel Lab (Weiss 2012, Snitz 2013)<jupyter_code>file_path = DATA / 'snitz_2013' / 'snitz.csv'
df = pd.read_csv(file_path)
cids['sobel-2013'] = set(df['CID']) - {0}<jupyter_output><empty_output><jupyter_text>Leffingwell<jupyter_code>file_path = DATA / 'westeros' / 'westeros.csv'
df = pd.read_csv(file_path)
cids['leffingwell'] = set(df['CID']) - {0}<jupyter_output><empty_output><jupyter_text>Davison<jupyter_code>file_path = DATA / 'davison_2007' / 'davison-katz.csv'
df = pd.read_csv(file_path, index_col=0)
cids['davison-2007'] = set(df['CID']) - {0}<jupyter_output><empty_output><jupyter_text>FDB<jupyter_code>file_path = DATA / 'fragrancedb' / 'FragranceDB_CIDs.txt'
df = pd.read_csv(file_path)
cids['fragrance-db'] = set(df['CID']) - {0}<jupyter_output><empty_output><jupyter_text>Mainland<jupyter_code>file_path = DATA / 'cabinets' / 'Mainland Odor Cabinet with CIDs.csv'
df = pd.read_csv(file_path)
cids['mainland-cabinet'] = set(df['CID']) - {0}
file_path = DATA / 'mainland_intensity' / 'mainland-intensity-odorant-info.csv'
df = pd.read_csv(file_path)
cids['mainland-intensity'] = set(df['CID']) - {0}
file_path = DATA / 'mainland_2015' / 'Odors.tsv'
df = pd.read_csv(file_path, sep='\t')
cids['mainland-receptors'] = set(df['CID'].dropna().astype(int)) - {0}<jupyter_output><empty_output><jupyter_text>Enantiomers<jupyter_code>file_path = DATA / 'shadmany' / 'enantiomers.csv'
df = pd.read_csv(file_path)
cids['enantiomers'] = set(df['CID']) - {0}<jupyter_output><empty_output><jupyter_text>Haddad (just the clusters)<jupyter_code>file_path = DATA / 'haddad_2008' / 'haddad-clusters.csv'
df = pd.read_csv(file_path)
cids['haddad-2008'] = set(df['CID']) - {0}<jupyter_output><empty_output><jupyter_text>U19 PIs<jupyter_code>from rickpy import get_sheet
gerkin_sheet = '1PlU4zHyRXtcI7Y-O6xYtlIyKoKk8hX1I9zfx8KFELdc'
u19_sheet = '1B2sEj9pCk2_zS1X1Cg2ulAB4E_BWPboJBSvH4Gwc8fs'
dfs = {}
dfs['gerkin-cabinet'] = get_sheet(gerkin_sheet, 'gerkin-compounds').set_index('CID')
dfs['smith-cabinet'] = get_sheet(gerkin_sheet, 'smith-compounds').set_index('CID')
dfs['rinberg-glomeruli'] = get_sheet(u19_sheet, 'rinberg').set_index('CID')
dfs['fleischmann-cabinet'] = get_sheet(u19_sheet, 'fleischmann').set_index('CID')
dfs['datta-cabinet'] = get_sheet(u19_sheet, 'datta').set_index('CID')
dfs['bozza-cabinet'] = get_sheet(u19_sheet, 'bozza').set_index('CID')
for name, df in dfs.items():
    cids[name] = set(df.index) - {0}<jupyter_output><empty_output><jupyter_text>Goodscents<jupyter_code>file_path = DATA / 'goodscents' / 'goodscents_cids.txt'
df = pd.read_csv(file_path, index_col=False)
cids['goodscents'] = set(df['CID']) - {0}<jupyter_output><empty_output><jupyter_text>Arctander<jupyter_code>file_path = DATA / 'arctander_1960' / 'arctander_cids.txt'
df = pd.read_csv(file_path, index_col=False)
cids['arctander-1960'] = set(df['CID']) - {0}<jupyter_output><empty_output><jupyter_text>Flavornet<jupyter_code>file_path = DATA / 'flavornet' / 'flavornet.csv'
df = pd.read_csv(file_path)
cids['flavornet'] = set(df['CID']) - {0}<jupyter_output><empty_output><jupyter_text>Scott et al, 2014<jupyter_code>file_path = DATA / 'scott_2014' / 'data.csv'
df = pd.read_csv(file_path)
cids['scott-2014'] = set(df['CID']) - {0}<jupyter_output><empty_output><jupyter_text>Superscent<jupyter_code>file_path = DATA / 'superscent' / 'superscent_cids.txt'
df = pd.read_csv(file_path)
cids['superscent'] = set(df['CID']) - {0}<jupyter_output><empty_output><jupyter_text>SenseLab<jupyter_code>file_path = DATA / 'senselab' / 'senselab.csv'
df = pd.read_csv(file_path)
cids['senselab'] = set(df['CID']) - {0}
file_path = DATA / 'wakayama_2019' / 'wakayama-intensity_with-CIDs.txt'
df = pd.read_csv(file_path, sep='\t')
cids['wakayama-2019'] = set(df['CID']) - {0}<jupyter_output><empty_output><jupyter_text>Save<jupyter_code>file_path = DATA / 'odorants' / 'cids.pkl'
with open(file_path, 'wb') as f:
    pickle.dump(cids, f)<jupyter_output><empty_output><jupyter_text>Load<jupyter_code>#with open(file_path, 'rb') as f:
#    cids2 = pickle.load(f)<jupyter_output><empty_output><jupyter_text>Merge<jupyter_code>all_cids = set()
for key in cids:
    all_cids |= cids[key]
all_cids = pd.DataFrame(index=sorted(list(all_cids)), columns=sorted(list(cids))).fillna(0)
all_cids.index.name = 'CID'
for key in cids:
    all_cids.loc[list(cids[key]), key] = 1
file_path = DATA / 'odorants' / 'all_cids.csv'
all_cids.to_csv(file_path)
all_cids.shape<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1738634511976536, 'fraction_numerical': 0.029656183803161155, 'mean_word_length': 7.5966386554621845, 'pattern_counts': {'":': 0, '<': 97, '<?xml version=': 0, '>': 97, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '24', 'dir': 'jupyter-structured-clean-dedup', 'id': '219384', 'license': 'MIT', 'n_tokens_mistral': 2839, 'n_tokens_neox': 2674, 'n_words': 355, 'path': 'notebooks/cids_from_sources.ipynb', 'repo_name': 'brilee/pyrfume', 'size': '6137'}