<jupyter_start><jupyter_text>Analysis of the covid moonshot rationale`COVID_moonshot_submissions` folder is the gitrepo https://github.com/postera-ai/COVID_moonshot_submissions<jupyter_code>gitrepo_folder = 'textual-analysis-of-COVID-Moonshot'
import os

import pandas as pd

submission_filename = '../COVID_moonshot_submissions/covid_submissions_all_info.csv'

postera = pd.read_csv(submission_filename)
# the table contains multiple rows/submitted compounds per submission.
# Say the CID `ANT-DIA-3c79be55-1` finishes in `-1`
postera = postera.assign(CID_group=postera.CID.str.extract(r'(.*)\-\d+'))
minpostera = postera.drop_duplicates('CID_group')<jupyter_output><empty_output><jupyter_text>Bag of wordsMaking a bag of words for a word cloud and to see which terms are enriched<jupyter_code>import re
from collections import Counter

word_block = ' '.join(minpostera.rationale) + ' '.join(minpostera['Submission Notes'].astype(str))
word_block = word_block.replace('by eye', 'by-eye').replace("n't", '')
words = re.findall('[\w\-]+', word_block.lower())
wordbag = Counter(words)
wordbag['fragalysis']
# common words (`the`, `of` etc.) need to be remove from the wordbag

import requests

r = requests.get('https://gist.githubusercontent.com/deekayen/4148741/raw/98d35708fa344717d8eee15d11987de6c8e26d7d/1-1000.txt')

common_words = set(r.text.split('\n')) - {'molecule', 'machine', 'dock', 'learn'}

# shoddy way of expanding it.
# `againsted` is not a real word, but it's presence does not harm anyone.
expanded_common_words = set(list(common_words) + \
                            [w+'s' for w in common_words] + \
                            [w+'es' for w in common_words] + \
                            [w+'ed' for w in common_words] + \
                            [w+'ing' for w in common_words] + \
                            [w+'ly' for w in common_words] + \
                            ['http', 'https', 'com', 'org', 'www']
                           )
## remove them

for k in expanded_common_words.intersection(wordbag.keys()):
    del wordbag[k]
    
for k in list(wordbag.keys()):
    if k.isdigit() or len(k) < 3:
        del wordbag[k]
wordbag.most_common()
# make word cloud
with open('wordcloud.txt', 'w') as w:
    w.write('\n'.join([' '.join([k] * wordbag[k]) for k in wordbag if wordbag[k] > 20 and k != 'nan']))<jupyter_output><empty_output><jupyter_text>Find enrichment<jupyter_code>from typing import Dict

def get_data_on_term(term:str) -> Dict[str, float]:
    if term == 'by-eye':
        term = 'by eye'
    subtable = postera.loc[postera.rationale.str.contains(term, case=False)]
    return {'term': term, **get_data_on_table(subtable)}
    
def get_data_on_table(table:pd.DataFrame) -> Dict[str, float]:
    data = {}
    data['N'] = len(table)
    if data['N'] == 0:
        print(f'Table failed!')
        return data
    # ordered
    ordered = table.ORDERED.value_counts()
    data['N_ordered'] = ordered[True] if True in ordered else 0
    data['N_not_ordered'] = ordered[False] if False in ordered else 0
    data['freq_ordered (of total)'] = round(data['N_ordered'] / data['N'], 4)
    # made
    made = table.MADE.value_counts()
    data['N_made'] = made[True] if True in made else 0
    data['N_not_made'] = made[False] if False in made else 0
    data['freq_made (of total)'] = round(data['N_made'] / data['N'], 4)
    data['freq_made (of ordered)'] = round(data['N_made']/data['N_ordered'], 4) if data['N_ordered'] != 0 else 0
    # xstalised
    xstalised = table.structure_ID.value_counts().sum()
    data['N_crystallised'] = xstalised
    data['N_not_crystallised'] = data['N'] - xstalised
    data['freq_crystallised (of total)'] = round(xstalised/data['N'], 4)
    data['freq_crystallised (of made)'] = round(xstalised/data['N_made'], 4) if data['N_made'] != 0 else 0
    # assayed
    assayed = table.ASSAYED.value_counts()
    data['N_assayed'] = assayed[True] if True in assayed else 0
    data['N_not_assayed'] = assayed[False] if False in assayed else 0
    data['freq_assayed (of total)'] = round(data['N_assayed'] / data['N'], 4)
    data['freq_assayed (of made)'] = round(data['N_assayed']/data['N_made'], 4)  if data['N_made'] != 0 else 0
    # return
    return data
# pd.DataFrame({'term': pd.Series(data = list(wordbag.keys()), dtype=str),
#               'count': pd.Series(data = list(wordbag.values()), dtype=int)})
data = [get_data_on_term(term) for term in wordbag]

term_table = pd.DataFrame(data)
term_table
term_table.to_csv(f'{gitrepo_folder}/term_frequencies.csv')
data.append({'term': '<TALLY>', **get_data_on_table(postera)})
term_table = pd.DataFrame(data)
tally_row = term_table.loc[term_table.term == '<TALLY>'].iloc[0]
tally_row

term_table = term_table[['term', 'N', 
                         'N_ordered', 'N_not_ordered',
                         'N_made', 'N_not_made',
                         'N_assayed', 'N_not_assayed',
                         'N_crystallised', 'N_not_crystallised',
                         'freq_ordered (of total)',
                         'freq_made (of total)', 'freq_assayed (of total)', 'freq_crystallised (of total)',
                         'freq_made (of ordered)', 'freq_assayed (of made)', 'freq_crystallised (of made)'
                        ]]
from scipy.stats import chisquare, nbinom, fisher_exact

new_columns = {}
for field in ('ordered', 'made',  'assayed', 'crystallised'):
    fisherise = lambda row: fisher_exact([[row.N, row[f'N_{field}'] ],
                                          [tally_row.N, tally_row[f'N_{field}'] ]
                                         ])[1] if str(row[f'N_{field}']) != 'nan' else float('nan')
    chiify = lambda row: chisquare(f_obs=[row.N, row[f'N_{field}']],
                                   f_exp=[tally_row.N, tally_row[f'N_{field}'] ]).pvalue
    new_columns[f'fisher_p_{field}'] = term_table.apply(fisherise,axis=1)
    new_columns[f'χ2_p_{field}'] = term_table.apply(chiify,axis=1)

term_table = term_table.assign(**new_columns)
import numpy as np

new_columns = {}
for field in ('freq_ordered (of total)',
              'freq_made (of total)',
              'freq_assayed (of total)', 
              'freq_crystallised (of total)',
              'freq_made (of ordered)', 
              'freq_assayed (of made)',
              'freq_crystallised (of made)'):
    new_columns[f'log2_{field}'] = np.log2(term_table[field].values / tally_row[field])

term_table = term_table.assign(**new_columns)
term_table.to_csv(f'{gitrepo_folder}/terms.csv')
## Crystallised

total = len(term_table)
crystal = term_table.loc[term_table.fisher_p_crystallised < 0.05/total]\
                    .sort_values('freq_crystallised (of total)', ascending=False)\
                    .head(100)
crystal.to_csv(f'{gitrepo_folder}/Top100_terms_enchriched_for_crystallisation.csv')
crystal
## Crystallised 2

total = len(term_table)
crystal = term_table.loc[term_table.fisher_p_crystallised < 0.05/total]\
                    .sort_values('freq_crystallised (of made)', ascending=False)\
                    .head(100)
crystal.to_csv(f'{gitrepo_folder}/Top100_terms_enchriched_for_crystallisation_of_made.csv')
crystal
headers = ['term','N','N_ordered','N_made','N_assayed','N_crystallised','fisher_p_made','fisher_p_crystallised','log2_freq_made (of total)','log2_freq_crystallised (of total)','log2_freq_crystallised (of made)']
term_table.loc[term_table.term.isin(['dock', 'score', 'merge', 'enumerate', 'calculate', 'by eye', 'fragalysis'])][headers]#.to_markdown()
import shutil, os
from git import Repo
shutil.copy('rationale analysis.ipynb', f'{gitrepo_folder}/rationale_analysis.ipynb')

repo = Repo(gitrepo_folder)
repo.index.add(os.listdir(gitrepo_folder))   
repo.index.commit(':truck: data')
repo.remotes.origin.push()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15035860655737704, 'fraction_numerical': 0.012423155737704918, 'mean_word_length': 3.4802065404475044, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 18, 'https://': 2, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '643463', 'license': 'MIT', 'n_tokens_mistral': 2863, 'n_tokens_neox': 2660, 'n_words': 540, 'path': 'rationale_analysis.ipynb', 'repo_name': 'matteoferla/textual-analysis-of-COVID-Moonshot', 'size': '7808'}