<jupyter_start><jupyter_text>Fingerprint moleculesThe whole set of fingerprints won't fit in memory (even sparse) so we have to save them as chunks. This iterates over the SMILES codes, generating fingerprint_matrices and score arrays, saving them as chunks of 10,000,000<jupyter_code>from rdkit import Chem
from rdkit.Chem import rdMolDescriptors
from scipy import sparse

def makeChunk(fileobj, chunkSize, outFileName, fingerprint_function, fpSize=8192):    
    #store bit indices in these:
    row_idx = list()
    col_idx = list()
    count=0
    
    scores = list()
    
    #iterate through file, 
    for line in tqdm.tqdm_notebook(fileobj, total=chunkSize-1, smoothing=0):
        if line=='': #end of file
            break
        
        words = line[17:-1].split(',') #removes the zinc ID and trailing newline   
        if len(words[1])<1:
            continue
        if words[1]=='no_score':
            break
        else:
            try:
                mol = Chem.MolFromSmiles(words[0])
                fp = fingerprint_function(mol, **pars)
                onbits = list(fp.GetOnBits())
                #these bits all have the same row:
                row_idx += [count]*len(onbits)
                count+=1
                #and the column indices of those bits:
                col_idx+=onbits
            
                scores.append(float(words[1]))

            except KeyboardInterrupt:
                raise
            except:
                print('failed molecule')
                
        if count>chunkSize:
            break
                
    #if we reach the chunkSize:
    #generate a sparse matrix out of the row,col indices:
    unfolded_size = 8192
    fingerprint_matrix = sparse.coo_matrix((np.ones(len(row_idx)).astype(bool), (row_idx, col_idx)), 
                          shape=(max(row_idx)+1, unfolded_size))
    #convert to csr matrix, it is better:
    fingerprint_matrix =  sparse.csr_matrix(fingerprint_matrix)
    
    sparse.save_npz(outFileName+'.npz', fingerprint_matrix)
    np.save(outFileName+'.npy', np.array(scores) )<jupyter_output><empty_output><jupyter_text>Count number of valid molecules:<jupyter_code>fname = '../data/table_name_smi_energy_hac_lte_25_title.csv'
fileobj = open(fname)
fileobj.readline()


count = 0
for line in fileobj:
    words = line[:-1].split(',')
    if len(words[2])<1:
        continue
    if words[2]=='no_score':
        break
    count+=1
fileobj.close()
count // (10*1e6)
chunksize = 10_000_000
fingerprint_function = rdMolDescriptors.GetMorganFingerprintAsBitVect
pars = { "radius": 2,
                     "nBits": 8192,
                     "invariants": [],
                     "fromAtoms": [],
                     "useChirality": False,
                     "useBondTypes": True,
                     "useFeatures": True,
            }


fname = '../data/table_name_smi_energy_hac_lte_25_title.csv'
fileobj = open(fname)
fileobj.readline()


for i in range( np.ceil(count / chunksize).astype(int) ):
    makeChunk(fileobj, chunksize, '../processed_data/D4_all'+str(i), fingerprint_function)<jupyter_output><ipython-input-4-3807d4261a91>:14: TqdmDeprecationWarning: This function will be removed in tqdm==5.0.0
Please use `tqdm.notebook.tqdm` instead of `tqdm.tqdm_notebook`
  for line in tqdm.tqdm_notebook(fileobj, total=chunkSize-1, smoothing=0):
{'fraction_non_alphanumeric': 0.1042287075640262, 'fraction_numerical': 0.022334723049434187, 'mean_word_length': 2.4066937119675456, 'pattern_counts': {'":': 7, '<': 11, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '102245', 'license': 'MIT', 'n_tokens_mistral': 1135, 'n_tokens_neox': 1047, 'n_words': 247, 'path': 'code/D4_all_generate_FPs.ipynb', 'repo_name': 'TalusBio/dockop', 'size': '3358'}