<jupyter_start><jupyter_text>Filter using WCVP<jupyter_code>rm_char='[]()'
for char in rm_char:
    bold_df['sci_name'] = bold_df['sci_name'].str.replace(char,'')
print('sending',bold_df.sci_name.nunique(),'species names to WCVP_taxo')
bold_df.groupby('sci_name').head(1).sci_name.to_csv('BOLD_species.csv',index=False)
bold_df.columns
print('running wcvp_taxo',end='...')
print(os.system('python ../../PAFTOL_DB/wcvp_taxo.py ../../PAFTOL_DB/wcvp_v5_jun_2021.txt ' + \
          'BOLD_species.csv' + ' -g -s similarity_genus -d divert_genusOK'))
wcvp = pd.read_csv('BOLD_species' + '_wcvp.csv')
wcvp = wcvp[wcvp.sci_name.notnull()]
print('found',wcvp.sci_name.nunique(),'species in WCVP')
print(bold_df.shape[0],end=' > ')
bold_df = pd.merge(bold_df.rename(columns={'sci_name':'Ini_sci_name'}).drop(columns=['family','genus']),
                   wcvp,how='inner',on='Ini_sci_name')
print(bold_df.shape[0])
print('f:',bold_df.family.nunique(),'g:',bold_df.genus.nunique(),'s:',bold_df.sci_name.nunique())
rm_char='[]()×'
for char in rm_char:
    bold_df['sci_name'] = bold_df['sci_name'].str.replace(char,'')<jupyter_output>C:\Users\kle10kg\AppData\Roaming\Python\Python37\site-packages\ipykernel_launcher.py:3: FutureWarning: The default value of regex will change from True to False in a future version. In addition, single character regular expressions will*not* be treated as literal strings when regex=True.
  This is separate from the ipykernel package so we can avoid doing imports until<jupyter_text>Save to fasta<jupyter_code>def writefasta(rec_df, marker):
    rec_fasta=[]
    for idx, row in rec_df.iterrows():
        record = SeqRecord(Seq(row.Seq))
        record.id = str(row.Locus)
        record.description = ';' \
        + 'f=' + row.family + ',g=' + row.genus + ',s=' + row.sci_name + ',ini_s=' + row.Ini_sci_name + ';'
        rec_fasta.append(record)
#     return rec_fasta
    SeqIO.write(rec_fasta,'BOLD_' + marker + '.fasta',format='fasta')
marker = 'rbcLa'
bold_marker=bold_df[bold_df.markercode=='rbcL']
print('f:',bold_marker.family.nunique(),'g:',bold_marker.genus.nunique(),'s:',bold_marker.sci_name.nunique())
print(bold_marker.Len.quantile([0.01, 0.05, .1, .5, 0.9, 0.95, 0.99]).to_dict())
bold_marker.Len.hist(bins=50);
print(bold_marker.shape[0],end=' > ')
bold_marker = bold_marker[(bold_marker.Len>=520) & (bold_marker.Len<=580)]
print(bold_marker.shape[0],end=' > ')
bold_marker = bold_marker.sort_values('family').groupby('sci_name').head(2)
print(bold_marker.shape[0])
print(bold_marker.Len.quantile([0.01, 0.05, .1, .5, 0.9, 0.95, 0.99]).to_dict())
writefasta(bold_marker,marker)
bold_marker[['Locus','Len',
          'sci_name', 'kew_id','family', 'genus', 'species', 'infraspecies', 'Duplicates',
          'Ini_sci_name']].to_csv('BOLD_' + marker + '_TAXO.csv',index=False)
print('f:',bold_marker.family.nunique(),'g:',bold_marker.genus.nunique(),'s:',bold_marker.sci_name.nunique())
marker = 'rbcL'
bold_marker=bold_df[bold_df.markercode==marker]
print('f:',bold_marker.family.nunique(),'g:',bold_marker.genus.nunique(),'s:',bold_marker.sci_name.nunique())
print(bold_marker.Len.quantile([0.01, 0.05, .1, .5, 0.9, 0.95, 0.99]).to_dict())
bold_marker.Len.hist(bins=50);
print(bold_marker.shape[0],end=' > ')
bold_marker = bold_marker[(bold_marker.Len>=1200) & (bold_marker.Len<=1500)]
print(bold_marker.shape[0],end=' > ')
bold_marker = bold_marker.sort_values('family').groupby('sci_name').head(2)
print(bold_marker.shape[0])
print(bold_marker.Len.quantile([0.01, 0.05, .1, .5, 0.9, 0.95, 0.99]).to_dict())
writefasta(bold_marker,marker)
bold_marker[['Locus','Len',
          'sci_name', 'kew_id','family', 'genus', 'species', 'infraspecies', 'Duplicates',
          'Ini_sci_name']].to_csv('BOLD_' + marker + '_TAXO.csv',index=False)
print('f:',bold_marker.family.nunique(),'g:',bold_marker.genus.nunique(),'s:',bold_marker.sci_name.nunique())
marker = 'matK'
bold_marker=bold_df[bold_df.markercode==marker]
print('f:',bold_marker.family.nunique(),'g:',bold_marker.genus.nunique(),'s:',bold_marker.sci_name.nunique())
print(bold_marker.Len.quantile([0.01, 0.05, .1, .5, 0.9, 0.95, 0.99]).to_dict())
bold_marker.Len.hist(bins=50);
print(bold_marker.shape[0],end=' > ')
bold_marker = bold_marker[(bold_marker.Len>=500) & (bold_marker.Len<=1000)]
print(bold_marker.shape[0],end=' > ')
bold_marker = bold_marker.sort_values('family').groupby('sci_name').head(2)
print(bold_marker.shape[0])
print(bold_marker.Len.quantile([0.01, 0.05, .1, .5, 0.9, 0.95, 0.99]).to_dict())
writefasta(bold_marker,marker)
bold_marker[['Locus','Len',
          'sci_name', 'kew_id','family', 'genus', 'species', 'infraspecies', 'Duplicates',
          'Ini_sci_name']].to_csv('BOLD_' + marker + '_TAXO.csv',index=False)
print('f:',bold_marker.family.nunique(),'g:',bold_marker.genus.nunique(),'s:',bold_marker.sci_name.nunique())
marker = 'ITS2'
bold_marker=bold_df[bold_df.markercode==marker]
print('f:',bold_marker.family.nunique(),'g:',bold_marker.genus.nunique(),'s:',bold_marker.sci_name.nunique())
print(bold_marker.Len.quantile([0.01, 0.05, .1, .5, 0.9, 0.95, 0.99]).to_dict())
bold_marker.Len.hist(bins=50);
print(bold_marker.shape[0],end=' > ')
bold_marker = bold_marker[(bold_marker.Len>=170) & (bold_marker.Len<=280)]
print(bold_marker.shape[0],end=' > ')
bold_marker = bold_marker.sort_values('family').groupby('sci_name').head(2)
print(bold_marker.shape[0])
print(bold_marker.Len.quantile([0.01, 0.05, .1, .5, 0.9, 0.95, 0.99]).to_dict())
writefasta(bold_marker,marker)
bold_marker[['Locus','Len',
          'sci_name', 'kew_id','family', 'genus', 'species', 'infraspecies', 'Duplicates',
          'Ini_sci_name']].to_csv('BOLD_' + marker + '_TAXO.csv',index=False)
print('f:',bold_marker.family.nunique(),'g:',bold_marker.genus.nunique(),'s:',bold_marker.sci_name.nunique())
marker = 'trnH-psbA'
bold_marker=bold_df[bold_df.markercode==marker]
print('f:',bold_marker.family.nunique(),'g:',bold_marker.genus.nunique(),'s:',bold_marker.sci_name.nunique())
print(bold_marker.Len.quantile([0.01, 0.05, .1, .5, 0.9, 0.95, 0.99]).to_dict())
bold_marker.Len.hist(bins=50);
print(bold_marker.shape[0],end=' > ')
bold_marker = bold_marker[(bold_marker.Len>=200) & (bold_marker.Len<=650)]
print(bold_marker.shape[0],end=' > ')
bold_marker = bold_marker.sort_values('family').groupby('sci_name').head(2)
print(bold_marker.shape[0])
print(bold_marker.Len.quantile([0.01, 0.05, .1, .5, 0.9, 0.95, 0.99]).to_dict())
writefasta(bold_marker,marker)
bold_marker[['Locus','Len',
          'sci_name', 'kew_id','family', 'genus', 'species', 'infraspecies', 'Duplicates',
          'Ini_sci_name']].to_csv('BOLD_' + marker + '_TAXO.csv',index=False)
print('f:',bold_marker.family.nunique(),'g:',bold_marker.genus.nunique(),'s:',bold_marker.sci_name.nunique())<jupyter_output>f: 191 g: 1176 s: 3205
{0.01: 196.0, 0.05: 236.0, 0.1: 269.0, 0.5: 431.0, 0.9: 618.0, 0.95: 640.0, 0.99: 822.0}
8370 > 7942 > 4767
{0.01: 217.0, 0.05: 241.3, 0.1: 274.0, 0.5: 428.0, 0.9: 600.4000000000005, 0.95: 620.0, 0.99: 641.0}
f: 188 g: 1135 s: 3016
{'fraction_non_alphanumeric': 0.2147317627022424, 'fraction_numerical': 0.05322168606301448, 'mean_word_length': 6.279958677685951, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '352865', 'license': 'Apache-2.0', 'n_tokens_mistral': 3158, 'n_tokens_neox': 2895, 'n_words': 357, 'path': 'Barcode_Databases/Processing_BOLD.ipynb', 'repo_name': 'RBGKew/PAFTOL_Validation_Pipeline', 'size': '7046'}