<jupyter_start><jupyter_text>Trim low abundance reads<jupyter_code>!  trim-low-abund.py --help
# ! trim-low-abund.py --cutoff 3 --trim-at-coverage 18 --variable-coverage --summary-info json --max-memory-usage 4e9 *fastq.gz<jupyter_output><empty_output><jupyter_text>Need to reorder reads by pairs<jupyter_code>! extract-paired-reads.py
ls -lha<jupyter_output>total 149G
drwxrwxrwx 13 ubuntu ubuntu  12K May 15 15:52 [0m[34;42m.[0m/
drwxrwxrwx  3 ubuntu ubuntu 4.0K Apr 24 23:25 [34;42m..[0m/
drwxrwxr-x  3 ubuntu ubuntu 4.0K May 13 19:21 [01;34m293[0m/
drwxrwxr-x  2 ubuntu ubuntu 4.0K May 13 19:22 [01;34mkhmer_countgraph[0m/
drwxrwxr-x  2 ubuntu ubuntu 4.0K May 13 19:22 [01;34mkhmer_k32[0m/
-rw-rw-rw-  1 ubuntu ubuntu 4.0K Apr  6 21:43 metadata.csv
-rw-rw-rw-  1 ubuntu ubuntu  28K Apr  6 21:43 metadata.json
drwxrwxr-x  2 ubuntu ubuntu 4.0K May 13 19:19 [01;34mMG[0m/
drwxrwxr-x  2 ubuntu ubuntu 4.0K May 13 19:19 [01;34mOS[0m/
-rw-rw-r--  1 ubuntu ubuntu  77M Feb 23 22:09 [01;31msratoolkit.tar.gz[0m
-rw-rw-rw-  1 ubuntu ubuntu 2.5G Apr  6 21:42 [01;31mSRR4098609_pass_1.fastq.gz[0m
-rw-rw-r--  1 ubuntu ubuntu 332K May 15 15:52 SRR4098609_pass_1.fastq.gz.abundtrim
-rw-rw-rw-  1 ubuntu ubuntu 2.5G Apr  6 21:42 [01;31mSRR4098609_pass_2.fastq.gz[0m
-rw-rw-r--  1 ubuntu ubuntu 5.6G May 15 01:48 SRR4098609_pass_2.fast[...]<jupyter_text>Calculate RNA signatures<jupyter_code>for cell_line, df in metadata.groupby('cell_line'):
    print('----', cell_line, df.shape, '----')
    # Sanitize the cell line name
    cell_line = cell_line.replace(' ', '_')

    # Use previously calculated countgraphs
    read1 = df.acc + "_pass_1.fastq.gz"
    read2 = df.acc + "_pass_2.fastq.gz"
    reads = ' '.join(read1) + ' ' + ' '.join(read2)
    khmer_folder = f"/mnt/data/khmer_k32/{cell_line}"
    khmer_countgraph = f'{khmer_folder}/khmer_countgraph'
    ! trim-low-abund.py --cutoff 3 --trim-at-coverage 18 --variable-coverage --max-memory-usage 64e9 --loadgraph $khmer_countgraph $reads
        
    trimmed_read1 = df.acc + "_pass_1.fastq.gz.abundtrim"
    trimmed_read2 = df.acc + "_pass_2.fastq.gz.abundtrim"
    trimmed_reads = ' '.join(trimmed_read1) + ' ' + ' '.join(trimmed_read2)
    
    output_folder = f"/mnt/data/sourmash/{cell_line}"
    output_file = f'{output_folder}/sourmash_rna_signature'
    

#     print(output_file)
    ! mkdir -p $output_folder
    # ksize of 32 is the default
    ! sourmash compute --track-abundance --merge $cell_line --output $output_file $trimmed_reads<jupyter_output>---- A-431 (2, 18) ----

|| This is the script trim-low-abund.py in khmer.
|| You are running khmer version 2.1.1
|| You are also using screed version 1.0
||
|| If you use this script in a publication, please cite EACH of the following:
||
||   * MR Crusoe et al., 2015. http://dx.doi.org/10.12688/f1000research.6924.1
||   * <NAME>, <NAME>, <NAME>, https://dx.doi.org/10.7287/peerj.preprints.890v1
||
|| Please see http://khmer.readthedocs.io/en/latest/citations.html for details.


*** WARNING: You have asked that the graph size be automatically calculated
*** (by using -U or -M).
*** But you are loading an existing graph!
*** Size will NOT be set automatically.

PARAMETERS:
 - kmer size =     32 		(-k)
 - n tables =      4 		(-N)
 - max tablesize = 1.6e+10 	(-x)
Estimated memory usage is 64.0 Gb (6.4e+10 bytes = 4 bytes x 1.6e+10 entries / 1 entries per byte)
--------
loading countgraph from /mnt/data/khmer_k32/A-431/khmer_countgraph
created temporary directory ./tmpugw3rwapkhmer;
use[...]<jupyter_text>Calculate protein signaturesAlso calculate protein signatures, but ksizes must be divisible by 3<jupyter_code>for cell_line, df in metadata.groupby('cell_line'):
    print('----', cell_line, df.shape, '----')
    read1 = df.acc + "_pass_1.fastq.gz"
    read2 = df.acc + "_pass_2.fastq.gz"
    reads = ' '.join(read1) + ' ' + ' '.join(read2)
    
    # Sanitize the cell line name
    cell_line = cell_line.replace(' ', '_')
    output_folder = f"/mnt/data/sourmash/{cell_line}"
    output_file = f'{output_folder}/sourmash_protein_signature'
#     print(output_file)
    ! mkdir -p $output_folder
    # ksize of 32 is the default
    ! sourmash compute --track-abundance --merge $cell_line --output $output_file --ksizes 21, 33, 51 --protein $reads<jupyter_output><empty_output><jupyter_text>Look at data folders<jupyter_code>ls -lha /mnt/data/sourmash/*/*
cat /mnt/data/sourmash/A-431/sourmash_rna_signature<jupyter_output>[{"class":"sourmash_signature","email":"","filename":"SRR629559_pass_2.fastq.gz.abundtrim","hash_function":"0.murmur64","license":"CC0","name":"A-431","signatures":[{"abundances":[1,1,1,10,1,1,1,1,1,1,6,1,2,3,2,1,1,4,1,202,1,1,1,1,3,22,1,1,1,1,5,1,1,1,3,1,1,1,1,8,1,1,1,3,1,2,1,1,1,1,1,6,1,1,1,1,61,2,16,1,392,1,1,1,1,1,1,83,1,1,1,25,10,3,1,1,1,1,2,1,1,1,1,1,14,1,90,1,1,1,102,1,1,1,1,1,2,1,1,1,2,1,1,1,11,1,1,1,1,1,1,1,1,1,2,1,1,1,1,1,2,1,1,15,1,1,1,1,1,9,1,5,1,1,1,1,5,1,3,3,1,1,1,1,1,52,12,1,53,1,2,1,1,2,1,1,6,1,1,1,1,1,1,1,1,1,1,1,8,1,1,1,1,156,1,1,1,2,2,1,1,1,1,1,1,3,1,1,1,7,1,74,1,1,2,1,1,1,10,1,1,641,1,1,1,3,1,1,1,2,1,5,1,3,1,1,1,5,1,1,1,7,1,1,1,10,1,1,18,2,1,1,1,1,1,32,1,1,1,1,2,7,1,6,1,2,1,1,116,27,1,1,1,19,1,1,1,1,8,1,1,1,406,1,1,1,1,2,1,1,1,2,1,1,1,1,1,1,1,55,1,1,1,1,1,1,3,1,2,11,1,57,1,1,8,1,1,1,1,2,9,1,56,3,1,1,1,2,1,1,1,1,1,1197,1,8,1,1,1,1,1,1,4,1,1,425,1,1,1,1,7,4,1,1,1,1,1,1,2,1,1,1,1,1,1,5,1,1,1,1,103,1,1,646,1,1,1,21,1,1,2,1,1,4,1,1,20,1,4,1,1,1,1,1,1,4,1,1,1,5,16,1,88,1,[...]
{'fraction_non_alphanumeric': 0.19796029867055182, 'fraction_numerical': 0.14150427973046803, 'mean_word_length': 4.421520236920039, 'pattern_counts': {'":': 8, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '156858', 'license': 'MIT', 'n_tokens_mistral': 2963, 'n_tokens_neox': 2606, 'n_words': 414, 'path': 'notebooks/005_sourmash.ipynb', 'repo_name': 'czbiohub/sequence-localization', 'size': '5494'}