<jupyter_start><jupyter_text>FASTQ QC  Aims of this notebook- Trim the FASTQ by base quality, remove potential illumina adapter that may cause problem in mapping, and filter out short reads- Update the metadata table using new file names  Installation Install cutadapt, fastqc`conda install -n genome_book -c bioconda cutadapt fastqc` Install trim_galoresee [trim galore github](https://github.com/FelixKrueger/TrimGalore) for instruction.```{shell}curl -fsSL https://github.com/FelixKrueger/TrimGalore/archive/0.6.5.tar.gz -o trim_galore.tar.gztar xvzf trim_galore.tar.gz Run Trim Galore./TrimGalore-0.6.5/trim_galore```<jupyter_code>import pandas as pd
import pathlib
import subprocess
fastq_meta = pd.read_csv('./data/fastq/fastq_metadata.csv', index_col=0)
fastq_meta
output_dir = pathlib.Path('./data/fastq/trimmed/').absolute()
# another way to make dir
output_dir.mkdir(exist_ok=True)

fastq_dir = pathlib.Path('./data/fastq/').absolute()<jupyter_output><empty_output><jupyter_text>Run commands sequentially<jupyter_code># change this to your path to trim_galore installation
trim_galore_path = '/Users/hq/Documents/pkg/TrimGalore-0.6.5/trim_galore'

# iterate through each file in the fastq_meta
for file_name in fastq_meta['file_name']:
    file_path = fastq_dir / file_name
    # make the command using f-string
    command = f'{trim_galore_path} {file_path} --fastqc -o {output_dir}'
    # execute the command using subprocess
    subprocess.run(command, shell=True, check=True)
    print(command)
    print(file_name, 'trim finished.')<jupyter_output>/Users/hq/Documents/pkg/TrimGalore-0.6.5/trim_galore /Users/hq/Documents/pkg/py_genome_sci_book/analysis/salmon_demo/data/fastq/forebrain_E11.5_1_ENCFF329ACL.fastq.gz --fastqc -o /Users/hq/Documents/pkg/py_genome_sci_book/analysis/salmon_demo/data/fastq/trimmed
forebrain_E11.5_1_ENCFF329ACL.fastq.gz trim finished.
/Users/hq/Documents/pkg/TrimGalore-0.6.5/trim_galore /Users/hq/Documents/pkg/py_genome_sci_book/analysis/salmon_demo/data/fastq/forebrain_E11.5_2_ENCFF251LNG.fastq.gz --fastqc -o /Users/hq/Documents/pkg/py_genome_sci_book/analysis/salmon_demo/data/fastq/trimmed
forebrain_E11.5_2_ENCFF251LNG.fastq.gz trim finished.
/Users/hq/Documents/pkg/TrimGalore-0.6.5/trim_galore /Users/hq/Documents/pkg/py_genome_sci_book/analysis/salmon_demo/data/fastq/forebrain_E11.5_2_ENCFF896COV.fastq.gz --fastqc -o /Users/hq/Documents/pkg/py_genome_sci_book/analysis/salmon_demo/data/fastq/trimmed
forebrain_E11.5_2_ENCFF896COV.fastq.gz trim finished.
/Users/hq/Documents/pkg/TrimGalore-0.6.5/trim_galore[...]<jupyter_text>[OPTIONAL] Run commands in parallel- When you have multiple samples, it is important to learn how to execute commands in parallel, so you can utilize power of your multi-core laptop or server.<jupyter_code># Based on the number of cores your computer has
process = 4

# this is one way to run things in parallel in python
from concurrent.futures import ProcessPoolExecutor, as_completed
import subprocess
trim_galore_path = '/Users/hq/Documents/pkg/TrimGalore-0.6.5/trim_galore'
with ProcessPoolExecutor(process) as executor:
    #executor is a process pool, it will control the number of processes run in parallel
    
    futures = {}
    for file_name in fastq_meta['file_name']:
        file_path = fastq_dir / file_name
        command = f'{trim_galore_path} {file_path} --fastqc -o {output_dir}'
        
        # executor.submit a job (run a function with certain parameter, here the function is subprocess.run) 
        # and return a future obj. The future obj refer to this job, save it into futures first
        future = executor.submit(subprocess.run, command, shell=True, check=True)
        futures[future] = file_name
    
    # as_complete determine which future is finished
    for future in as_completed(futures):
        # get back the file name associated to this future
        file_name = futures[future]
        
        # this line is important, it check wheter the job finished without error
        # it will also got the job (subprocess.run) return, here that's not important
        _ = future.result()
        
        print(file_name, 'trim finished.')<jupyter_output>forebrain_E13.5_2_ENCFF959PSX.fastq.gz trim finished.
forebrain_E11.5_1_ENCFF329ACL.fastq.gz trim finished.
forebrain_E11.5_2_ENCFF251LNG.fastq.gz trim finished.
forebrain_E11.5_2_ENCFF896COV.fastq.gz trim finished.
forebrain_E13.5_1_ENCFF235DNM.fastq.gz trim finished.
forebrain_E14.5_1_ENCFF460TCF.fastq.gz trim finished.
forebrain_E14.5_1_ENCFF270GKY.fastq.gz trim finished.
forebrain_E14.5_2_ENCFF126IRS.fastq.gz trim finished.
forebrain_E14.5_2_ENCFF748SRJ.fastq.gz trim finished.
forebrain_P0_1_ENCFF037JQC.fastq.gz trim finished.
forebrain_P0_2_ENCFF447EXU.fastq.gz trim finished.
forebrain_P0_2_ENCFF458NWF.fastq.gz trim finished.
forebrain_E15.5_2_ENCFF891HIX.fastq.gz trim finished.
forebrain_E12.5_2_ENCFF203BWA.fastq.gz trim finished.
forebrain_P0_1_ENCFF358MFI.fastq.gz trim finished.
forebrain_E15.5_1_ENCFF179JEC.fastq.gz trim finished.
forebrain_E10.5_1_ENCFF920CNZ.fastq.gz trim finished.
forebrain_E12.5_1_ENCFF294JRP.fastq.gz trim finished.
forebrain_E12.5_1_ENCFF920QAY.fastq.gz t[...]<jupyter_text>Make metadata for trimmed fastq<jupyter_code># find out all the trimmed fastq, make a dict
fastq_list = list(output_dir.glob('*trimmed.fq.gz'))
print(len(fastq_list))
fastq_list[:5]
# replace the fastq path with trimmed ones
fastq_series = pd.Series({i.name.split('_')[3]: str(i) 
                          for i in fastq_list})
fastq_meta['file_name'] = fastq_series
fastq_meta
fastq_meta.to_csv('data/fastq/trimmed/trimmed_fastq_metadata.csv')
!ls -hl data/fastq/trimmed/<jupyter_output>total 90016
-rw-r--r--  1 hq  staff   2.8K Jun  1 09:55 forebrain_E10.5_1_ENCFF320FJX.fastq.gz_trimming_report.txt
-rw-r--r--  1 hq  staff   669K Jun  1 09:55 forebrain_E10.5_1_ENCFF320FJX_trimmed.fq.gz
-rw-r--r--@ 1 hq  staff   633K Jun  1 09:55 forebrain_E10.5_1_ENCFF320FJX_trimmed_fastqc.html
-rw-r--r--  1 hq  staff   352K Jun  1 09:55 forebrain_E10.5_1_ENCFF320FJX_trimmed_fastqc.zip
-rw-r--r--  1 hq  staff   2.8K Jun  1 09:55 forebrain_E10.5_1_ENCFF920CNZ.fastq.gz_trimming_report.txt
-rw-r--r--  1 hq  staff   672K Jun  1 09:55 forebrain_E10.5_1_ENCFF920CNZ_trimmed.fq.gz
-rw-r--r--@ 1 hq  staff   640K Jun  1 09:55 forebrain_E10.5_1_ENCFF920CNZ_trimmed_fastqc.html
-rw-r--r--  1 hq  staff   360K Jun  1 09:55 forebrain_E10.5_1_ENCFF920CNZ_trimmed_fastqc.zip
-rw-r--r--  1 hq  staff   2.9K Jun  1 09:55 forebrain_E10.5_2_ENCFF528EVC.fastq.gz_trimming_report.txt
-rw-r--r--  1 hq  staff   667K Jun  1 09:55 forebrain_E10.5_2_ENCFF528EVC_trimmed.fq.gz
-rw-r--r--@ 1 hq  staff   633K[...]
{'fraction_non_alphanumeric': 0.09783420463032114, 'fraction_numerical': 0.05392083644510829, 'mean_word_length': 4.3568, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 14, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '98956', 'license': 'MIT', 'n_tokens_mistral': 2829, 'n_tokens_neox': 2554, 'n_words': 527, 'path': 'analysis/salmon_demo/step3.fastqTrimAndQC.ipynb', 'repo_name': 'lhqing/py_genome_sci_book', 'size': '6695'}