<commit_before>"""Provide trimming of input reads from Fastq or BAM files.
"""
import os
import sys

from bcbio.utils import (file_exists, append_stem, replace_directory, safe_makedir, splitext_plus)
from bcbio.log import logger
from bcbio.distributed import objectstore
from bcbio.provenance import do
from Bio.Seq import Seq
from bcbio.distributed.transaction import file_transaction
import bcbio.pipeline.datadict as dd

MINIMUM_LENGTH = 25

SUPPORTED_ADAPTERS = {
    "illumina": ["AACACTCTTTCCCT", "AGATCGGAAGAGCG"],
    "truseq": ["AGATCGGAAGAG"],
    "polya": ["AAAAAAAAAAAAA"],
    "nextera": ["AATGATACGGCGA", "CAAGCAGAAGACG"]}

def trim_adapters(data):
    to_trim = [x for x in data["files"] if x is not None]
    logger.info("Trimming low quality ends and read through adapter "
                "sequence from %s." % (", ".join(to_trim)))
    out_dir = safe_makedir(os.path.join(dd.get_work_dir(data), "trimmed"))
    config = dd.get_config(data)
    name = dd.get_sample_name(data)
    return _trim_adapters(to_trim, out_dir, name, config)

def _trim_adapters(fastq_files, out_dir, name, config):
    """
    for small insert sizes, the read length can be longer than the insert
    resulting in the reverse complement of the 3' adapter being sequenced.
    this takes adapter sequences and trims the only the reverse complement
    of the adapter

    MYSEQUENCEAAAARETPADA -> MYSEQUENCEAAAA (no polyA trim)

    """
    quality_format = _get_quality_format(config)
    to_trim = _get_sequences_to_trim(config, SUPPORTED_ADAPTERS)
    out_files = replace_directory(append_stem(fastq_files, "_%s.trimmed" % name), out_dir)
    log_file = "%s_log_cutadapt.txt" % splitext_plus(out_files[0])[0]
    out_files = _cutadapt_trim(fastq_files, quality_format, to_trim, out_files, log_file, config)
    if file_exists(log_file):
        content = open(log_file).read().replace(fastq_files[0], name)
        if len(fastq_files) > 1:
            content = content.replace(fastq_files[1], name)<commit_msg>Skip trimming if using prealigned BAM files.<commit_after>"""Provide trimming of input reads from Fastq or BAM files.
"""
import os
import sys

from bcbio.utils import (file_exists, append_stem, replace_directory,
                         safe_makedir, splitext_plus)
from bcbio.bam.fastq import is_fastq
from bcbio.log import logger
from bcbio.distributed import objectstore
from bcbio.provenance import do
from Bio.Seq import Seq
from bcbio.distributed.transaction import file_transaction
import bcbio.pipeline.datadict as dd

MINIMUM_LENGTH = 25

SUPPORTED_ADAPTERS = {
    "illumina": ["AACACTCTTTCCCT", "AGATCGGAAGAGCG"],
    "truseq": ["AGATCGGAAGAG"],
    "polya": ["AAAAAAAAAAAAA"],
    "nextera": ["AATGATACGGCGA", "CAAGCAGAAGACG"]}

def trim_adapters(data):
    to_trim = [x for x in data["files"] if x is not None and is_fastq(x)]
    if not to_trim:
        return data["files"]

    logger.info("Trimming low quality ends and read through adapter "
                "sequence from %s." % (", ".join(to_trim)))
    out_dir = safe_makedir(os.path.join(dd.get_work_dir(data), "trimmed"))
    config = dd.get_config(data)
    name = dd.get_sample_name(data)
    return _trim_adapters(to_trim, out_dir, name, config)

def _trim_adapters(fastq_files, out_dir, name, config):
    """
    for small insert sizes, the read length can be longer than the insert
    resulting in the reverse complement of the 3' adapter being sequenced.
    this takes adapter sequences and trims the only the reverse complement
    of the adapter

    MYSEQUENCEAAAARETPADA -> MYSEQUENCEAAAA (no polyA trim)

    """
    quality_format = _get_quality_format(config)
    to_trim = _get_sequences_to_trim(config, SUPPORTED_ADAPTERS)
    out_files = replace_directory(append_stem(fastq_files, "_%s.trimmed" % name), out_dir)
    log_file = "%s_log_cutadapt.txt" % splitext_plus(out_files[0])[0]
    out_files = _cutadapt_trim(fastq_files, quality_format, to_trim, out_files, log_file, config)
    if file_exists(log_file):
        content = open(log_file).read().replace(fastq_files[0], name)
        if len(fastq_files) > 1:
            content = content.replace(fastq_files[1], name)
{'fraction_non_alphanumeric': 0.10011976047904192, 'fraction_numerical': 0.0038323353293413173, 'mean_word_length': 4.279393173198483, 'pattern_counts': {'":': 8, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4179810', 'n_tokens_mistral': 1446, 'n_tokens_neox': 1394, 'n_words': 366}