<commit_before>#author: <NAME>, <EMAIL>

'''
Clean and trim raw Illumina read files
'''

import os
import sys
import glob
import csv
import shutil
import subprocess
import numpy as np
import pandas as pd
from secapr.helpers import FullPaths


def add_arguments(parser):
    parser.add_argument(
        '--input',
        required=True,
        action=FullPaths,
        default=None,
        help='The directory containing the .fastq or .fq files (raw read files). Files can be zipped or unzipped.'
    )
    parser.add_argument(
        '--sample_annotation_file',
        required=True,
        action=FullPaths,
        default=None,
        help='A simple comma-delimited text file containing the sample names in the first column and the name-stem of the corresponding fastq files in the second column (file should not have any column headers).'
    )
    parser.add_argument(
        '--output',
        required=True,
        action=FullPaths,
        default=None,
        help='The output directory where the cleaned reads will be stored.'
    )
    parser.add_argument(
        '--read_min',
        type=int,
        default=400000,
        help='Set the minimum read count threshold. Any sample with fewer reads than this minimum threshold will not be processed further. Default: %(default)s'
    )
    parser.add_argument(
        '--qualified_quality_phred',
        type=int,
        default=20,
        help='Specifies how accurate the match between any adapter etc. sequence must be against a read. For more information see trimmoatic tutorial. Default: %(default)s'
    )
    parser.add_argument(
        '--unqualified_percent_limit',
        type=int,
        default=40,
        help='Set the maximum percent of low-quality nucleotides allowed. Any read with a higher percentage of unqualified (low quality) nucleotides will be discarded. Default: %(default)s'
    )
    parser.add_argument(
        '--cut_window_size',
        type=int,
        default=5,
        help='Set the size of the moving window (in nucleotides) for quality trimming. The window will start moving from the end toward the beginning of the read, applying the quality threshold set with the --cut_mean_quality flag. Default: %(default)s'
    )
    parser.add_argument(
        '--cut_mean_quality',
        type=int,
        default=20,
        help='Set quality threshold for moving window. If the mean quality across the window drops below this threshold, the nucleotides within the window are removed (cut), as well as all trailing nucleotides. Default: %(default)s'
    )
    parser.add_argument(
        '--trim_front',
        type=int,
        default=0,
        help='Remove this number of nucleotides from the beginning of each read. Default: %(default)s'
    )
    parser.add_argument(
        '--trim_tail',
        type=int,
        default=0,
        help='Remove this number of nucleotides from the end of each read. Default: %(default)s'
    )
    parser.add_argument(
        '--required_read_length',
        type=int,
        default=0,
        help='Set this value to only allow reads to pass which are equal to or longer than this threshold. Default: %(default)s'
    )
    parser.add_argument(
        '--disable_complexity_filter',
        action='store_true',
        default=False,
        help='Use this flag if you want to disable the removal of low-complexity reads, e.g. AAAAAAACCCCCCCCAAAAAAAAAAAAAAAAAAGGGGG (activated by default).'
    )
    parser.add_argument(
        '--complexity_threshold',
        type=int,
        default=10,
        help='Reads below this complexity threshold will be removed (consult fastp manual for more explanation). Default: %(default)s'
    )
    parser.add_argument(
        '--disable_poly_g_trimming',
        action='store_true',
        default=False,
        help='Use this flag if you want to disable trimming of G repeats at the end of the read. Poly-G read ends are common when working with very short fragments. (activated by default).'
    )
    parser.add_argument(
        '--poly_g_min_len',
        type=int,
        default=7,
        help='Specifies the length of the nucleotide repeat region at end of read to be trimmed. Default: %(default)s'
    )
    parser.add_argument(
        '--disable_poly_x_trimming',
        action='store_true',
        default=False,
        help='Use this flag if you want to disable trimming of nucleotide repeats of any kind at the end of the read, e.g. poly-A tails (activated by default).'
    )
    parser.add_argument(
        '--poly_x_min_len',
        type=int,
        default=7,
        help='Specifies the length of the nucleotide repeat region at end of read to be trimmed. Default: %(default)s'
    )


def longest_common_substring(s1, s2):
    m = [[0] * (1 + len(s2)) for i in range(1 + len(s1))]
    longest, x_longest = 0, 0
    for x in range(1, 1 + len(s1)):
        for y in range(1, 1 + len(s2)):
            if s1[x - 1] == s2[y - 1]:
                m[x][y] = m[x - 1][y - 1] + 1
                if m[x][y] > longest:
                    longest = m[x][y]
                    x_longest = x
            else:
                m[x][y] = 0
    return s1[x_longest - longest: x_longest]

def read_count(input):
    # Use bash command to count the read number
    number_reads = subprocess.getstatusoutput("gzip -cd %s | wc -l | awk '{ print $1/4 }' " %input)[1]
    num = int(number_reads)
    return num

def quality_trim(read1_path,read2_path,sample_id,out_dir,args,pair_index=0):
    # Get all user settings
    qualified_quality_phred = args.qualified_quality_phred
    unqualified_percent_limit = args.unqualified_percent_limit
    cut_window_size = args.cut_window_size
    cut_mean_quality = args.cut_mean_quality
    trim_front = args.trim_front
    trim_tail = args.trim_tail
    required_read_length = args.required_read_length
    disable_complexity_filter = args.disable_complexity_filter
    complexity_threshold = args.complexity_threshold<commit_msg>fixed longest common substring function<commit_after>#author: <NAME>, <EMAIL>

'''
Clean and trim raw Illumina read files
'''

import os
import sys
import glob
import csv
import shutil
import subprocess
import numpy as np
import pandas as pd
from secapr.helpers import FullPaths


def add_arguments(parser):
    parser.add_argument(
        '--input',
        required=True,
        action=FullPaths,
        default=None,
        help='The directory containing the .fastq or .fq files (raw read files). Files can be zipped or unzipped.'
    )
    parser.add_argument(
        '--sample_annotation_file',
        required=True,
        action=FullPaths,
        default=None,
        help='A simple comma-delimited text file containing the sample names in the first column and the name-stem of the corresponding fastq files in the second column (file should not have any column headers).'
    )
    parser.add_argument(
        '--output',
        required=True,
        action=FullPaths,
        default=None,
        help='The output directory where the cleaned reads will be stored.'
    )
    parser.add_argument(
        '--read_min',
        type=int,
        default=400000,
        help='Set the minimum read count threshold. Any sample with fewer reads than this minimum threshold will not be processed further. Default: %(default)s'
    )
    parser.add_argument(
        '--qualified_quality_phred',
        type=int,
        default=20,
        help='Specifies how accurate the match between any adapter etc. sequence must be against a read. For more information see trimmoatic tutorial. Default: %(default)s'
    )
    parser.add_argument(
        '--unqualified_percent_limit',
        type=int,
        default=40,
        help='Set the maximum percent of low-quality nucleotides allowed. Any read with a higher percentage of unqualified (low quality) nucleotides will be discarded. Default: %(default)s'
    )
    parser.add_argument(
        '--cut_window_size',
        type=int,
        default=5,
        help='Set the size of the moving window (in nucleotides) for quality trimming. The window will start moving from the end toward the beginning of the read, applying the quality threshold set with the --cut_mean_quality flag. Default: %(default)s'
    )
    parser.add_argument(
        '--cut_mean_quality',
        type=int,
        default=20,
        help='Set quality threshold for moving window. If the mean quality across the window drops below this threshold, the nucleotides within the window are removed (cut), as well as all trailing nucleotides. Default: %(default)s'
    )
    parser.add_argument(
        '--trim_front',
        type=int,
        default=0,
        help='Remove this number of nucleotides from the beginning of each read. Default: %(default)s'
    )
    parser.add_argument(
        '--trim_tail',
        type=int,
        default=0,
        help='Remove this number of nucleotides from the end of each read. Default: %(default)s'
    )
    parser.add_argument(
        '--required_read_length',
        type=int,
        default=0,
        help='Set this value to only allow reads to pass which are equal to or longer than this threshold. Default: %(default)s'
    )
    parser.add_argument(
        '--disable_complexity_filter',
        action='store_true',
        default=False,
        help='Use this flag if you want to disable the removal of low-complexity reads, e.g. AAAAAAACCCCCCCCAAAAAAAAAAAAAAAAAAGGGGG (activated by default).'
    )
    parser.add_argument(
        '--complexity_threshold',
        type=int,
        default=10,
        help='Reads below this complexity threshold will be removed (consult fastp manual for more explanation). Default: %(default)s'
    )
    parser.add_argument(
        '--disable_poly_g_trimming',
        action='store_true',
        default=False,
        help='Use this flag if you want to disable trimming of G repeats at the end of the read. Poly-G read ends are common when working with very short fragments. (activated by default).'
    )
    parser.add_argument(
        '--poly_g_min_len',
        type=int,
        default=7,
        help='Specifies the length of the nucleotide repeat region at end of read to be trimmed. Default: %(default)s'
    )
    parser.add_argument(
        '--disable_poly_x_trimming',
        action='store_true',
        default=False,
        help='Use this flag if you want to disable trimming of nucleotide repeats of any kind at the end of the read, e.g. poly-A tails (activated by default).'
    )
    parser.add_argument(
        '--poly_x_min_len',
        type=int,
        default=7,
        help='Specifies the length of the nucleotide repeat region at end of read to be trimmed. Default: %(default)s'
    )


def longest_common_substring(s1, s2):
    m = []
    for i, letter in enumerate(list(s1)):
        if letter == list(s2)[i]:
            m.append(letter)
        else:
            break
    return ''.join(m)

# def longest_common_substring(s1, s2):
#     m = [[0] * (1 + len(s2)) for i in range(1 + len(s1))]
#     longest, x_longest = 0, 0
#     for x in range(1, 1 + len(s1)):
#         for y in range(1, 1 + len(s2)):
#             if s1[x - 1] == s2[y - 1]:
#                 m[x][y] = m[x - 1][y - 1] + 1
#                 if m[x][y] > longest:
#                     longest = m[x][y]
#                     x_longest = x
#             else:
#                 m[x][y] = 0
#     return s1[x_longest - longest: x_longest]

def read_count(input):
    # Use bash command to count the read number
    number_reads = subprocess.getstatusoutput("gzip -cd %s | wc -l | awk '{ print $1/4 }' " %input)[1]
    num = int(number_reads)
    return num

def quality_trim(read1_path,read2_path,sample_id,out_dir,args,pair_index=0):
    # Get all user settings
    qualified_quality_phred = args.qualified_quality_phred
    unqualified_percent_limit = args.unqualified_percent_limit
    cut_window_size = args.cut_window_size
    cut_mean_quality = args.cut_mean_quality
    trim_front = args.trim_front
    trim_tail = args.trim_tail
    required_read_length = args.required_read_length
    disable_complexity_filter = args.disable_complexity_filter
    complexity_threshold = args.complexity_threshold
{'fraction_non_alphanumeric': 0.08983258472846059, 'fraction_numerical': 0.008493262556145366, 'mean_word_length': 2.6155890168290523, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 6, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5357019', 'n_tokens_mistral': 3647, 'n_tokens_neox': 3510, 'n_words': 1176}