<commit_before>__author__ = 'olga'

from glob import iglob, glob
from gscripts.qtools._Submitter import Submitter
import sys
import os

try:
    os.mkdir('filtered/')
except OSError:
    pass

commands = []
for filename in iglob('*.fastq.gz'):
    #TODO: the -l argument "20" should be a % of read length
    commands.append('echo {}; zcat {0} | fastx_artifacts_filter | '
                    'fastq_quality_trimmer -l 20 -t 30 | '
                    'fastq_quality_filter -q 30 -p 90 -z '
                    '> filtered/{0}'.format(filename))


def submit_and_write(name, command_list):
    sub = Submitter(queue_type='PBS', sh_file=name + '.sh',
                    command_list=command_list,
                    job_name=name)

    sub.write_sh(submit=True, nodes=1, ppn=2, queue='home', array=True,
                 max_running=20, walltime='1:00:00')


submit_and_write('fastx_filter', commands)<commit_msg>fix string formatting for fastx<commit_after>__author__ = 'olga'

from glob import iglob, glob
from gscripts.qtools._Submitter import Submitter
import sys
import os

try:
    os.mkdir('filtered/')
except OSError:
    pass

commands = []
for filename in iglob('*.fastq.gz'):
    #TODO: the -l argument "20" should be a % of read length
    commands.append('echo {0}; zcat {0} | fastx_artifacts_filter | '
                    'fastq_quality_trimmer -l 20 -t 30 | '
                    'fastq_quality_filter -q 30 -p 90 -z '
                    '> filtered/{0}'.format(filename))


def submit_and_write(name, command_list):
    sub = Submitter(queue_type='PBS', sh_file=name + '.sh',
                    command_list=command_list,
                    job_name=name)

    sub.write_sh(submit=True, nodes=1, ppn=2, queue='home', array=True,
                 max_running=20, walltime='1:00:00')


submit_and_write('fastx_filter', commands)
{'fraction_non_alphanumeric': 0.1212777476989713, 'fraction_numerical': 0.023280996210070383, 'mean_word_length': 2.673956262425447, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4533167', 'n_tokens_mistral': 649, 'n_tokens_neox': 592, 'n_words': 144}