<commit_before>#!/usr/bin/env/python
# Merges all fastq_files from each samples into one file. 
# ARGV 1 Base directory with files to be merged
# ARGV 2 Destination dir of the merged output files

import re
import os
import sys
import shutil
def merge_files(dest_dir, fastq_files):
    #Match NGI sample number from flowcell
    sample_pattern=re.compile("^(.+)_S[0-9]+_.+_R([1-2])_")
    #Remove files that already have the right name (i.e have been merged already)
    matches=[]
    for fastq in fastq_files:
        try:
            match=sample_pattern.search(fastq).group(1)
            if match:
                matches.append(fastq)
        except AttributeError: 
            continue
    fastq_files=matches
   
    ## Sort on basename to avoid collison
    basenames=[]
    for sample in matches:
        basenames.append(os.path.basename(sample))
    basenames.sort()
    fastq_files=[x for (y,x) in sorted(zip(basenames,fastq_files))]
    
    while fastq_files:
        tomerge=[]
        try:
            first=fastq_files[0]
            fq_bn=os.path.basename(first)
            sample_name=sample_pattern.match(fq_bn).group(1)
            read_nb=sample_pattern.match(fq_bn).group(2)
        #Avoid crashing if no match
        except AttributeError: 
            continue
        for fq in fastq_files:
            #If fq belongs to the same sample and read, add it to tomerge then remove from file list
            if sample_name in os.path.basename(fq) and "_R{}_".format(read_nb) in os.path.basename(fq):
                tomerge.append(fq)
        for fq in tomerge:
            fastq_files.remove(fq)
        #Save under Pname, i.e P001_001_R1.fastq.gz
        outfile=os.path.join(dest_dir, "{}_R{}.fastq.gz".format(sample_name, read_nb))
        with open(outfile, 'wb') as wfp:
            for fn in tomerge:
                with open(fn, 'rb') as rfp:
                    shutil.copyfileobj(rfp, wfp)

def main():
   input_dir=sys.argv[1]
   destination_dir = sys.argv[2] 
    
   fastq_files=[]
   #gather all fastq files in inputdir and subdirs
   for subdir, dirs, files in os.walk(input_dir):
       for fastq in  files:
           if fastq.endswith('.fastq.gz'):
               fastq_files.append(os.path.join(subdir, fastq))
   
   merge_files(destination_dir,fastq_files)

if __name__ == "__main__":
    main()<commit_msg>Reworked to work on pairs Also added argparse instead of ARGV. Not tested yet.<commit_after>#!/usr/bin/env/python
# Merges all fastq_files from each samples into one file. 
# ARGV 1 Base directory with files to be merged
# ARGV 2 Destination dir of the merged output files

import re
import os
import sys
import shutil
import argparse
import collections
def merge_files(input_dir, dest_dir):
   #Gather all fastq files in inputdir and its subdirs
   fastq_files=[]
   for subdir, dirs, files in os.walk(input_dir):
       for fastq in  files:
           if fastq.endswith('.fastq.gz'):
               fastq_files.append(os.path.join(subdir, fastq))
   
    
    #Match NGI sample number from flowcell
    sample_pattern=re.compile("^(P[0-9]+_[0-9]+)_S[0-9]+_.+_R([1-2])_")
    #Remove files that already have the right name (i.e have been merged already)
    matches=[]
    for fastq_file in fastq_files:
        try:
            match=sample_pattern.search(fastq_file).group(1)
            if match:
                matches.append(fastq_file)
        except AttributeError: 
            continue
    fastq_files=matches
   
    ## Sort on basename to avoid collison
    
    while fastq_files:
        tomerge=[]
        
        #grab one sample to work on
        first=fastq_files[0]
        fq_bn=os.path.basename(first)
        sample_name=sample_pattern.match(fq_bn).group(1)
        fastq_files_read1=[]
        fastq_files_read2=[]
        def getKey(item):
                return item[1]
        #make a list of lists with abs path and basename
        for fq in fastq_files:
            if sample_name in os.path.basename(fq) and "_R1_" in os.path.basename(fq):
                fastq_files_read1.append([fq, os.path.basename(fq)])
                
            if sample_name in os.path.basename(fq) and "_R2_" in os.path.basename(fq):
                fastq_files_read2.append([fq, os.path.basename(fq)])
        #sort both lists on the basename.  
        fastq_files_read1.sort(key=getKey)
        fastq_files_read2.sort(key=getKey)
        #Split the lists
        fastq_files_read1, throw_away = zip(*fastq_files_read1)
        fastq_files_read2, throw_away = zip(*fastq_files_read2)

        actual_merging(sample_name,read_nb, fastq_files_read1)
        actual_merging(sample_name,read_nb, fastq_files_read2)
        #need to give only the abspaths  to the merge function. 
    def actual_merging(sample_name,read_nb, tomerge):
        outfile=os.path.join(dest_dir, "{}_R{}.fastq.gz".format(sample_name, read_nb))
        print "Merging the following files:"
        for x in tomerge:
            print x
            print "as {}".format(outfile) 
        with open(outfile, 'wb') as wfp:
            for fn in tomerge:
                with open(fn, 'rb') as rfp:
                    shutil.copyfileobj(rfp, wfp)
        
        for fq in tomerge:
            fastq_files.remove(fq)

if __name__ == "__main__":
   parser = argparse.ArgumentParser(description=' Merges all fastq-files from each samples into one file. Looks through the given dir and subdirs')
   parser.add_argument("input_dir", metavar='<Path/to/input/files>', nargs='?', default='.'
                                   help="Base directory for the fastq files that should be merged. ")
   parser.add_argument("dest_dir", metavar='<Path/to/destination/dir>', nargs='?', default='.'
                                   help="Path path to where the merged files should be outputed. ")
   kwargs = vars(parser.parse_args()) 
   merge_files((**kwargs)
{'fraction_non_alphanumeric': 0.09736618521665251, 'fraction_numerical': 0.00798640611724724, 'mean_word_length': 2.4380841121495327, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5813177', 'n_tokens_mistral': 1930, 'n_tokens_neox': 1898, 'n_words': 495}