<commit_before><commit_msg>Version: 0.8.0 - New FastQ Shuffle Util<commit_after>#! /home/users/cordier/.linuxbrew/bin/python3

if __name__ == "__main__":

    # Imports
    import argparse
    import random
    # Library Imports
    from Bio import SeqIO

    # 
    # Parse Arguments
    # 

    parser = argparse.ArgumentParser()
    parser.add_argument("-i", "--input", type = str, help = "Input Paired-End FastQ File")
    parser.add_argument("-o", "--outputprefix", type = str, help = "Prefix to Output Single End FastQ Files (i.e. prefix.1.fastq and prefix.2.fastq")
    parser.add_argument("-s", "--chunksize", type = int, help = "Size of Chunk (i.e. n Paired Reads) to Shuffle. Default = 50000")
    argsDict = vars(parser.parse_args())

    fastq = argsDict["input"]
    prefix = argsDict["outputprefix"]
    chunksize = argsDict["chunksize"]

    # Assertions for Required Input
    assert (fastq is not None), "No FastQ input provided!"

    #  If No Prefix, Use Same as FASTQ
    if prefix is None: 
        prefix = ".".join(fastq.split(".")[0:-1])
    print(prefix)
    print(fastq)

    # If No Chunk Size, Set Default
    if chunksize is None:
        chunksize = 50000
    assert (chunksize % 2 == 0), "Chunksize must be even!"
    
    # 
    # Conversion
    #

    # Open FastQ, Chunk File, Shuffle, & Write Chunk to Output
    with open(fastq, "r") as handle, open(prefix + ".shuffled.fastq", "w") as output:
        records = SeqIO.parse(handle, "fastq")
        chunk = []
        i = 0
        for record in records:
            try:
                chunk.append([record, next(records)])
                if len(chunk) > chunksize:
                    random.shuffle(chunk) # Inplace
                    flattened = [item for sublist in chunk for item in sublist]
                    SeqIO.write(flattened, output, "fastq")
                    chunk = []
                    i += 1
                    print("Meow: %d Chunks Written (n = %d)" % (i, chunksize))
            except:
                random.shuffle(chunk) # Inplace
                flattened = [item for sublist in chunk for item in sublist]
                SeqIO.write(flattened, output, "fastq")
                chunk = []
                i += 1
                print("Moo: %d Chunks Written (n = %d)" % (i, chunksize))
        # Write Tail
        try:
            random.shuffle(chunk) # Inplace
            flattened = [item for sublist in chunk for item in sublist]
            SeqIO.write(flattened, output, "fastq")
            chunk = []
            i += 1
            print("Moo: %d Chunks Written (n = %d)" % (i, chunksize))
        except:
            pass

else:

    pass
{'fraction_non_alphanumeric': 0.11868214152002995, 'fraction_numerical': 0.008985398727068513, 'mean_word_length': 1.8886486486486487, 'pattern_counts': {'":': 1, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1344460', 'n_tokens_mistral': 850, 'n_tokens_neox': 807, 'n_words': 244}