<commit_before>#!/usr/bin/env python3


def build_baited_bloom_filter(
    transcriptome,
    kmer,
    bloom_size,
    levels,
    output_bloom,
    threads,
    reads
):
    '''(str, int, str, int, str, int , list of str) -> None

    Run the build_baited_bloom_filter pipeline:
    - Build a secondary Bloom filter of the transcriptome with biobloommaker
    - Categorize reads with biobloomcategorizer and pipe to abyss-bloom build
        to build the primary Bloom filter

    Inputs are:
    - A transcriptome in Fasta format
    - A kmer size (better > 25)
    - The total memory to use in the primary Bloom filter
    - The number of levels to use in the Bloom filter (better to use 2)
    - The output file name for the primary Bloom filter
    - The number of threads to use in construction
    - A list of read filenames
    '''

    # Imports
    from subprocess import Popen, PIPE
    from sys import stderr
    from os.path import dirname

    output_dir = dirname(output_bloom)
    if output_dir == "":
        output_dir = "./"

    # Convert single read library to list
    if isinstance(reads, str):
        reads = [reads]

    # Prepare the commands
    build_transcriptome_bf = [
        'biobloommaker',
        '--file_prefix', "transcriptome",
        '--output_dir', output_dir,
        '--threads', str(threads),
        '--kmer_size', str(kmer),
        transcriptome
    ]

    categorize = [
        'biobloomcategorizer',
        '--prefix', output_dir + '/categories',
        '--filter_files', output_dir + '/transcriptome.bf',
        '--threads', str(threads),
        '--score', str(kmer),
        '--fa',
        '--stdout_filter', 'transcriptome',
    ] + reads

    build_bf = [
        'abyss-bloom', 'build',
        '--verbose',
        '--kmer', str(kmer),
        '--bloom-size', bloom_size,
        '--levels', str(levels),
        '--threads', str(threads),
        output_bloom,
        '/dev/stdin'
    ]

    # Run the pipeline
    stderr.write(
        "\n\nRunning command: {command}\n".format(
            command=" ".join(build_transcriptome_bf)
        )
    )

    p_build_transcriptome_bf = Popen(build_transcriptome_bf, shell=False)

    p_build_transcriptome_bf.wait()

    stderr.write(
        "\n\nRunning commands: {command1} | {command2}\n".format(
            command1=" ".join(categorize),
            command2=" ".join(build_bf)
        )
    )

    p_categorize = Popen(categorize, stdout=PIPE, shell=False)
    p_build_bf = Popen(build_bf, stdin=p_categorize.stdout, shell=False)

    p_categorize.stdout.close()
    p_categorize.wait()
    p_build_bf.wait()<commit_msg>Added helper functions<commit_after>#!/usr/bin/env python3

import shutil

def _get_biobloommaker_command(output_dir, threads, kmer, transcriptome):
    """Helper function to compose the command to execute"""
    biobloommaker_path = shutil.which('biobloommaker')
    build_transcriptome_bf = [
        biobloommaker_path,
        '--file_prefix', "transcriptome",
        '--output_dir', output_dir,
        '--threads', str(threads),
        '--kmer_size', str(kmer),
        transcriptome
    ]
    return build_transcriptome_bf

def _get_categorize_command(output_dir, threads, kmer, reads):
    """Helper function to compose the categorize command"""
    categorize_path = shutil.which('biobloomcategorizer')
    categorize = [
        categorize_path,
        '--prefix', output_dir + '/categories',
        '--filter_files', output_dir + '/transcriptome.bf',
        '--threads', str(threads),
        '--score', str(kmer),
        '--fa',
        '--stdout_filter', 'transcriptome',
    ] + reads
    return categorize

def _get_build_bf_command(kmer, bloom_size, levels, threads, output_bloom, input_fns):
    """Helper function to compose command to get the final Bloom Filter"""
    abyss_bloom_path = shutil.which('abyss-bloom')
    build_bf = [
        abyss_bloom_path, 'build',
        '--verbose',
        '--kmer', str(kmer),
        '--bloom-size', bloom_size,
        '--levels', str(levels),
        '--threads', str(threads),
        output_bloom,
    ] + input_fns
    return build_bf

def build_baited_bloom_filter(
    transcriptome,
    kmer,
    bloom_size,
    levels,
    output_bloom,
    threads,
    reads
):
    '''(str, int, str, int, str, int , list of str) -> None

    Run the build_baited_bloom_filter pipeline:
    - Build a secondary Bloom filter of the transcriptome with biobloommaker
    - Categorize reads with biobloomcategorizer and pipe to abyss-bloom build
        to build the primary Bloom filter

    Inputs are:
    - A transcriptome in Fasta format
    - A kmer size (better > 25)
    - The total memory to use in the primary Bloom filter
    - The number of levels to use in the Bloom filter (better to use 2)
    - The output file name for the primary Bloom filter
    - The number of threads to use in construction
    - A list of read filenames
    '''

    # Imports
    from subprocess import Popen, PIPE
    from sys import stderr
    from os.path import dirname

    output_dir = dirname(output_bloom)
    if output_dir == "":
        output_dir = "./"

    # Convert single read library to list
    if isinstance(reads, str):
        reads = [reads]

    # Prepare the commands
    build_transcriptome_bf = _get_biobloommaker_command(
        output_dir, threads, kmer, transcriptome
    )

    categorize = _get_categorize_command(
        output_dir, threads, kmer, reads
    )

    build_bf =_get_build_bf_command(
        kmer, bloom_size, levels, threads, output_bloom, ["/dev/stdin"]
    )

    # Run the pipeline
    stderr.write(
        "\n\nRunning command: {command}\n".format(
            command=" ".join(build_transcriptome_bf)
        )
    )

    p_build_transcriptome_bf = Popen(build_transcriptome_bf, shell=False)

    p_build_transcriptome_bf.wait()

    stderr.write(
        "\n\nRunning command: {command1} | {command2}\n".format(
            command1=" ".join(categorize),
            command2=" ".join(build_bf)
        )
    )

    p_categorize = Popen(categorize, stdout=PIPE, shell=False)
    p_build_bf = Popen(build_bf, stdin=p_categorize.stdout, shell=False)

    p_categorize.stdout.close()
    p_categorize.wait()
    p_build_bf.wait()
{'fraction_non_alphanumeric': 0.10834530078187331, 'fraction_numerical': 0.002553055688527206, 'mean_word_length': 2.5593412833617264, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4506511', 'n_tokens_mistral': 2070, 'n_tokens_neox': 1997, 'n_words': 508}