<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
import io
import sys
import click
import subprocess

from . import _io, _pairsam_format, cli, _headerops

UTIL_NAME = 'pairsam_sort'

@cli.command()

@click.argument(
    'pairsam_path', 
    type=str,
    required=False)

@click.option(
    '-o', "--output", 
    type=str, 
    default="", 
    help='output pairsam file.'
        ' If the path ends with .gz, the output is bgzip-compressed.'
        ' By default, the output is printed into stdout.')

@click.option(
    "--nproc", 
    type=int, 
    default=8, 
    show_default=True,
    help='Number of processes to split the work between.'
    )

@click.option(
    "--tmpdir", 
    type=str, 
    default='', 
    help='Custom temporary folder for sorting intermediates.'
    )

@click.option(
    "--memory", 
    type=str, 
    default='2G', 
    show_default=True,
    help='The amount of memory used by default.',

    )


def sort(pairsam_path, output, nproc, tmpdir, memory):
    '''sort a pairs/pairsam file. 
    
    The resulting order is lexicographic along chrom1 and chrom2, numeric 
    along pos1 and pos2 and lexicographic along pair_type.

    PAIRSAM_PATH : input .pairsam file. If the path ends with .gz, the input is
    gzip-decompressed. By default, the input is read from stdin.
    '''
    sort_py(pairsam_path, output, nproc, tmpdir, memory)

def sort_py(pairsam_path, output, nproc, tmpdir, memory):

    instream = (_io.open_bgzip(pairsam_path, mode='r', nproc=nproc) 
                if pairsam_path else sys.stdin)
    outstream = (_io.open_bgzip(output, mode='w', nproc=nproc) 
                 if output else sys.stdout)

    header, body_stream = _headerops.get_header(instream)
    header = _headerops.append_new_pg(header, ID=UTIL_NAME, PN=UTIL_NAME)
    header = _headerops.mark_header_as_sorted(header)

    outstream.writelines((l+'\n' for l in header))

    outstream.flush()

    command = r'''
        /bin/bash -c 'export LC_COLLATE=C; export LANG=C; sort 
        -k {0},{0} -k {1},{1} -k {2},{2}n -k {3},{3}n -k {4},{4} 
        --stable
        --field-separator=$'\''{5}'\'' 
        {6}
        {7}
        -S {8}
        '''.replace('\n',' ').format(
                _pairsam_format.COL_C1+1, 
                _pairsam_format.COL_C2+1, 
                _pairsam_format.COL_P1+1, 
                _pairsam_format.COL_P2+1,
                _pairsam_format.COL_PTYPE+1,
                _pairsam_format.PAIRSAM_SEP_ESCAPE,
                ' --parallel={} '.format(nproc) if nproc > 1 else ' ',
                ' -T {} '.format(tmpdir) if tmpdir else ' ',
                memory
        )
    command += "'"

    with subprocess.Popen(
            command, stdin=subprocess.PIPE, bufsize=-1, shell=True,
            stdout=outstream) as process:
        stdin_wrapper = io.TextIOWrapper(process.stdin, 'utf-8')
        for line in body_stream:<commit_msg>sort: add an option to compress temprorary sorted sub-chunks<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
import io
import sys
import click
import subprocess

from . import _io, _pairsam_format, cli, _headerops

UTIL_NAME = 'pairsam_sort'

@cli.command()

@click.argument(
    'pairsam_path', 
    type=str,
    required=False)

@click.option(
    '-o', "--output", 
    type=str, 
    default="", 
    help='output pairsam file.'
        ' If the path ends with .gz, the output is bgzip-compressed.'
        ' By default, the output is printed into stdout.')

@click.option(
    "--nproc", 
    type=int, 
    default=8, 
    show_default=True,
    help='Number of processes to split the work between.'
    )

@click.option(
    "--tmpdir", 
    type=str, 
    default='', 
    help='Custom temporary folder for sorting intermediates.'
    )

@click.option(
    "--memory", 
    type=str, 
    default='2G', 
    show_default=True,
    help='The amount of memory used by default.',

    )

@click.option(
    "--compress-program",
    type=str,
    default='',
    show_default=True,
    help='A binary to compress temporary sorted chunks. '
    'Must decompress input when the flag -d is provided. '
    'Suggested alternatives: gzip, lzop, lz4c, snzip.'
     )


def sort(pairsam_path, output, nproc, tmpdir, memory, compress_program):
    '''sort a pairs/pairsam file. 
    
    The resulting order is lexicographic along chrom1 and chrom2, numeric 
    along pos1 and pos2 and lexicographic along pair_type.

    PAIRSAM_PATH : input .pairsam file. If the path ends with .gz, the input is
    gzip-decompressed. By default, the input is read from stdin.
    '''
    sort_py(pairsam_path, output, nproc, tmpdir, memory, compress_program)

def sort_py(pairsam_path, output, nproc, tmpdir, memory, compress_program):

    instream = (_io.open_bgzip(pairsam_path, mode='r', nproc=nproc) 
                if pairsam_path else sys.stdin)
    outstream = (_io.open_bgzip(output, mode='w', nproc=nproc) 
                 if output else sys.stdout)

    header, body_stream = _headerops.get_header(instream)
    header = _headerops.append_new_pg(header, ID=UTIL_NAME, PN=UTIL_NAME)
    header = _headerops.mark_header_as_sorted(header)

    outstream.writelines((l+'\n' for l in header))

    outstream.flush()

    command = r'''
        /bin/bash -c 'export LC_COLLATE=C; export LANG=C; sort 
        -k {0},{0} -k {1},{1} -k {2},{2}n -k {3},{3}n -k {4},{4} 
        --stable
        --field-separator=$'\''{5}'\'' 
        {6}
        {7}
        -S {8}
        {9}
        '''.replace('\n',' ').format(
                _pairsam_format.COL_C1+1, 
                _pairsam_format.COL_C2+1, 
                _pairsam_format.COL_P1+1, 
                _pairsam_format.COL_P2+1,
                _pairsam_format.COL_PTYPE+1,
                _pairsam_format.PAIRSAM_SEP_ESCAPE,
                ' --parallel={} '.format(nproc) if nproc > 1 else ' ',
                ' --temporary-directory={} '.format(tmpdir) if tmpdir else ' ',
                memory,
                (' --compress-program={} '.format(compress_program)
                    if compress_program else ' '),

        )
    command += "'"

    with subprocess.Popen(
            command, stdin=subprocess.PIPE, bufsize=-1, shell=True,
            stdout=outstream) as process:
        stdin_wrapper = io.TextIOWrapper(process.stdin, 'utf-8')
        for line in body_stream:
{'fraction_non_alphanumeric': 0.12796730073887755, 'fraction_numerical': 0.010690143059267411, 'mean_word_length': 2.3697033898305087, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5081859', 'n_tokens_mistral': 2237, 'n_tokens_neox': 2128, 'n_words': 523}