<commit_before>import sys
import argparse
from _distiller_common import open_bgzip

def main():
    parser = argparse.ArgumentParser(
      'Read a pairsam file and print only the pairs of a certain type(s).'
    )
    parser.add_argument(
        'pair_types', 
        nargs='+',
        type=str, 
        help='The list of requested pair types.')
    parser.add_argument(
        '--input',
        type=str, 
        default="",
        help='input file. By default, the input is read from stdin.')
    parser.add_argument(
        "--output", 
        type=str, 
        default="", 
        help="output file. By default, the output is printed into stdout.")
    parser.add_argument(
        "--output-rest", 
        type=str, 
        default="", 
        help="output file for pairs of other types. By default, such pairs are dropped."
        )
    
    args = vars(parser.parse_args())
    
    instream = (open_bgzip(args['input'], mode='r') 
                if args['input'] else sys.stdin)
    outstream = (open_bgzip(args['outstream'], mode='w') 
                 if args['output'] else sys.stdout)
    outstream_rest = (open_bgzip(args['outstream_rest'], mode='w') 
                      if args['output_rest'] else None)

    pair_types = args['pair_types']

    for line in instream.readlines():
        if line.startswith('#'):

            outstream.write(line)
            if outstream_rest:
                outstream_rest.write(line)
        else:
            cols = line.split('\v')
            if any((pair_type == cols[7] for pair_type in pair_types)):
                outstream.write(line)
            elif outstream_rest:
                outstream_rest.write(line)

if __name__ == '__main__':
    main()<commit_msg>pairsam_separate_pair_type.py : close files<commit_after>import sys
import argparse
from _distiller_common import open_bgzip

def main():
    parser = argparse.ArgumentParser(
      'Read a pairsam file and print only the pairs of a certain type(s).'
    )
    parser.add_argument(
        'pair_types', 
        nargs='+',
        type=str, 
        help='The list of requested pair types.')
    parser.add_argument(
        '--input',
        type=str, 
        default="",
        help='input file. By default, the input is read from stdin.')
    parser.add_argument(
        "--output", 
        type=str, 
        default="", 
        help="output file. By default, the output is printed into stdout.")
    parser.add_argument(
        "--output-rest", 
        type=str, 
        default="", 
        help="output file for pairs of other types. By default, such pairs are dropped."
        )
    
    args = vars(parser.parse_args())
    
    instream = (open_bgzip(args['input'], mode='r') 
                if args['input'] else sys.stdin)
    outstream = (open_bgzip(args['outstream'], mode='w') 
                 if args['output'] else sys.stdout)
    outstream_rest = (open_bgzip(args['outstream_rest'], mode='w') 
                      if args['output_rest'] else None)

    pair_types = args['pair_types']

    for line in instream.readlines():
        if line.startswith('#'):

            outstream.write(line)
            if outstream_rest:
                outstream_rest.write(line)
        else:
            cols = line.split('\v')
            if any((pair_type == cols[7] for pair_type in pair_types)):
                outstream.write(line)
            elif outstream_rest:
                outstream_rest.write(line)

    if hasattr(instream, 'close'):
        instream.close()
    if hasattr(outstream, 'close'):
        outstream.close()
    if outstream_rest:
        outstream_rest.close()

if __name__ == '__main__':
    main()
{'fraction_non_alphanumeric': 0.11806119685892229, 'fraction_numerical': 0.0005415651232060655, 'mean_word_length': 2.068106312292359, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7132305', 'n_tokens_mistral': 1109, 'n_tokens_neox': 1067, 'n_words': 278}