<commit_before># Python Version: 3.x
import onlinejudge
import onlinejudge.implementation.utils as utils
import onlinejudge.implementation.logging as log
import sys
import subprocess
import time

def non_block_read(fh):
    # workaround
    import fcntl
    import os
    fd = fh.fileno()
    fl = fcntl.fcntl(fd, fcntl.F_GETFL)
    fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NONBLOCK)
    try:
        return fh.read()
    except:
        return ''

split_input_auto_footer = ('__AUTO_FOOTER__', )  # this shouldn't be a string, so a tuple

def split_input(args):
    with open(args.input) as fh:
        inf = fh.read()
    if args.footer == split_input_auto_footer:
        args.footer = inf.splitlines(keepends=True)[-1]
    with subprocess.Popen(args.command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=sys.stderr) as proc:
        index = 0
        acc = ''
        for line in inf.splitlines(keepends=True):
            if args.ignore:
                args.ignore -= 1
            else:
                acc += line
            proc.stdin.write(line.encode())
            proc.stdin.flush()
            time.sleep(args.time)
            if non_block_read(proc.stdout): # if output exists
                index += 1
                path = utils.parcentformat(args.output, { 'i': str(index) })
                log.info('case found: %d', index)
                if args.header:
                    acc = args.header + acc
                if args.footer:
                    acc = acc + args.footer
                log.emit(log.bold(acc))
                with open(path, 'w') as fh:
                    fh.write(acc)
                log.success('saved to: %s', path)
                acc = ''
                while non_block_read(proc.stdout): # consume all
                    pass
<commit_msg>#152: add newline for --header option<commit_after># Python Version: 3.x
import onlinejudge
import onlinejudge.implementation.utils as utils
import onlinejudge.implementation.logging as log
import sys
import subprocess
import time

def non_block_read(fh):
    # workaround
    import fcntl
    import os
    fd = fh.fileno()
    fl = fcntl.fcntl(fd, fcntl.F_GETFL)
    fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NONBLOCK)
    try:
        return fh.read()
    except:
        return ''

split_input_auto_footer = ('__AUTO_FOOTER__', )  # this shouldn't be a string, so a tuple

def split_input(args):
    with open(args.input) as fh:
        inf = fh.read()
    if args.footer == split_input_auto_footer:
        args.footer = inf.splitlines(keepends=True)[-1]
    with subprocess.Popen(args.command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=sys.stderr) as proc:
        index = 0
        acc = ''
        for line in inf.splitlines(keepends=True):
            if args.ignore:
                args.ignore -= 1
            else:
                acc += line
            proc.stdin.write(line.encode())
            proc.stdin.flush()
            time.sleep(args.time)
            if non_block_read(proc.stdout): # if output exists
                index += 1
                path = utils.parcentformat(args.output, { 'i': str(index) })
                log.info('case found: %d', index)
                if args.header:
                    if args.header == args.header.strip():
                        acc = '\n' + acc
                    acc = args.header + acc
                if args.footer:
                    acc = acc + args.footer
                log.emit(log.bold(acc))
                with open(path, 'w') as fh:
                    fh.write(acc)
                log.success('saved to: %s', path)
                acc = ''
                while non_block_read(proc.stdout): # consume all
                    pass

{'fraction_non_alphanumeric': 0.1047695605573419, 'fraction_numerical': 0.0034833869239013935, 'mean_word_length': 1.7228300510576222, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6713436', 'n_tokens_mistral': 1172, 'n_tokens_neox': 1115, 'n_words': 271}