<commit_before>#!myfunction(param1="a",param2="b")|prog {param1} "param2"
import fileinput
def cat(*names):
    def cat(input):
        yield from fileinput.input(*names)
    return cat

def output():
    def _output(input):
        for line in input:
            print(line,end="")
    return _output
def split(separator=""):
    #print("splitting",file=stderr)
    def _split(input):
        yield from (line.split(separator) for line in input)
    return _split
def sort(key=None):
    def _sort(input):
        yield from sorted(input,key=key)
    return _sort
def join(separator=""):
    def _join(input):
        yield from (separator.join(line) for line in input)
    return _join

import sys
def chain_generators(*generators):
    input = None
    for generator in generators:
        input = generator(input)
    return generator

chain_generators(cat(sys.argv[1:]),split(","),sort(key=lambda k:int(k[1])),join("\t"),output())<commit_msg>first happy version (correct commit)<commit_after>#!myfunction(param1="a",param2="b")|prog {param1} "param2"
import fileinput
import sys

def cat(*names):
    def _cat(input):
        yield from fileinput.input(*names)
    return _cat

def output():
    def _output(input):
        for line in input:
            print(line,end="")
    return _output

def split(separator=None):
    #print("splitting",file=stderr)
    def _split(input):
        yield from (line.split(separator) for line in input)
    return _split

def sort(key=None,reverse=False):
    def _sort(input):
        yield from sorted(input,key=key,reverse=reverse)
    return _sort

def join(separator=""):
    def _join(input):
        yield from (separator.join(line) for line in input)
    return _join


def chain_generators(*generators):
    input = None
    for generator in generators:
        input = generator(input)
    return generator

chain_generators(cat(sys.argv[1:]),
                 split(","),
                 sort(key=lambda k:int(k[1]),reverse=True),
                 join("\t"),
                 output())
{'fraction_non_alphanumeric': 0.13032827045565898, 'fraction_numerical': 0.005879470847623714, 'mean_word_length': 3.2016460905349793, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3126875', 'n_tokens_mistral': 632, 'n_tokens_neox': 619, 'n_words': 161}