<commit_before>import luigi
import sciluigi as sl
import time

class T1(sl.Task):

    # Parameter

    text = luigi.Parameter()

    # I/O

    def out_data1(self):
        return sl.TargetInfo(self, self.text + '.txt') # TODO: Of course make the target spec into an object with "get target" method!

    # Implementation

    def run(self):
        with self.out_data1().open('w') as outfile:
            outfile.write(self.text)

# ========================================================================

class Merge(sl.Task):

    # I/O

    in_data1 = None
    in_data2 = None

    def out_merged(self): 
        return sl.TargetInfo(self, self.in_data1().path + '.merged.txt')

    # Implementation

    def run(self):
        with self.in_data1().open() as in1, self.in_data2().open() as in2, self.out_merged().open('w') as outfile:

            for row in in1:
                outfile.write(row+'\n')<commit_msg>Add sleeps in example3<commit_after>import luigi
import sciluigi as sl
import time

class T1(sl.Task):

    # Parameter

    text = luigi.Parameter()

    # I/O

    def out_data1(self):
        return sl.TargetInfo(self, self.text + '.txt') # TODO: Of course make the target spec into an object with "get target" method!

    # Implementation

    def run(self):
        time.sleep(1)
        with self.out_data1().open('w') as outfile:
            outfile.write(self.text)

# ========================================================================

class Merge(sl.Task):

    # I/O

    in_data1 = None
    in_data2 = None

    def out_merged(self): 
        return sl.TargetInfo(self, self.in_data1().path + '.merged.txt')

    # Implementation

    def run(self):
        time.sleep(2)
        with self.in_data1().open() as in1, self.in_data2().open() as in2, self.out_merged().open('w') as outfile:

            for row in in1:
                outfile.write(row+'\n')
{'fraction_non_alphanumeric': 0.19313984168865436, 'fraction_numerical': 0.013192612137203167, 'mean_word_length': 2.6744186046511627, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1858844', 'n_tokens_mistral': 632, 'n_tokens_neox': 583, 'n_words': 150}