<commit_before>import json 
from datetime import datetime as time
class Task:
    def __init__(self, id, r):
        self.r  = r
        self.id = id

    def addData(self, data):
        self.data = data

    def getData(self):
        return self.data

    def setTimestamp(self):
        self.timestamp = time.now()

    def getTimestamp(self):
        return self.timestamp

    def json(self):
        return json.dumps({
            'start' : self.r.start,
            'stop'  : self.r.stop,
            'step'  : self.r.step
        })

class TaskList:
    def __init__(self):
        self.new = []
        self.complete = []
        self.running = {}

    def addTask(self, id, r):
        self.new.append(Task(id, r))

    def getTask(self, ip):
        if len(self.new) > 0 and ip not in self.running:
            task = self.new.pop()
            task.setTimestamp()
            self.running[ip] = task
            return task.json()
        elif ip in self.running:
            return self.running[ip].json()

    def getJSON(self, ip):
        if ip in self.running:
            return self.running[ip].json()

    def updateStatus(self, ip):
        if ip in self.running:
            self.running[ip].setTimestamp()

    def failTask(self, ip):
        if ip in self.running:
            task = self.running[ip]
            del self.running[ip]
            self.new.append(task)

    def completeTask(self, ip, data):
        if ip in self.running:
            task = self.running[ip]
            task.addData(data)
            del self.running[ip]
            self.complete.append(task)<commit_msg>Added data export<commit_after>import json 
from datetime import datetime as time
class Task:
    def __init__(self, id, r):
        self.r    = r
        self.id   = id
        self.data = ''

    def addData(self, data):
        self.data = data

    def getData(self):
        return self.data

    def setTimestamp(self):
        self.timestamp = time.now()

    def getTimestamp(self):
        return self.timestamp

    def json(self):
        return json.dumps({
            'start' : self.r.start,
            'stop'  : self.r.stop,
            'step'  : self.r.step
        })

class TaskList:
    def __init__(self):
        self.new = []
        self.complete = []
        self.running = {}

    def addTask(self, id, r):
        self.new.append(Task(id, r))

    def getTask(self, ip):
        if len(self.new) > 0 and ip not in self.running:
            task = self.new.pop()
            task.setTimestamp()
            self.running[ip] = task
            return task.json()

        elif ip in self.running:
            return self.running[ip].json()

        else:
            return '{}'

    def getJSON(self, ip):
        if ip in self.running:
            return self.running[ip].json()
        else:
            return '{}'

    def updateStatus(self, ip):
        if ip in self.running:
            self.running[ip].setTimestamp()

    def failTask(self, ip):
        if ip in self.running:
            task = self.running[ip]
            self.new.append(task)
            del self.running[ip]

    def exportData(self):
        data = []
        filename = 'games.' + str(time.now()) + '.json'
        
        for res in self.complete:
            for game in res.getData():
                data.append(game)


        with open(filename, 'w') as w:
            w.write(json.dumps(data, indent=4, separators=(',', ': ')))

        print('Exported data to:', filename)


    def completeTask(self, ip, data):
        if ip in self.running:
            
            task = self.running[ip]
            task.addData(data)
            self.complete.append(task)
            del self.running[ip]

            if len(self.new) == 0 and len(self.running) == 0:
                self.exportData()
{'fraction_non_alphanumeric': 0.11872384937238493, 'fraction_numerical': 0.001307531380753138, 'mean_word_length': 1.5985054347826086, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1242291', 'n_tokens_mistral': 1164, 'n_tokens_neox': 1129, 'n_words': 291}