<commit_before>import sys
import os
import json
import numpy as np

MAX_ACTIONS = 20
PLAYER_RANGE = (4, 7)

def init_vec(num_players):
    input_vec = [0]*8
    input_vec[num_players - PLAYER_RANGE[0]] = 1
    return input_vec

def gen_training_data(hand):
    num_players = hand['num_players']
    if PLAYER_RANGE[0] <= num_players <= PLAYER_RANGE[1]:
        players = []
        for action in hand['actions'][:num_players]:
            players.append(action[0])

        i = 0
        j = 0
        count = 0

        inputs = [init_vec(num_players)]
        outputs = []
        while i < len(hand['actions']):
            action = hand['actions'][i]
            if action == 'NEXT':
                break

            input_vec = init_vec(num_players)
            output_vec = [0] * 4

            if action[0] == players[j]:
                if action[1] == 0:
                    input_vec[4] = 1
                    output_vec[0] = 1
                elif action[1] in [1, 2]:
                    input_vec[5] = 1
                    output_vec[1] = 1
                else:
                    input_vec[6] = 1
                    output_vec[2] = 1
                i += 1
            else:
                input_vec[7] = 1
                output_vec[3] = 1
                count += 1
                if count == 10:
                    return False

            inputs.append(input_vec)
            outputs.append(output_vec)

            j = (j + 1) % num_players

        inputs = inputs[:-1]
        if len(inputs) < MAX_ACTIONS:
            for _ in range(MAX_ACTIONS - len(inputs)):
                inputs.append([0]*8)
                outputs.append([0, 0, 0, 1])
        elif len(inputs) > MAX_ACTIONS:
            inputs = inputs[:MAX_ACTIONS]
            outputs = outputs[:MAX_ACTIONS]

        return inputs, outputs
    return False

if __name__ == '__main__':
    input_dir = sys.argv[1]
    output_dir = sys.argv[2]

    inputs = []
    outputs = []
    BUCKET_SIZE = 100
    i = 0
    ind = 0<commit_msg>Add extra boolean to training data to indicate finish training.<commit_after>import sys
import os
import json
import numpy as np

MAX_ACTIONS = 20
PLAYER_RANGE = (4, 7)

def init_vec(num_players):
    input_vec = [0]*9
    input_vec[num_players - PLAYER_RANGE[0]] = 1
    return input_vec

def gen_training_data(hand):
    num_players = hand['num_players']
    if PLAYER_RANGE[0] <= num_players <= PLAYER_RANGE[1]:
        players = []
        for action in hand['actions'][:num_players]:
            players.append(action[0])

        i = 0
        j = 0
        count = 0

        inputs = [init_vec(num_players)]
        outputs = []
        while i < len(hand['actions']):
            action = hand['actions'][i]
            if action == 'NEXT':
                break

            input_vec = init_vec(num_players)
            output_vec = [0] * 5

            if action[0] == players[j]:
                if action[1] == 0:
                    input_vec[4] = 1
                    output_vec[0] = 1
                elif action[1] in [1, 2]:
                    input_vec[5] = 1
                    output_vec[1] = 1
                else:
                    input_vec[6] = 1
                    output_vec[2] = 1
                i += 1
            else:
                input_vec[7] = 1
                output_vec[3] = 1

                count += 1
                if count == 10:
                    return False

            outputs.append(output_vec)
            inputs.append(input_vec)

            j = (j + 1) % num_players

        inputs = inputs[:-1]
        while True:
            if inputs[-1][7] == 1:
                inputs.pop()
                outputs.pop()
            else:
                break
        if len(inputs) < MAX_ACTIONS:
            for _ in range(MAX_ACTIONS - len(inputs)):
                inputs.append([0, 0, 0, 0, 0, 0, 0, 0, 1])
                outputs.append([0, 0, 0, 0, 1])
        elif len(inputs) > MAX_ACTIONS:
            inputs = inputs[:MAX_ACTIONS]
            outputs = outputs[:MAX_ACTIONS]

        return inputs, outputs
    return False

if __name__ == '__main__':
    input_dir = sys.argv[1]
    output_dir = sys.argv[2]

    inputs = []
    outputs = []
    BUCKET_SIZE = 100
    i = 0
    ind = 0
{'fraction_non_alphanumeric': 0.10771028037383178, 'fraction_numerical': 0.029205607476635514, 'mean_word_length': 1.3783333333333334, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1964947', 'n_tokens_mistral': 1414, 'n_tokens_neox': 1293, 'n_words': 318}