<jupyter_start><jupyter_text>Test Notebook  Test saving and continuing training<jupyter_code>from __future__ import division
from IPython.display import clear_output
# imports to run OpenAI Gym in Jupyter
import gym
import gym.spaces
import gym_gvgai
import matplotlib.pyplot as plt
from IPython import display

# how to render in Jupyter: 
# https://stackoverflow.com/questions/40195740/how-to-run-openai-gym-render-over-a-server
# https://www.youtube.com/watch?v=O84KgRt6AJI
def show_state(env, step=0, name='', info=''):
    plt.figure(3)
    plt.clf()
    plt.imshow(env.render(mode='rgb_array'))
    plt.title("%s | Step: %d %s" % (name, step, info))
    plt.axis('off')

    display.clear_output(wait=True)
    display.display(plt.gcf())
    
# transforms the state into what the tpg agent can use.
def getState(state):
    state2 = []
    for x in state:
        for y in x:
            state2.append(y[0]/8 + y[1]*4 + y[2]*128)
            
    return state2

# run agent in function to work with multiprocessing
def runAgent(agenteqsq):
    agent = agenteqsq[0] # get agent
    eq = agenteqsq[1] # get environment queue
    sq = agenteqsq[2] # get score queue
    
    # check if agent already has score
    if agent.taskDone():
        print('Agent #' + str(agent.getAgentNum()) + ' can skip.')
        sq.put((agent.getUid(), agent.getOutcomes()))
        return
        
    print('envs in queue:',eq.qsize())
    env = eq.get() # get an environment
    state = env.reset() # get initial state and prep environment
    
    valActs = range(env.action_space.n)
    
    score = 0
    for i in range(1000): # run episodes that last 200 frames
        act = agent.act(getState(state), valActs=valActs) # get action from agent

        # feedback from env
        state, reward, isDone, debug = env.step(act)
        score += reward # accumulate reward in score
        if isDone:
            break # end early if losing state
            
    agent.reward(score) # must reward agent
    
    print('Agent #' + str(agent.getAgentNum()) + ' finished with score ' + str(score))
    sq.put((agent.getUid(), agent.getOutcomes())) # get outcomes with id
    eq.put(env) # put environment back
%matplotlib inline
import time # for tracking time
from tpg.tpg_trainer import TpgTrainer
import pickle

env = gym.make('Assault-v0') # make the environment

trainer = TpgTrainer(actions=range(env.action_space.n), teamPopSizeInit=30)

tStart = time.time()

curScores = [] # hold scores in a generation
summaryScores = [] # record score summaries for each gen (min, max, avg)

# 5 generations isn't much (not even close), but some improvements
# should be seen.
for gen in range(3): # generation loop
    curScores = [] # new list per gen
    
    while True: # loop to go through agents
        teamNum = trainer.remainingAgents()
        agent = trainer.getNextAgent()
        if agent is None:
            break # no more agents, so proceed to next gen
        
        # check if agent already has score
        if agent.taskDone():
            score = agent.getOutcome()
        else:
            state = env.reset() # get initial state and prep environment
            score = 0
            for i in range(50): # run episodes that last 200 frames

                act = agent.act(getState(state)) # get action from agent

                # feedback from env
                state, reward, isDone, debug = env.step(act)
                score += reward # accumulate reward in score
                if isDone:
                    break # end early if losing state

            agent.reward(score) # must reward agent (if didn't already score)
            
        print('Agent got score', score)
        curScores.append(score) # store score
            
    # at end of generation, make summary of scores
    summaryScores.append((min(curScores), max(curScores),
                    sum(curScores)/len(curScores))) # min, max, avg
    trainer.evolve()
    clear_output(wait=True)
    
clear_output(wait=True) 
print('Time Taken (Seconds): ' + str(time.time() - tStart))
print('Results:\nMin, Max, Avg')
for result in summaryScores:
    print(result[0],result[1],result[2])
    
print(trainer.generateScoreStats())
    
# save the model
with open('test-model.pkl','wb') as f:
    pickle.dump(trainer,f)
# test reobtaining file
with open('test-model.pkl','rb') as f:
    trainer = pickle.load(f)
    
print(trainer.generateScoreStats())
# make sure can continue training
tStart = time.time()

curScores = [] # hold scores in a generation
summaryScores = [] # record score summaries for each gen (min, max, avg)

# 5 generations isn't much (not even close), but some improvements
# should be seen.
for gen in range(3): # generation loop
    curScores = [] # new list per gen
    
    while True: # loop to go through agents
        teamNum = trainer.remainingAgents()
        agent = trainer.getNextAgent()
        if agent is None:
            break # no more agents, so proceed to next gen
        
        # check if agent already has score
        if agent.taskDone():
            score = agent.getOutcome()
        else:
            state = env.reset() # get initial state and prep environment
            score = 0
            for i in range(50): # run episodes that last 200 frames

                act = agent.act(getState(state)) # get action from agent

                # feedback from env
                state, reward, isDone, debug = env.step(act)
                score += reward # accumulate reward in score
                if isDone:
                    break # end early if losing state

            agent.reward(score) # must reward agent (if didn't already score)
            
        print('Agent got score', score)
        curScores.append(score) # store score
            
    # at end of generation, make summary of scores
    summaryScores.append((min(curScores), max(curScores),
                    sum(curScores)/len(curScores))) # min, max, avg
    trainer.evolve()
    clear_output(wait=True)
    
clear_output(wait=True) 
print('Time Taken (Seconds): ' + str(time.time() - tStart))
print('Results:\nMin, Max, Avg')
for result in summaryScores:
    print(result[0],result[1],result[2])
    
print(trainer.generateScoreStats())<jupyter_output>Time Taken (Seconds): 20.191575288772583
Results:
Min, Max, Avg
0.0 42.0 8.75
0.0 42.0 11.375
0.0 42.0 17.181818181818183
{'scores': [21.0, 42.0, 21.0, 21.0, 21.0, 21.0, 21.0, 21.0, 21.0, 21.0, 42.0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 'min': 0, 'max': 42.0, 'average': 12.409090909090908}<jupyter_text>Test GVGAI Env<jupyter_code>%matplotlib inline
import time # for tracking time
from tpg.tpg_trainer import TpgTrainer
import pickle

env = gym.make('gvgai-testgame3-lvl0-v0') # make the environment

trainer = TpgTrainer(actions=range(env.action_space.n), teamPopSizeInit=30)

tStart = time.time()

curScores = [] # hold scores in a generation
summaryScores = [] # record score summaries for each gen (min, max, avg)

# 5 generations isn't much (not even close), but some improvements
# should be seen.
for gen in range(3): # generation loop
    curScores = [] # new list per gen
    
    while True: # loop to go through agents
        teamNum = trainer.remainingAgents()
        agent = trainer.getNextAgent()
        if agent is None:
            break # no more agents, so proceed to next gen
        
        # check if agent already has score
        if agent.taskDone():
            score = agent.getOutcome()
        else:
            state = env.reset() # get initial state and prep environment
            score = 0
            for i in range(2000): # run episodes that last 200 frames
                show_state(env, i, 'Assault', 'Gen #' + str(gen) + 
                           ', Team #' + str(teamNum) +
                           ', Score: ' + str(score)) # render env

                act = agent.act(getState(state)) # get action from agent

                # feedback from env
                state, reward, isDone, debug = env.step(act)
                score += reward # accumulate reward in score
                if isDone:
                    break # end early if losing state

            agent.reward(score) # must reward agent (if didn't already score)
            
        print('Agent got score', score)
        curScores.append(score) # store score
            
    # at end of generation, make summary of scores
    summaryScores.append((min(curScores), max(curScores),
                    sum(curScores)/len(curScores))) # min, max, avg
    trainer.evolve()
    clear_output(wait=True)
    
clear_output(wait=True) 
print('Time Taken (Seconds): ' + str(time.time() - tStart))
print('Results:\nMin, Max, Avg')
for result in summaryScores:
    print(result[0],result[1],result[2])
    
print(trainer.generateScoreStats())
    
# save the model
with open('test-model.pkl','wb') as f:
    pickle.dump(trainer,f)
env = gym.make('gvgai-testgame1-lvl0-v0') # make the environment
print(env.action_space)
env = gym.make('gvgai-testgame2-lvl0-v0') # make the environment
print(env.action_space)
env = gym.make('gvgai-testgame3-lvl0-v0') # make the environment
print(env.action_space)
print(str(env.env))<jupyter_output>Connecting to host 127.0.0.1 at port 47053 ...
Client connected to server [OK]
Discrete(6)
Connecting to host 127.0.0.1 at port 37309 ...
Client connected to server [OK]
Discrete(6)
Connecting to host 127.0.0.1 at port 47219 ...
Client connected to server [OK]
Discrete(5)
<GVGAI_Env<gvgai-testgame3-lvl0-v0>><jupyter_text>Test multigame playing with multiprocessing<jupyter_code>import time
import random
from tpg.tpg_trainer import TpgTrainer
from tpg.tpg_agent import TpgAgent
import pickle
import multiprocessing as mp

tStart = time.time()

lock = mp.Lock()
processes = 4 # how many to run concurrently (4 is best for my local desktop)
m = mp.Manager()

allGames = ['gvgai-testgame1-lvl0-v0','gvgai-testgame1-lvl1-v0',
            'gvgai-testgame2-lvl0-v0','gvgai-testgame2-lvl0-v0',
            'gvgai-testgame3-lvl0-v0','gvgai-testgame3-lvl1-v0']
allGames = ['Assault-v0']

envs = {}
for game in allGames:
    envs[game] = []
    for p in range(processes): # each process needs its own environment
        envs[game].append(gym.make(game))
        
gameQueue = list(allGames)
random.shuffle(gameQueue)
    
trainer = TpgTrainer(actions=range(6), teamPopSizeInit=360)

pool = mp.Pool(processes=processes)
    
summaryScores = [] # record score summaries for each gen (min, max, avg) 
    
for gen in range(100): # generation loop
    scoreQueue = m.Queue() # hold agents when finish, to actually apply score
    envQueue = m.Queue() # hold envs for current gen
    
    # get right env in envQueue
    game = gameQueue.pop() # take out last game
    for p in range(processes):
        envQueue.put(envs[game][p])
    # re-get games list
    if len(gameQueue) == 0:
        gameQueue = list(allGames)
        random.shuffle(gameQueue)
        
    # tasks = [str(envs[game][0].env)]
    
    # run generation
    # skipTasks=[] so we get all agents, even if already scored,
    # just to report the obtained score for all agents.
    pool.map(runAgent, 
                 [(agent, envQueue, scoreQueue) 
                  for agent in trainer.getAllAgents(skipTasks=[])])
    
    scores = [] # convert scores into list
    while not scoreQueue.empty():
        scores.append(scoreQueue.get())
    if len(scores) == 0:
        print('wtf this is bs')
        x = 1/0
    
    # apply scores
    trainer.applyScores(scores)
    trainer.evolve() # go into next gen
    
    # save model after every gen
    with open('gvgai-model-1.pkl','wb') as f:
        pickle.dump(trainer,f)

    # at end of generation, make summary of scores
    summaryScores.append((trainer.scoreStats['min'], 
                    trainer.scoreStats['max'],
                    trainer.scoreStats['average'])) # min, max, avg
    
    clear_output(wait=True)
    print('Time Taken (Seconds): ' + str(time.time() - tStart))
    print('Results so far: ' + str(summaryScores))
    
clear_output(wait=True)
print('Time Taken (Seconds): ' + str(time.time() - tStart))
print('Results: ' + str(summaryScores))<jupyter_output>envs in queue: 4
envs in queue: 3
envs in queue: 2
envs in queue: 1
envs in queue: 0
envs in queue: 0
envs in queue: 0
envs in queue: 0
{'fraction_non_alphanumeric': 0.11130928252938792, 'fraction_numerical': 0.023510336441021483, 'mean_word_length': 2.6400118028917086, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 2, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 3, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '256899', 'license': 'MIT', 'n_tokens_mistral': 4115, 'n_tokens_neox': 3849, 'n_words': 1165, 'path': 'test-notebook.ipynb', 'repo_name': 'Ryan-Amaral/gvgai-tpg', 'size': '12335'}