<jupyter_start><jupyter_text>Evolutionary strategies for Pommerman Dependencies below. You need to install the Pommerman environment as well as all the modules in fitness.py, model.py and evolutionarystrategy.py<jupyter_code>from evolutionarystrategy import EvolutionaryStrategy
from fitness import Fitness
from model import Model
import sys
import multiprocessing as mp
import numpy as np
import csv
import torch<jupyter_output><empty_output><jupyter_text>Epsilon generating function Do not change.<jupyter_code>def generate_epsilon(seed, model):
	torch.manual_seed(seed)
	epsilon = {}
	for key, shape in model.shape().items():
		if model.params[key].type() == "torch.FloatTensor":
			epsilon[key] = torch.randn(shape).float()
		elif model.params[key].type() == "torch.LongTensor":
			epsilon[key] = torch.randn(shape).long()
		else:
			epsilon[key] = torch.randn(shape)

	return epsilon<jupyter_output><empty_output><jupyter_text>Main optimization loopPopulation size needs to be 30 and range needs to be 10000 for you to replicate our results. However you can run it on your computer with population size = 2 and range(10). We highly recommend to run the python files on a cluster to get the desired performance.<jupyter_code>if __name__ == '__main__':
	impact = {
	'imp_team': [0.01],
	'imp_enemies': [0.1,0.1,0.1],
	'imp_powerup': [0.02]
	}
	evo_strat = EvolutionaryStrategy(Model, Fitness, impact, populationsize=2, learning_rate = 0.01)
	rewardcsv = open("Rewards.csv", "w")  
	winratecsv = open("Winrate.csv", "w")
	rewardcsv.close()
	winratecsv.close()
	for i in range(10):
		manager = mp.Manager()
		output = manager.Queue()
		processes = [mp.Process(target=evo_strat.evolution, args=(x, output)) for x in range(evo_strat.populationsize)]
		for p in processes:
			p.start()
		for p in processes:
			p.join()
		results = [output.get() for p in processes]
		rewards = [r[0] for r in results]
		epsilons = []
		seed = [epsilons.append(generate_epsilon(r[1], evo_strat.model)) for r in results]
		evo_strat.model.update_params(epsilons, rewards, evo_strat.learning_rate)
		print("Done with iteration {}".format(i))
		if (i)%10==0:
			winrate = evo_strat.play_game(10)
			print("Average win rate over 10 games {}".format(winrate))
			rewardcsv = open("Rewards.csv", "a")  
			winratecsv = open("Winrate.csv", "a")
			with rewardcsv:
				writer = csv.writer(rewardcsv)
				writer.writerow(rewards)
			with winratecsv:
				writer = csv.writer(winratecsv)
				writer.writerow([winrate])
			rewardcsv.close()
			winratecsv.close()
	torch.save(evo_strat.model.policy.state_dict(),'Model.pt')<jupyter_output>Done with iteration 0
Average win rate over 10 games 0.0
{'fraction_non_alphanumeric': 0.12016522718738265, 'fraction_numerical': 0.015771686068343973, 'mean_word_length': 5.040816326530612, 'pattern_counts': {'":': 2, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '419637', 'license': 'MIT', 'n_tokens_mistral': 991, 'n_tokens_neox': 894, 'n_words': 244, 'path': 'evolutionarystrategies/ReproduceResults.ipynb', 'repo_name': 'ChristianIngwersen/BombermanRL', 'size': '2663'}