<jupyter_start><jupyter_text>TEST ENV<jupyter_code>import csv
from tqdm import tqdm
from datetime import datetime


# 실행 환경 클래스

class TestEnvironment:

    def __init__(self, env_type, BS_level):
        self.env = gym.make('BeerGame-v0', n_agents=4, env_type=env_type, BS_level=BS_level, n_turns_per_game=200)
        print(env_type)
        num_states = len(torch.tensor(self.env.reset()).type(torch.FloatTensor)[:,:5].reshape(-1))
        num_alphas = 11 # 0.0 ~ 1.0까지 총 11개
        num_actions = 21 * 11
        
        tmp = []
        for each in [[i]*11 for i in range(-10,11)] :
            tmp += each
        
        self.d_x_rule_dict = {k:tmp[k] for k in range(len(tmp))} 
        self.alpha_dict = {k:i for k,i in enumerate([i/10 for i in range(11)]*21)}
                
        self.BS_level = BS_level
        
        self.agent_0 = Agent(num_states, num_actions)  # retailer 역할을 할 객체를 생성
        self.agent_1 = Agent(num_states, num_actions)  # wholesaler 역할을 할 객체를 생성
        self.agent_2 = Agent(num_states, num_actions)  # distributor 역할을 할 객체를 생성
        self.agent_3 = Agent(num_states, num_actions)  # manufacturer 역할을 할 객체를 생성
        
    def test(self):
        plt.style.use('fivethirtyeight')
        
        episode = 0
        
        self.agent_0.brain.main_q_network.load_state_dict(torch.load('./result/{}/main_q_net_ret_EPISODE {:04}.pt'.format(PATH, episode)))
        self.agent_1.brain.main_q_network.load_state_dict(torch.load('./result/{}/main_q_net_who_EPISODE {:04}.pt'.format(PATH, episode)))
        self.agent_2.brain.main_q_network.load_state_dict(torch.load('./result/{}/main_q_net_dis_EPISODE {:04}.pt'.format(PATH, episode)))
        self.agent_3.brain.main_q_network.load_state_dict(torch.load('./result/{}/main_q_net_man_EPISODE {:04}.pt'.format(PATH, episode)))
        self.agent_0.brain.target_q_network.load_state_dict(torch.load('./result/{}/targ_q_net_ret_EPISODE {:04}.pt'.format(PATH, episode)))
        self.agent_1.brain.target_q_network.load_state_dict(torch.load('./result/{}/targ_q_net_who_EPISODE {:04}.pt'.format(PATH, episode)))
        self.agent_2.brain.target_q_network.load_state_dict(torch.load('./result/{}/targ_q_net_dis_EPISODE {:04}.pt'.format(PATH, episode)))
        self.agent_3.brain.target_q_network.load_state_dict(torch.load('./result/{}/targ_q_net_man_EPISODE {:04}.pt'.format(PATH, episode)))


        stock_list_ret = []
        stock_list_who = []
        stock_list_dis = []
        stock_list_man = []

        demand_list_ret = []
        demand_list_who = []
        demand_list_dis = []
        demand_list_man = []

        action_real_list_ret = []
        action_real_list_who = []
        action_real_list_dis = []
        action_real_list_man = []
        
        action_coop_list_ret = []
        action_coop_list_who = []
        action_coop_list_dis = []
        action_coop_list_man = []

        rewards_list_ret = []
        rewards_list_who = []
        rewards_list_dis = []
        rewards_list_man = []
        


        observation = self.env.reset()  # 환경 초기화
        rewards_list = []
        state = observation  # 관측을 변환없이 그대로 상태 s로 사용

        state = torch.tensor(state).type(torch.FloatTensor).cuda()
        


        for step in range(200):  # 1 에피소드에 해당하는 반복문

            state_ret = state[:,:5].reshape(-1).unsqueeze(0)
            state_who = state[:,5:10].reshape(-1).unsqueeze(0)
            state_dis = state[:,10:15].reshape(-1).unsqueeze(0)
            state_man = state[:,15:].reshape(-1).unsqueeze(0)


            action_ret = self.agent_0.get_action(state_ret, episode, istest=True).cuda()  # 다음 행동을 결정
            action_ret_ = self.d_x_rule_dict[action_ret.item()] # d + x 를 위한 변환
            action_who = self.agent_1.get_action(state_who, episode, istest=True).cuda()  # 다음 행동을 결정
            action_who_ = self.d_x_rule_dict[action_who.item()] # d + x 를 위한 변환
            action_dis = self.agent_2.get_action(state_dis, episode, istest=True).cuda()  # 다음 행동을 결정
            action_dis_ = self.d_x_rule_dict[action_dis.item()] # d + x 를 위한 변환
            action_man = self.agent_3.get_action(state_man, episode, istest=True).cuda()  # 다음 행동을 결정
            action_man_ = self.d_x_rule_dict[action_man.item()] # d + x 를 위한 변환
            action = [action_ret_, action_who_, action_dis_, action_man_]

            action_coop_list_ret.append(self.alpha_dict[action_ret.item()])
            action_coop_list_who.append(self.alpha_dict[action_who.item()])
            action_coop_list_dis.append(self.alpha_dict[action_dis.item()])
            action_coop_list_man.append(self.alpha_dict[action_man.item()])
            
            
            observation_next, reward, done, info = self.env.step(action) # action은 주문량 d + x rule의 x에 해당됨

            stock_list_ret.append(info[0][0])
            stock_list_who.append(info[0][1])
            stock_list_dis.append(info[0][2])
            stock_list_man.append(info[0][3])

            demand_list_ret.append(info[1][0])
            demand_list_who.append(info[1][1])
            demand_list_dis.append(info[1][2])
            demand_list_man.append(info[1][3])

            action_real_list_ret.append(info[2][0][-1])
            action_real_list_who.append(info[2][1][-1])
            action_real_list_dis.append(info[2][2][-1])
            action_real_list_man.append(info[2][3][-1])

            rewards_list_ret.append(reward[0])
            rewards_list_who.append(reward[1])
            rewards_list_dis.append(reward[2])
            rewards_list_man.append(reward[3])



            # 보상을 부여하고 episode의 종료 판정 및 state_next를 설정한다
            if done:
                state_next = None  # 다음 상태가 없으므로 None으로
                break

            else:
                state_next = observation_next  # 관측 결과를 그대로 상태로 사용
                state_next = torch.tensor(state_next).type(torch.FloatTensor).cuda()
                state_ret_next = state_next[:,:5].reshape(-1).unsqueeze(0)
                state_who_next = state_next[:,5:10].reshape(-1).unsqueeze(0)
                state_dis_next = state_next[:,10:15].reshape(-1).unsqueeze(0)
                state_man_next = state_next[:,15:].reshape(-1).unsqueeze(0)

            # 관측 결과를 업데이트
            state = state_next
        
        rewards_avg_ret = np.mean(np.array(rewards_list_ret))
        rewards_avg_who = np.mean(np.array(rewards_list_who))
        rewards_avg_dis = np.mean(np.array(rewards_list_dis))
        rewards_avg_man = np.mean(np.array(rewards_list_man))
        total_avg_reward = np.mean(np.array([rewards_avg_ret, rewards_avg_who, rewards_avg_dis, rewards_avg_man])) 

#         with open('./result/성능측정_Seasonal_AfterEval_StepAvg_LearnableAlpha.csv', 'a', newline='\n') as f :
#             newFileWriter = csv.writer(f)
#             newFileWriter.writerow([rewards_avg_ret, rewards_avg_who, rewards_avg_dis, rewards_avg_man, total_avg_reward, PATH])
        
        hist_alpha(PATH, action_coop_list_ret, action_coop_list_who, action_coop_list_dis, action_coop_list_man)
    

MAX_STEPS = 100  # 1에피소드 당 최대 단계 수
NUM_EPISODES = 1500  # 최대 에피소드 수
ENV = "BeerGame-v0"
ENV_TYPE = 'classical'
GAMMA = 0.9

for idx in range(10) :

    
    PATH = '{}_Learnable Alpha _ Seasonal _ {}'.format(datetime.now().strftime('%Y%m%d'), idx)
    if "result" not in os.listdir('./') :
        os.mkdir('./result')
    if PATH not in os.listdir('./result') :
        os.mkdir('./result/{}'.format(PATH))

    with open("./result/{}/학습 정보.txt".format(PATH), "w") as f:
        f.write("ENV >>> {}\n".format(ENV))
        f.write("ENV_TYPE >>> {}\n".format(ENV_TYPE))    
        f.write("GAMMA >>> {}\n".format(GAMMA))
        f.write("MAX_STEPS >>> {}\n".format(MAX_STEPS))
        f.write("NUM_EPISODES >>> {}\n".format(NUM_EPISODES))
    #     f.write("BETA >>> {}\n".format(BETA))
        f.write("CAPACITY >>> {}\n".format(CAPACITY))
        f.write("Step History >>> 10\n")
        f.write("Demand >>> normal\n")
        f.write("num_actions >>> 21")

    device = torch.device(f'cuda:{0}' if torch.cuda.is_available() else 'cpu')
    if device.type != 'cpu':
        torch.cuda.set_device(device) # change allocation of current GPU
    BS_level=[95,90,95,74]<jupyter_output><empty_output><jupyter_text>학습<jupyter_code># 학습 
simon_env = Environment(env_type=ENV_TYPE, BS_level=BS_level)
rewards_avg_list_ret, rewards_avg_list_who, rewards_avg_list_dis, rewards_avg_list_man = simon_env.train()
try :
    # 테스트 _ 학습 돌려보고 그림 그리기
    simon_env = Environment(env_type=ENV_TYPE, BS_level = BS_level)
    simon_env.test()
except Exception as e :
    print(e)<jupyter_output><empty_output><jupyter_text>테스트<jupyter_code>MAX_STEPS = 100  # 1에피소드 당 최대 단계 수
NUM_EPISODES = 1500  # 최대 에피소드 수
ENV = "BeerGame-v0"
ENV_TYPE = 'seasonal_beer'
GAMMA = 0.9

PATH = 'Learnable Alpha _ Seasonal'

# 성능 측정 _ 성능 기록하기
test_env = TestEnvironment(env_type=ENV_TYPE, BS_level = BS_level)
test_env.test()

print('done ... {} >>> '.format(datetime.now().strftime('%Y-%m-%d %H:%M:%S')))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13677576293675364, 'fraction_numerical': 0.019128704113224235, 'mean_word_length': 2.7813545150501673, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 46, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '48070', 'license': 'MIT', 'n_tokens_mistral': 3554, 'n_tokens_neox': 3676, 'n_words': 485, 'path': 'Beer Game(POMDP) - DQN agent with BS agents-Seasonal Data - Learnable Alpha.ipynb', 'repo_name': 'zerojsh00/Reinforcement-Learning', 'size': '9044'}