def evaluate_policy(env, policy, num_episodes):
    total_return = 0.0
    total_num_steps = 0.0
    for _ in range(num_episodes):
        time_step = env.reset()
        episode_return = 0.0
        episode_num_steps = 0.0
        while (not time_step.is_last()):
            action_step = policy.action(time_step)
            time_step = env.step(action_step.action)
            episode_return += time_step.reward
            episode_num_steps += 1
        total_return += episode_return
        total_num_steps += episode_num_steps
    return ((total_return / num_episodes).numpy()[0], (total_num_steps / num_episodes))
def evaluate_policy(env, policy, num_episodes):
    total_return = 0.0
    total_num_steps = 0.0
    for _ in range(num_episodes):
        time_step = env.reset()
        episode_return = 0.0
        episode_num_steps = 0.0
        while (not time_step.is_last()):
            action_step = policy.action(time_step)
            time_step = env.step(action_step.action)
            episode_return += time_step.reward
            episode_num_steps += 1
        total_return += episode_return
        total_num_steps += episode_num_steps
    return ((total_return / num_episodes).numpy()[0], (total_num_steps / num_episodes))