def __init__(self, model, train_config, model_config, current_step):
    self._optimizer = torch.optim.Adam(model.parameters(), betas=train_config['optimizer']['betas'], eps=train_config['optimizer']['eps'], weight_decay=train_config['optimizer']['weight_decay'])
    self.n_warmup_steps = train_config['optimizer']['warm_up_step']
    self.anneal_steps = train_config['optimizer']['anneal_steps']
    self.anneal_rate = train_config['optimizer']['anneal_rate']
    self.current_step = current_step
    self.init_lr = np.power(model_config['transformer']['encoder_hidden'], (- 0.5))
def __init__(self, model, train_config, model_config, current_step):
    self._optimizer = torch.optim.Adam(model.parameters(), betas=train_config['optimizer']['betas'], eps=train_config['optimizer']['eps'], weight_decay=train_config['optimizer']['weight_decay'])
    self.n_warmup_steps = train_config['optimizer']['warm_up_step']
    self.anneal_steps = train_config['optimizer']['anneal_steps']
    self.anneal_rate = train_config['optimizer']['anneal_rate']
    self.current_step = current_step
    self.init_lr = np.power(model_config['transformer']['encoder_hidden'], (- 0.5))