def adam_update_numpy_amsgrad(param, g_t, t, m, v, vhat, lr=0.001, beta1=0.9, beta2=0.999, epsilon=1e-07):
    lr_t = ((lr * np.sqrt((1 - (beta2 ** (t + 1))))) / (1 - (beta1 ** (t + 1))))
    m_t = ((beta1 * m) + ((1 - beta1) * g_t))
    v_t = ((beta2 * v) + (((1 - beta2) * g_t) * g_t))
    vhat_t = np.maximum(vhat, v_t)
    param_t = (param - ((lr_t * m_t) / (np.sqrt(vhat_t) + epsilon)))
    return (param_t, m_t, v_t, vhat_t)
def adam_update_numpy_amsgrad(param, g_t, t, m, v, vhat, lr=0.001, beta1=0.9, beta2=0.999, epsilon=1e-07):
    lr_t = ((lr * np.sqrt((1 - (beta2 ** (t + 1))))) / (1 - (beta1 ** (t + 1))))
    m_t = ((beta1 * m) + ((1 - beta1) * g_t))
    v_t = ((beta2 * v) + (((1 - beta2) * g_t) * g_t))
    vhat_t = np.maximum(vhat, v_t)
    param_t = (param - ((lr_t * m_t) / (np.sqrt(vhat_t) + epsilon)))
    return (param_t, m_t, v_t, vhat_t)