def train_by_numpy():
    x = init_value
    mt = np.zeros_like(x)
    vt = np.zeros_like(x)
    beta1 = betas[0]
    beta2 = betas[1]
    if adam_w_mode:
        l2 = 0
        wd = weight_decay
    else:
        l2 = weight_decay
        wd = 0

    def np_train_one_iter(step, grad):
        if (clip_grad_max_norm != (- 1)):
            (_, grad) = clip_grad_norm_np(grad, clip_grad_max_norm, clip_grad_norm_type)
        grad = (grad + (l2 * x))
        bias_correction1 = 1.0
        bias_correction2 = 1.0
        if do_bias_correction:
            bias_correction1 = (1.0 - np.power(beta1, (step + 1)))
            bias_correction2 = (1.0 - np.power(beta2, (step + 1)))
        m = ((beta1 * mt) + ((1 - beta1) * grad))
        v = ((beta2 * vt) + (((1 - beta2) * grad) * grad))
        denom = ((np.sqrt(v) / np.sqrt(bias_correction2)) + eps)
        adam_diff = ((m / bias_correction1) / denom)
        w_norm = np.linalg.norm(x, ord=2)
        g_norm = np.linalg.norm(adam_diff, ord=2)
        if ((w_norm > 0) and (g_norm > 0)):
            trust_ratio = (w_norm / g_norm)
        else:
            trust_ratio = 1.0
        param = (x - ((learning_rate * trust_ratio) * (adam_diff + (wd * x))))
        return (param, m, v)
    for i in range(train_iters):
        (x, mt, vt) = np_train_one_iter(i, random_grad_seq[i])
    return x
def train_by_numpy():
    x = init_value
    mt = np.zeros_like(x)
    vt = np.zeros_like(x)
    beta1 = betas[0]
    beta2 = betas[1]
    if adam_w_mode:
        l2 = 0
        wd = weight_decay
    else:
        l2 = weight_decay
        wd = 0

    def np_train_one_iter(step, grad):
        if (clip_grad_max_norm != (- 1)):
            (_, grad) = clip_grad_norm_np(grad, clip_grad_max_norm, clip_grad_norm_type)
        grad = (grad + (l2 * x))
        bias_correction1 = 1.0
        bias_correction2 = 1.0
        if do_bias_correction:
            bias_correction1 = (1.0 - np.power(beta1, (step + 1)))
            bias_correction2 = (1.0 - np.power(beta2, (step + 1)))
        m = ((beta1 * mt) + ((1 - beta1) * grad))
        v = ((beta2 * vt) + (((1 - beta2) * grad) * grad))
        denom = ((np.sqrt(v) / np.sqrt(bias_correction2)) + eps)
        adam_diff = ((m / bias_correction1) / denom)
        w_norm = np.linalg.norm(x, ord=2)
        g_norm = np.linalg.norm(adam_diff, ord=2)
        if ((w_norm > 0) and (g_norm > 0)):
            trust_ratio = (w_norm / g_norm)
        else:
            trust_ratio = 1.0
        param = (x - ((learning_rate * trust_ratio) * (adam_diff + (wd * x))))
        return (param, m, v)
    for i in range(train_iters):
        (x, mt, vt) = np_train_one_iter(i, random_grad_seq[i])
    return x