<jupyter_start><jupyter_text>бостон - градиент уходит в 0  ирис - спустя пару итераций в 0<jupyter_code>dataset = load_breast_cancer()
# dataset = load_boston()
df = pd.DataFrame(dataset['data'])
target = dataset['target']
df = (df - df.mean())/(df.max() - df.min())
df0 = df.copy()
print(df.shape)
print(target)
df.head()
num_nan_cols = 2
nan_fraction = 0.3
nan_cols = np.random.random_integers(0, df.shape[1] - 1, num_nan_cols)
# print(df.isnull().mean())
for col in set(nan_cols):
    df.loc[df.sample(int(nan_fraction * len(df))).index, col] = np.nan
# print(df.isnull().mean())
# print(df.isnull().mean())
nan_coords = np.array(np.where(df.isnull().values)).T
print('Num nan places: {}'.format(nan_coords.shape[0]))
df1 = df.loc[:, df.isnull().sum() == 0]
df2 = df.fillna(df.mean())
print(df1.shape, df2.shape)
arr0 = df0.values
arr1 = df1.values
arr2 = df2.values
print(arr0.shape, arr1.shape, arr2.shape)
print(np.isnan(arr0).sum(), np.isnan(arr1).sum(), np.isnan(arr2).sum())
arr2.shape<jupyter_output><empty_output><jupyter_text>Пилим необходимые функцииdf - исходная с прочерками    df0 - копия исходной таблицы без прочерков  df1 - сокращенная до ненулевых признаков  df2 - исходная с восстановленными  arr0,arr1,arr2 аналогично<jupyter_code>n = len(df0)
# таблица старых значений
Exp1_table = np.zeros((n, n))
for i in tqdm(range(n)):
    for j in range(n):
        Exp1_table[i][j] = np.exp(-norm(arr1[i] - arr1[j])**2)
        
P1_table = np.zeros((n, n))
for i in tqdm(range(n)):
    for j in range(n):
        a = Exp1_table[j][i]
        b = Exp1_table[i].sum() - 1 # 1 = Exp1_table[i][i]
        P1_table[j][i] = a / b
def get_exp2_table():
    Exp2_table = np.zeros((n, n))
    for i in tqdm(range(n)):
        for j in range(n):
            Exp2_table[i][j] = np.exp(-norm(arr2[i] - arr2[j])**2)
    return Exp2_table

def get_p2_table():
    Exp2_table = get_exp2_table()
    P2_table = np.zeros((n, n))
    for i in tqdm(range(n)):
        for j in range(n):
            a = Exp2_table[j][i]
            b = Exp2_table[i].sum() - 1 # 1 = Exp1_table[i][i]
            P2_table[j][i] = a / b
    return P2_table
P2_table = get_p2_table()
def KL():
    s = 0
    for i in range(n):
        for j in range(n):
            s += P1_table[j][i] * np.log(P1_table[j][i] / P2_table[j][i])
    return s

def get_grad(i1, i2):
    def get_i_part(i):
        d = 2*(arr2[i1][i2] - arr2[i][i2])
        s = (P1_table[i1][i]+P1_table[i][i1]) - \
            P2_table[i1][i]*(1+P1_table[i][i]) - \
            P2_table[i][i1]*(1+P1_table[i1][i1])
        return s*d
    return sum(get_i_part(i) for i in range(n) if i!=i1)

def get_full_grad(nan_coords):
    n_gaps = len(nan_coords)
    grad = np.zeros(n_gaps)
    for i in range(n_gaps):
        i1, i2 = nan_coords[i]
        grad[i] = get_grad(i1, i2)
    return grad<jupyter_output><empty_output><jupyter_text>Тестовый прогон  Iterative process<jupyter_code>c = KL()
c
mae = functions.get_mae(arr0, arr2, nan_coords)
mae
cs = []
maes = []
msqes = []
accs = []

mae = functions.get_mae(arr0, arr2, nan_coords)
msqe = functions.get_msqe(arr0, arr2, nan_coords)
acc = functions.get_acc(arr2, target)

cs.append(c)
maes.append(mae)
msqes.append(msqe)
accs.append(acc)
alpha = 0.1
max_iter = 30
print('\tIteration {} //  c {:.6} // mae {:.6} // rmse {:.6} // acc {}'.format(0, c, mae, msqe, acc))
for i in range(max_iter):
    grad = get_full_grad(nan_coords)
    for j, (x,y) in enumerate(nan_coords):
        arr2[x, y] = arr2[x, y] - alpha*grad[j]
    P2_table = get_p2_table()
    c = KL()
    mae = functions.get_mae(arr0, arr2, nan_coords)
    msqe = functions.get_msqe(arr0, arr2, nan_coords)
    acc = functions.get_acc(arr2, target)
    
    accs.append(acc)
    cs.append(c)
    maes.append(mae)
    msqes.append(msqe)
    print('\tIteration {} // c {:.6} // mae {:.6} // rmse {:.6} // acc {}'.format(i+1, c,mae, 
                                                                                                   msqe, acc))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17284257647351406, 'fraction_numerical': 0.031335488684406866, 'mean_word_length': 3.0099700897308077, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '176650', 'license': 'MIT', 'n_tokens_mistral': 1771, 'n_tokens_neox': 1771, 'n_words': 333, 'path': 'notebooks/baseline_sne.ipynb', 'repo_name': 'vasyarv/gap_sne', 'size': '4021'}