<jupyter_start><jupyter_text>Simulate regression analysis<jupyter_code># Get distances
x_e = [] # 2D Euclidean distances
x_r1 = [] # 1D Rank differences (competence)
x_r2 = [] # 1D Rank differences (popularity)
dists_p = []
dists_c = []
for idx1, idx2 in combinations(range(16),2):
    x_e.append(np.linalg.norm(x[idx1] - x[idx2]))
    x_r1.append(abs(x[idx1][0] - x[idx2][0]))
    x_r2.append(abs(x[idx1][1] - x[idx2][1]))
    
    f1_p = h_popularity[idx1]
    f2_p = h_popularity[idx2]
    dist_p = np.linalg.norm(f1_p - f2_p)
    dists_p.append(dist_p)
    
    f1_c = h_competence[idx1]
    f2_c = h_competence[idx2]
    dist_c = np.linalg.norm(f1_c - f2_c)
    dists_c.append(dist_c)

dependent = np.array(dists_p + dists_c)
x_r = np.expand_dims(np.array(x_r2 + x_r1), axis=1)
x_e = np.expand_dims(np.array(x_e + x_e), axis=1)
independent = np.concatenate([x_r, x_e], axis=1)
independent = sm.add_constant(independent)
stats_model = sm.OLS(dependent,independent).fit()         
print(stats_model.pvalues)
print(stats_model.tvalues)
print(stats_model.params) 
print(stats_model.bse)<jupyter_output>[1.15007939e-01 8.85262193e-85 7.24370152e-62]
[ 1.58189138 30.78478739 22.85815736]
[0.13617379 1.35548828 1.09888088]
[0.08608289 0.04403111 0.0480739 ]<jupyter_text>Results for simple grid<jupyter_code># Get distances
x_e = [] # 2D Euclidean distances
x_r1 = [] # 1D Rank differences (competence)
x_r2 = [] # 1D Rank differences (popularity)
dists_p = []
dists_c = []
for idx1, idx2 in combinations(range(16),2):
    x_e.append(np.linalg.norm(x[idx1] - x[idx2]))
    x_r1.append(abs(x[idx1][0] - x[idx2][0]))
    x_r2.append(abs(x[idx1][1] - x[idx2][1]))
    
    f1_p = h_popularity_b[idx1]
    f2_p = h_popularity_b[idx2]
    dist_p = np.linalg.norm(f1_p - f2_p)
    dists_p.append(dist_p)
    
    f1_c = h_competence_b[idx1]
    f2_c = h_competence_b[idx2]
    dist_c = np.linalg.norm(f1_c - f2_c)
    dists_c.append(dist_c)


dependent = np.array(dists_p + dists_c)
x_r = np.expand_dims(np.array(x_r2 + x_r1), axis=1)
x_e = np.expand_dims(np.array(x_e + x_e), axis=1)
independent = np.concatenate([x_r, x_e], axis=1)
independent = sm.add_constant(independent)

stats_model = sm.OLS(dependent,independent).fit()         
print(stats_model.pvalues)
print(stats_model.tvalues)
print(stats_model.params) 
print(stats_model.bse)<jupyter_output>[0.99205121 0.99180665 0.        ]
[-9.97300833e-03  1.02798603e-02  6.13544330e+02]
[-5.97636878e-05  3.15094750e-05  2.05328715e+00]
[0.00599254 0.00306517 0.0033466 ]
{'fraction_non_alphanumeric': 0.15125049622866216, 'fraction_numerical': 0.12068281063914252, 'mean_word_length': 4.384615384615385, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '487194', 'license': 'Apache-2.0', 'n_tokens_mistral': 1323, 'n_tokens_neox': 1170, 'n_words': 165, 'path': 'notebooks/fake_simulations/1D_orthogonal_simulation.ipynb', 'repo_name': 'MaryZolfaghar/WCSLS', 'size': '2519'}