<jupyter_start><jupyter_text>Feature Bias 2feature bias without noise<jupyter_code>import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
# we will have a=1 be the disadvantaged group

# rho_a - float
# rho_z - float
# beta - float array size 2, error rate for each group
# N - int, number of samples
# d - int, number of features
# mu - array of arrays with dimensions [a][z]
# d_shared - int, number of shared features

def feature_bias2(rho_a, rho_z, beta, N, d, d_shared, mu):
    # portion of disadvantaged group  
    p_a = [1-rho_a, rho_a]
    # portion of allocation of target variable
    p_z = [1-rho_z, rho_z]
    cov = np.eye(d)
    
    a = np.random.choice([0,1], p=p_a, size=N)
    z = np.random.choice([0,1], p=p_z, size=N)
    x = [np.random.multivariate_normal(mu[a_i][z_i],cov) for a_i, z_i in zip(a,z)]

    x = np.asarray(x)
    
    y = z
    data = np.asarray([a,z,y]).T
    df = pd.DataFrame(data=data, columns = ['a','z','y'])

    var_list = []
    for i in range(d):
        var = 'x' + str(i)
        df[var] = x[:,i]
        var_list.append(var)
    
    df.head()
    
    return df
feature_bias2(.4, .2, [0, .2], 200, 5, 3, [[[1,5,3,6,3],[5,1,1,5,2]], [[5,1,1,5,2],[1,5,3,6,3]]])<jupyter_output>a  z  y        x0        x1        x2        x3        x4
0    1  1  1  1.992228  4.062293  2.674484  5.716203  4.466058
1    1  0  0  4.871235  1.987268  1.441285  6.227233  2.278426
2    1  0  0  5.317381  0.344389  1.141238  4.057307  1.063364
3    1  0  0  5.322915  0.918424  1.341755  5.721117  1.400478
4    0  0  0  1.582240  5.566733  2.569709  6.951185  4.757884
..  .. .. ..       ...       ...       ...       ...       ...
195  1  0  0  5.896880  0.178956  1.860440  4.885493  2.566087
196  1  0  0  4.839298  1.991359  0.968907  4.724085 -0.463806
197  1  1  1 -0.154552  4.737878  4.027563  5.321231  2.342524
198  1  0  0  5.246604  1.672034  1.600368  4.306495  0.997459
199  0  0  0  0.193848  6.375725  4.782787  4.692225  3.742724

[200 rows x 8 columns]
{'fraction_non_alphanumeric': 0.1364082433758587, 'fraction_numerical': 0.22031403336604513, 'mean_word_length': 2.1082317073170733, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '43556', 'license': 'MIT', 'n_tokens_mistral': 1131, 'n_tokens_neox': 936, 'n_words': 242, 'path': 'sandbox/feature_bias2_generator.ipynb', 'repo_name': 'brownsarahm/ml-sim', 'size': '2038'}