<jupyter_start><jupyter_text>Just one fold (normal train/dev)<jupyter_code># K = 1 normal dev
K = 1
i = 0

path = '../data/folds/'+str(K)
os.makedirs(path)

y_t = train['label']
X_t = train.drop('label',axis=1,inplace=False)

y_d = dev['label']
X_d = dev.drop('label',axis=1,inplace=False)

X_t.to_csv(path+'/X_train_split_'+str(i+1)+'.csv',index=False)
X_d.to_csv(path+'/X_dev_split_'+str(i+1)+'.csv',index=False)
y_t.to_csv(path+'/y_train_split_'+str(i+1)+'.csv',index=False)
y_d.to_csv(path+'/y_dev_split_'+str(i+1)+'.csv',index=False)

test.to_csv(path+'/X_test.csv',index=False)<jupyter_output><empty_output><jupyter_text>KFOLD<jupyter_code>train = pd.concat([train,dev],axis=0)
K = [5,10]

y = train['label']
X = train.drop('label',axis=1,inplace=False)

for k in K:
    print('K = ',k)

    path = '../data/folds/'+str(k)
    os.makedirs(path)

    sss = StratifiedKFold(n_splits=k, random_state=420,shuffle=True)

    for i, (train_index, dev_index) in enumerate(sss.split(X,y)):
        print('--------FOLD ',i)
        X_t, X_d = X.iloc[train_index], X.iloc[dev_index]
        y_t, y_d = y.iloc[train_index], y.iloc[dev_index]

        X_t.to_csv(path+'/X_train_split_'+str(i+1)+'.csv',index=False)
        X_d.to_csv(path+'/X_dev_split_'+str(i+1)+'.csv',index=False)
        y_t.to_csv(path+'/y_train_split_'+str(i+1)+'.csv',index=False)
        y_d.to_csv(path+'/y_dev_split_'+str(i+1)+'.csv',index=False)

    test.to_csv(path+'/X_test.csv',index=False)<jupyter_output>K =  5
--------FOLD  0
--------FOLD  1
--------FOLD  2
--------FOLD  3
--------FOLD  4
K =  10
--------FOLD  0
--------FOLD  1
--------FOLD  2
--------FOLD  3
--------FOLD  4
--------FOLD  5
--------FOLD  6
--------FOLD  7
--------FOLD  8
--------FOLD  9
{'fraction_non_alphanumeric': 0.2557603686635945, 'fraction_numerical': 0.022465437788018433, 'mean_word_length': 3.2365853658536587, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '492504', 'license': 'MIT', 'n_tokens_mistral': 795, 'n_tokens_neox': 777, 'n_words': 102, 'path': 'notebooks/folds.ipynb', 'repo_name': 'VictorCallejas/FB_MMHM', 'size': '1736'}