<jupyter_start><jupyter_text>Training and Validation data<jupyter_code>train_val_data('lex', 3, index1, split_label1, data_label1, sense_count1, [], lex_cond=False, pos_cond=True)
train_val_data('sense', 4, index2, split_label2, data_label2, sense_count2, [], lex_cond=True, pos_cond=True)
train_val_data('full_sense', 5, index3, split_label3, data_label3, sense_count3, [], lex_cond=True, pos_cond=True)<jupyter_output>/users/btech/aviraj/envs/lib/python3.5/site-packages/sklearn/model_selection/_split.py:2026: FutureWarning: From version 0.21, test_size will always complement train_size unless both are specified.
  FutureWarning)<jupyter_text>Test data<jupyter_code>test_data('lex', 3, test_index1, data_test_label1, sense_count1, lex_cond=False, pos_cond=True)
test_data('sense', 4, test_index2, data_test_label2, sense_count2, lex_cond=True, pos_cond=True)
test_data('full_sense', 5, test_index3, data_test_label3, sense_count3, lex_cond=True, pos_cond=True)
sampled_sense_count1 = [('1:19', 10000),
 ('1:17', 10000),
 ('2:34', 10000),
 ('2:33', 10000),
 ('1:27', 10000),
 ('2:37', 8000),
 ('1:24', 8000),
 ('1:08', 8000),
 ('1:12', 7000),
 ('1:22', 5000),
 ('2:29', 5000),
 ('1:05', 3000),
 ('1:16', 3000),
 ('1:25', 3000),
 ('1:20', 3000),
 ('1:13', 2000)]
sampled_sense_count2= []
for s, c in sense_count2[120:]:
    sampled_sense_count2.append((s, 5000))
for s, c in sense_count2[75:120]:
    sampled_sense_count2.append((s, 8000))
for s, c in sense_count2[25:75]:
    sampled_sense_count2.append((s, 12000))
sampled_sense_count3= []
for s, c in sense_count3[130:]:
    sampled_sense_count3.append((s, 5000))
for s, c in sense_count3[70:130]:
    sampled_sense_count3.append((s, 8000))
for s, c in sense_count3[25:70]:
    sampled_sense_count3.append((s, 12000))
train_val_data('lex_sampled', 3, index1, split_label1, data_label1, sense_count1, sampled_sense_count1, lex_cond=False, pos_cond=True, sampling=True)
train_val_data('sense_sampled', 4, index2, split_label2, data_label2, sense_count2, sampled_sense_count2, lex_cond=True, pos_cond=True, sampling=True)
train_val_data('full_sense_sampled', 5, index3, split_label3, data_label3, sense_count3, sampled_sense_count3, lex_cond=True, pos_cond=True, sampling=True)<jupyter_output>/home/sshanukr/env/lib/python3.5/site-packages/sklearn/model_selection/_split.py:2026: FutureWarning: From version 0.21, test_size will always complement train_size unless both are specified.
  FutureWarning)
{'fraction_non_alphanumeric': 0.15118174409127955, 'fraction_numerical': 0.09983700081499593, 'mean_word_length': 8.194756554307116, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '451262', 'license': 'MIT', 'n_tokens_mistral': 1114, 'n_tokens_neox': 937, 'n_words': 183, 'path': 'one_million/One-Million All-Word Data-hierarchical Sampling-Fine.ipynb', 'repo_name': 'Sshanu/WSD', 'size': '2454'}