<jupyter_start><jupyter_text>Fields for Experts Initially, we chose 250 fields manually for experts. Here I will make more sets by code.<jupyter_code>import pandas as pd
param_in_data_dir = "/Users/hn/Documents/01_research_data/NASA/parameters/"<jupyter_output><empty_output><jupyter_text>Read all the ~6000 fields chosen previously<jupyter_code>evaluation_set = pd.read_csv(param_in_data_dir + "Eshwar_Extensive.csv")
evaluation_set.drop(labels=['Text box Text', 'Question_overall'], axis='columns', inplace=True)
print (evaluation_set.shape)
evaluation_set.head(1)
sorted(evaluation_set.CropTyp.unique())
wanted_crops = ['bean, dry', 'bean, green', 'buckwheat', 'canola', 'carrot',
                'triticale', 'market crops', 'grass hay', 'mint', 'onion', 
                'barley', 'barley hay', 'oat hay', 'yellow mustard', 'corn seed', 
                'corn, field', 'corn, sweet', 'wheat', 'potato', 'alfalfa seed', 'grass seed',
                'bluegrass seed', 'pea seed', 'pea, dry', 'pea, green']
wanted_crops = sorted(wanted_crops)
len(wanted_crops)
# Pick up the crops that Kirti wanted
evaluation_set = evaluation_set[evaluation_set.CropTyp.isin(wanted_crops)]
evaluation_set.shape<jupyter_output><empty_output><jupyter_text>Read the first set; 294 fields labeled in the first set<jupyter_code>perry_dir = "/Users/hn/Documents/01_research_data/NASA/Perry_and_Co/"
set1_fields = pd.read_csv(perry_dir + "set_1_experts_stats_extended_sortOpinionCrop.csv")<jupyter_output><empty_output><jupyter_text>choices_set_1_exel = pd.ExcelFile(perry_dir + "set1_PerryandCo.xlsx")choices_set_1_sheet_names = choices_set_1_exel.sheet_namesall_choices_exel = pd.DataFrame()for a_choice_sheet in choices_set_1_sheet_names:     read a damn sheet    a_choice_sheet = choices_set_1_exel.parse(a_choice_sheet)    all_choices_exel = pd.concat([all_choices_exel, a_choice_sheet])all_choices_exel.shape print('There are [{ques_count}] questions.'.format(ques_count=question_count)) sorted(set1_fields.ID) == sorted(all_choices_exel.ID)<jupyter_code>for a_crop in wanted_crops:
    a_df = evaluation_set[evaluation_set.CropTyp == a_crop]
    print('There are [{field_count}] crops of type [{curr_crop}].'.format(field_count=a_df.shape[0], 
                                                                          curr_crop=a_crop))
print (evaluation_set.shape)
evaluation_set = evaluation_set[~(evaluation_set.ID.isin(list(set1_fields.ID)))]
evaluation_set.shape
len(set1_fields.ID.unique())
1925-1640
min_no_fields = 10

for set_count in range(2, 5):
    curr_set = pd.DataFrame()
    for a_crop in wanted_crops:
        a_df = evaluation_set[evaluation_set.CropTyp == a_crop].copy()
        row = min(min_no_fields, a_df.shape[0])
        a_df = a_df.iloc[0:row, ].copy()
        curr_set = pd.concat([curr_set, a_df])
    
    out_name = perry_dir + "Newset" + str(set_count) + ".csv"
    curr_set.to_csv(out_name, index = False)
# set_2 = pd.read_csv(perry_dir + "set_2.csv")
# set_3 = pd.read_csv(perry_dir + "set_3.csv")
# set_4 = pd.read_csv(perry_dir + "set_4.csv")
del(curr_set, set_count)<jupyter_output><empty_output><jupyter_text>Write the sets in excel format<jupyter_code>no_questions = 50

writer_limited = pd.ExcelWriter(perry_dir + 'set_2_to_4.xlsx', engine='xlsxwriter') 

for set_count in range(2, 5):
    curr_set = pd.read_csv(perry_dir + "set_" + str(set_count) + ".csv")
    curr_set.sort_values(by=['CropTyp', 'county', "ID"], inplace=True)

    if curr_set.shape[0] % no_questions != 0:
        no_dfs = curr_set.shape[0] // no_questions + 1
    else:
        no_dfs = curr_set.shape[0] // no_questions

    for ii in range(no_dfs):
        curr_sheet = curr_set.loc[(ii*no_questions): ((ii+1) * no_questions) - 1, ]
        curr_sheet.reset_index(drop=True, inplace=True)
        curr_sheet.to_excel(writer_limited, sheet_name= "set_" + str(set_count) + "_form_" + str(ii+1), index=False)

writer_limited.save()
len(evaluation_set.CropTyp.unique())<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12987987987987987, 'fraction_numerical': 0.015265265265265265, 'mean_word_length': 4.877941176470588, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 1, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '471645', 'license': 'MIT', 'n_tokens_mistral': 1557, 'n_tokens_neox': 1483, 'n_words': 254, 'path': 'NASA/Python_codes/Perry_and_Co/pick_fields_4_experts_Set_2_useless.ipynb', 'repo_name': 'HNoorazar/Kirti', 'size': '3996'}