<jupyter_start><jupyter_text>Load CSV data<jupyter_code>df = pd.read_csv('via_export_csv.csv',header=0)<jupyter_output><empty_output><jupyter_text>Check for labels other than specified ones<jupyter_code>for i in range(len(df)):
    if ast.literal_eval(df.iloc[i, -1])['name'] != "class1":
        print(df.iloc[i,0],i)
len(df)
cols = ['filename',
        'file_size',
        'file_attributes',
        'region_count',
        'region_id',
        'region_shape_attributes',
        'region_attributes']
filenames = df['filename'].unique()
len(filenames)<jupyter_output><empty_output><jupyter_text>To convert  *img_name, x, y, width, height, class*  type CSV to df (OPTIONAL, use only if you have this kind of data)<jupyter_code># Convert to DataFrame
# final_data = []
# for f in filenames:
#     same_files_len = len(df[df['file_name']==f])
    
#     file_name = [f for _ in range(same_files_len)]
#     file_size = [0 for _ in range(same_files_len)]
#     region_count = [same_files_len for _ in range(same_files_len)]
#     region_id = [i for i in range(same_files_len)]
#     file_attributes = ['"{}"' for _ in range(same_files_len)]
    
#     region_attributes = []
#     clses = df[df['file_name']==f]['class_name']
#     for x in range(len(clses)):
#         t = '{"name":"'+clses.iloc[x]+'"}'
#         region_attributes.append(t)
        
#     region_shape_attributes = []
#     attrs = df[df['file_name']==f][['x_max','x_min','y_max','y_min']]
#     for x in range(len(attrs)):
#         t = '{"name":"rect","x":'+str(attrs.iloc[x,1])+',"y":'+str(attrs.iloc[x,3])+',"width":'+str(int(attrs.iloc[x,0])-int(attrs.iloc[x,1]))+',"height":'+str(int(attrs.iloc[x,2])-int(attrs.iloc[x,3]))+'}'
#         region_shape_attributes.append(t)
    
#     for a,b,c,d,e,f,g in zip(file_name,file_size,file_attributes,region_count,region_id,region_shape_attributes,region_attributes):
#         final_data.append([a,b,c,d,e,f,g])

# cols = ['filename',
#         'file_size',
#         'file_attributes',
#         'region_count',
#         'region_id',
#         'region_shape_attributes',
#         'region_attributes']
# new_df = pd.DataFrame(final_data)
# new_df.columns = cols
# new_df.to_csv("final_data_via.csv",index=False)<jupyter_output><empty_output><jupyter_text>Separate train val and test<jupyter_code># Get train val test files
# Split by index
train_filenames = filenames[:418] 
val_filenames = filenames[418:434]
test_filenames = filenames[434:]

train_files = pd.DataFrame(columns=cols)
for r in train_filenames:
    train_files = train_files.append(new_df[new_df['filename']==r])

val_files = pd.DataFrame(columns=cols)
for r in val_filenames:
    val_files = val_files.append(new_df[new_df['filename']==r])
val_files = pd.DataFrame(val_files, columns=cols)<jupyter_output><empty_output><jupyter_text>Create JSON for training set<jupyter_code>train_json = {}
i=0
for f in train_filenames:
    same_files = train_files[train_files['filename']==f]
    t = {}
    t['filename'] = same_files.iloc[0,0]
    t['size'] = same_files.iloc[0,1]
    
    all_regions = []
    for x in range(len(same_files)):
        t1 = {}
        t1['shape_attributes'] = eval(same_files.iloc[x,5])
        t1['region_attributes'] = eval(same_files.iloc[x,-1])
        all_regions.append(t1)
        
    t['regions'] = all_regions
    
    train_json['x'+str(i)] = t
    i+=1<jupyter_output><empty_output><jupyter_text>Create JSON for validation set<jupyter_code>val_json = {}
i=0
for f in val_filenames:
    same_files = val_files[val_files['filename']==f]
    t = {}
    t['filename'] = same_files.iloc[0,0]
    t['size'] = same_files.iloc[0,1]
    
    all_regions = []
    for x in range(len(same_files)):
        t1 = {}
        t1['shape_attributes'] = eval(same_files.iloc[x,5])
        t1['region_attributes'] = eval(same_files.iloc[x,-1])
        all_regions.append(t1)
        
    t['regions'] = all_regions
    
    val_json['x'+str(i)] = t
    i+=1<jupyter_output><empty_output><jupyter_text>Dump train val JSONs<jupyter_code>train_json = str(train_json).replace("'",'"')
val_json = str(val_json).replace("'",'"')

with open('train_json.json','w') as f:
    f.write(train_json)
    
with open('val_json.json','w') as f:
    f.write(val_json)<jupyter_output><empty_output><jupyter_text>Dump test filenames<jupyter_code>test_filenames
# Dump validation filenames
with open('validation.txt','w') as f:
    for i in val_filenames:
        f.write(i+'\n')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1664809449520838, 'fraction_numerical': 0.010474704702473813, 'mean_word_length': 4.330166270783848, 'pattern_counts': {'":': 7, '<': 33, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '307270', 'license': 'MIT', 'n_tokens_mistral': 1747, 'n_tokens_neox': 1732, 'n_words': 255, 'path': 'Mask_RCNN/docs/csv_to_json.ipynb', 'repo_name': 'adesgautam/objdet', 'size': '4487'}