<jupyter_start><jupyter_text>Start create<jupyter_code>origin_data_dir = '/root/notebooks/final/caltech_pedestrian_refine_no'
img_data = glob.glob(origin_data_dir+'/street/*.jpg', recursive=True)

json_data = glob.glob(origin_data_dir+'/street_json/*.json', recursive=True)
img_data[:10]
json_data[:10]
# Image read dir
street_dir = '/root/notebooks/0858611-2/final_project/caltech_pedestrian_extractor/video_extractor/*'

people_dir = '/root/notebooks/final/market_mask2'

# Image save dir
save_dir = '/root/notebooks/final/result_dataset_13'

num_imgs = 42000
# Check dir folder exit
# If not, create one
if os.path.exists(save_dir) == False:
    os.makedirs(save_dir)

for s in ['people', 'mask', 'street', 'street_json','json']:
    if os.path.exists(os.path.join(save_dir, s)) == False:
        os.makedirs(os.path.join(save_dir, s))
#street_imgs = glob.glob(street_dir+'/**/*.jpg', recursive=True)
street_imgs = img_data

#street_imgs = random.shuffle(random.sample(street_imgs, 5000))
street_imgs = random.sample(street_imgs, num_imgs)

random.shuffle(street_imgs)
#people_imgs_ = glob.glob(people_dir+'/people/*.jpg', recursive=True)

mask_imgs_ = glob.glob(people_dir+'/market_mask_refine_6467/*.jpg', recursive=True)

people_imgs_ = [i.replace('market_mask_refine_6467', 'people') for i in mask_imgs_]

#mask_imgs_ = [i.replace('people', 'mask') for i in people_imgs_]

#people_imgs = random.sample(people_imgs, num_imgs)

r_t_imgs = []
r_m_imgs = []
mask_imgs_[:10]
# for i in range(len(people_imgs_)):
#     if (i%1000==0):
#         print("Process (",i,"/",len(people_imgs_),")  ","{:.2f}".format(100*i/len(people_imgs_))," %")
#     r_t_imgs.append(cv2.imread(people_imgs_[i]))
#     r_m_imgs.append(cv2.imread(mask_imgs_[i]))
#people_imgs = [people_imgs_[random.randint(0,len(people_imgs_)-1)] for i in range(num_imgs)]


#random.shuffle(people_imgs)
street_imgs[:10]
# copy people image and mask to dir

pbar = tqdm(total=len(people_imgs_))

for i in range(len(people_imgs_)):
    shutil.copyfile(mask_imgs_[i], save_dir+'/mask/'+str('{0:06}'.format(i))+'.jpg')
    shutil.copyfile(people_imgs_[i], save_dir+'/people/'+str('{0:06}'.format(i))+'.jpg')
    pbar.update()
    
pbar.close()
for i in range(num_imgs):
    
    if (i%100==0):
        print("Process (",i,"/",num_imgs,")  ","{:.2f}".format(100*i/num_imgs)," %")
        
    people_pick = random.randint(0,len(people_imgs_)-1)
    
    # create mask and save
    #try:
    #    mask_img = detectron_mask_img(people_imgs[i],(64,128))
    #    mask_img = Image.fromarray(mask_img)
    #except Exception as e:
    #    print("Skip image :",i)
    #    continue
    
    #mask_img = mask_imgs_[people_pick]
        
    #mask_img.save(save_dir+'/mask/'+str('{0:06}'.format(i))+'.jpg')
    
    #shutil.copyfile(mask_img, save_dir+'/mask/'+str('{0:06}'.format(i))+'.jpg')
    
    # save street img
    street_img = cv2.imread(street_imgs[i])
    street_img = cv2.resize(street_img,(640,480))
    cv2.imwrite(save_dir+'/street/'+str('{0:06}'.format(i))+'.jpg', street_img)
    
    ################################################################
    img_path = street_imgs[i]
    json_dir = img_path.replace('street', 'street_json')
    json_dir = json_dir.replace('jpg', 'json')
    shutil.copyfile(json_dir, save_dir+'/street_json/'+str('{0:06}'.format(i))+'.json')
    ################################################################
    
    # save poeple img
    #people_img = cv2.imread(people_imgs[i])
    people_img = people_imgs_[people_pick]
    #people_img = cv2.resize(people_img,(64,128))
    #cv2.imwrite(save_dir+'/people/'+str('{0:06}'.format(i))+'.jpg', people_img)
    #shutil.copyfile(people_img, save_dir+'/people/'+str('{0:06}'.format(i))+'.jpg')
    
    if random.randint(0,100)>50:
        nump = [str('{0:06}'.format(random.randint(0,len(people_imgs_)-1)))+'.jpg' for i in range(2)]
    else:
        nump = [str('{0:06}'.format(random.randint(0,len(people_imgs_)-1)))+'.jpg' for i in range(3)]
    
    # create json file and save
    create_json_file(save_dir+'/street/'+str('{0:06}'.format(i))+'.jpg',
                     save_dir+'/street_json/'+str('{0:06}'.format(i))+'.json',
                     save_dir+'/json/'+str('{0:06}'.format(i))+'.json',
                     function="random",
                     num_of_people = nump)<jupyter_output>Process ( 0 / 42000 )   0.00  %
Process ( 100 / 42000 )   0.24  %
Process ( 200 / 42000 )   0.48  %
Process ( 300 / 42000 )   0.71  %
Process ( 400 / 42000 )   0.95  %
Process ( 500 / 42000 )   1.19  %
Process ( 600 / 42000 )   1.43  %
Process ( 700 / 42000 )   1.67  %
Process ( 800 / 42000 )   1.90  %
Process ( 900 / 42000 )   2.14  %
Process ( 1000 / 42000 )   2.38  %
Process ( 1100 / 42000 )   2.62  %
Process ( 1200 / 42000 )   2.86  %
Process ( 1300 / 42000 )   3.10  %
Process ( 1400 / 42000 )   3.33  %
Process ( 1500 / 42000 )   3.57  %
Process ( 1600 / 42000 )   3.81  %
Process ( 1700 / 42000 )   4.05  %
Process ( 1800 / 42000 )   4.29  %
Process ( 1900 / 42000 )   4.52  %
Process ( 2000 / 42000 )   4.76  %
Process ( 2100 / 42000 )   5.00  %
Process ( 2200 / 42000 )   5.24  %
Process ( 2300 / 42000 )   5.48  %
Process ( 2400 / 42000 )   5.71  %
Process ( 2500 / 42000 )   5.95  %
Process ( 2600 / 42000 )   6.19  %
Process ( 2700 / 42000 )   6.43  %
Process ( 2800 / 42000 )   6.67 [...]
{'fraction_non_alphanumeric': 0.2005227781926811, 'fraction_numerical': 0.08737864077669903, 'mean_word_length': 3.532148900169205, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '110400', 'license': 'Apache-2.0', 'n_tokens_mistral': 2467, 'n_tokens_neox': 2125, 'n_words': 332, 'path': '01_gene_train_dataset/gandatamask5_multi.ipynb', 'repo_name': 'tony92151/pedestrian_generator', 'size': '5356'}