<jupyter_start><jupyter_text>Config Params<jupyter_code>labels_file = "uo_dress_labels.csv"
url_file = "uo_dress_urls.csv"
min_file_size = 3500
save_dir = './uo_dress'<jupyter_output><empty_output><jupyter_text>Get all of the product IDs from labels file<jupyter_code>!head {labels_file}
product_dir_map = {}
with open(labels_file, "r") as f:
    reader = csv.reader(f)
    for row in reader:
        product_id = row[0].split('.')[0].split('/')[-1]
        product_dir_map[product_id] = row[0]
        
print("{} entries loaded from {}".format(len(product_dir_map), labels_file))<jupyter_output>5452 entries loaded from urbn_dress_labels.csv<jupyter_text>Get all of the URLs<jupyter_code>!head {url_file}
product_url_map = {}
with open(url_file, "r") as f:
    reader = csv.reader(f)
    for row in reader:
        product_id, product_url = row
        product_url_map[product_id] = product_url
        
print("{} entries loaded from {}".format(len(product_dir_map), url_file))<jupyter_output>5452 entries loaded from urbn_dress_urls.csv<jupyter_text>Make output dir<jupyter_code>!mkdir -p {save_dir}/train
!mkdir -p {save_dir}/test
!mkdir -p {save_dir}/val<jupyter_output><empty_output><jupyter_text>Loop through images, download, and save to appropriate directory<jupyter_code>cnt = 0
t1 = time.time()
for product_id, product_dir in product_dir_map.items():
    if cnt % 100 == 0:
        print("{}/{}. Elapsed: {} sec ...".format(cnt, len(product_dir_map), time.time()-t1))
    cnt += 1
    product_url = product_url_map[product_id]
    save_file = "{}/{}".format(save_dir, product_dir)
    _,msg = urllib.request.urlretrieve(product_url, save_file)
    sz = int(msg.get_all('Content-Length')[0])
    if sz < min_file_size:
        print("Warning: file size a bit small {}. Check {} to make sure it's valid. URL for ID {}: {}".format(sz, product_dir, product_id, product_url))
print("Complete! Check logs above. Done in {} sec.".format(t2-time.time()))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14098690835850958, 'fraction_numerical': 0.013091641490433032, 'mean_word_length': 4.896142433234421, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '643572', 'license': 'Apache-2.0', 'n_tokens_mistral': 745, 'n_tokens_neox': 724, 'n_words': 153, 'path': 'data_generation/uo_dress/download_uo_dress_data.ipynb', 'repo_name': 'tszumowski/custom-vision-study', 'size': '1986'}