<jupyter_start><jupyter_text>Load the data set * There are totally 1,225,029 training images and 117,703 test images. * Totoally 14,951 landmarks<jupyter_code>train = pd.read_csv('./data/train.csv')
print('Train:\t\t', train.shape)<jupyter_output>Train:		 (1225029, 3)<jupyter_text>Download Images<jupyter_code>def fetch_image(url):
    """ Get image from given url """
    response=requests.get(url, stream=True)
    
    with open('./data/image_9.jpg', 'wb') as out_file:
        shutil.copyfileobj(response.raw, out_file)
        
    del response
# Download images to ./train/
urls = train['url'].values
t0 = time.time()

# Loop through urls to download images
for idx in range(900000, 1000000):
    url = urls[idx]
    # Check if already downloaded
    if os.path.exists('./data/tmp9/' + str(idx) + '.jpg'):
        continue
        
    # Get image from url
    fetch_image(url)
    os.rename('./data/image_9.jpg', './data/tmp9/'+ str(idx) + '.jpg')
    
    # Helpful information
    if idx % 10000 == 0:
        t = time.time() - t0
        print('\nProcess: {:9d}'.format(idx), '   Used time: {} s'.format(np.round(t, 0)))
        t0 = time.time()
    if idx % 125 == 0:
        print('=', end='')<jupyter_output>Process:    900000    Used time: 0.0 s
================================================================================
Process:    910000    Used time: 4563.0 s
================================================================================
Process:    920000    Used time: 4607.0 s
================================================================================
Process:    930000    Used time: 4859.0 s
================================================================================
Process:    940000    Used time: 4302.0 s
================================================================================
Process:    950000    Used time: 4640.0 s
================================================================================
Process:    960000    Used time: 4409.0 s
================================================================================
Process:    970000    Used time: 4534.0 s
================================================================================
Process:    980000[...]
{'fraction_non_alphanumeric': 0.3885893980233603, 'fraction_numerical': 0.06693620844564241, 'mean_word_length': 4.252358490566038, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '16067', 'license': 'MIT', 'n_tokens_mistral': 704, 'n_tokens_neox': 573, 'n_words': 151, 'path': 'other/deprecated/tmp9.ipynb', 'repo_name': 'JifuZhao/Landmark_Recognition', 'size': '2226'}