<jupyter_start><jupyter_text>Access the MET collection via API for the Cultural Data Sculpting course at EPFL<jupyter_code>import urllib.request, urllib.error, urllib.parse
import csv
from bs4 import BeautifulSoup
import html2text
import os
from datetime import datetime
import numpy as np
import pandas as pd
import json
from urllib.request import urlopen
from PIL import Image
# function to check if 'nan'

def isnan(x):
    return x != x
filename = 'artist_qid.csv'
csv_file = csv.reader(open(filename, "r"))
df = pd.read_csv(filename)
#df<jupyter_output><empty_output><jupyter_text>TO DO- DONE!! load the object csv file and retrieve all the relevant Object ID - DONE! call for all the object to get the URLs- Download Images- name them properly for cables implementation  load MET Objects.csv with all the objects of the MET collection<jupyter_code>filename = 'MetObjects.csv'
csv_file = csv.reader(open(filename, "r"))
MET = pd.read_csv(filename,  low_memory=False)

MET<jupyter_output><empty_output><jupyter_text>get the column MET['Artist Wikidata URL'] to compare with Vasari Dataframe<jupyter_code># try and test
MET_urlist = np.array(MET['Artist Wikidata URL']) 
MET_urlist[475726].strip('|').split('|')[0]
# prepare MET artist wikidata list:
liste = []
for i in MET_urlist:
    if isnan(i):
        liste.append('nan')
    else:
        liste.append(i.strip('|').split('|')[0]) 
    
liste[3555]
# prepare df for MET artist wikidata list
wikidata = []
for i in df['Wikidata-Q-ID']:
    wikidata.append('https://www.wikidata.org/wiki/' + str(i))

df['wikidata'] = wikidata    
#df<jupyter_output><empty_output><jupyter_text>if the Wikidata artist URL is in the MET Object list, store the index of the object<jupyter_code>liste = []
for i in df['wikidata']:
    if i in MET_urlist:
        liste.append(np.where(MET_urlist == i)[0])
    else:
        liste.append('nan')
df['MET Object Indices'] = liste
#df<jupyter_output><empty_output><jupyter_text>get the MET Object IDs for the artworks<jupyter_code># process indices to real MET Object IDs

liste = []
for i in df['MET Object Indices']:
    if i == 'nan':
        liste.append('nan')
    else:
        storage = []
        for j in i:
            storage.append(MET.iloc[j]['Object ID'])
        liste.append(storage)
            
df['REAL MET Object IDS'] = liste
#df<jupyter_output><ipython-input-14-ef4217cdccb2>:5: FutureWarning: elementwise comparison failed; returning scalar instead, but in the future will perform elementwise comparison
  if i == 'nan':<jupyter_text>save as csv<jupyter_code>df
df.to_csv('artist_qid.csv', index = False, header=True)<jupyter_output><empty_output><jupyter_text>get all the image URLs from MET API<jupyter_code># make list of image URLs per artist

index = 0
liste = []

for i in range(len(df)):
    
    if df['REAL MET Object IDS'][i] == 'nan':
        storage = ['nan']
        liste.append(storage)
    else:
        
        storage = []
        for j in df['REAL MET Object IDS'][i]:
            
            url = 'https://collectionapi.metmuseum.org/public/collection/v1/objects/' + str(j)
            response = urllib.request.urlopen(url) 
            webContent = response.read()
            my_json = webContent.decode('utf8')
            data = json.loads(my_json)
            storage.append(data['primaryImageSmall'])
        
    
        liste.append(storage)
    
    
    print(storage, index)
    index += 1
    
len(liste) 
df['URL-Liste-small'] = liste


# make new DataFrame with reduced columns

new_df = pd.DataFrame()
new_df["URL-Liste"] = df["URL-Liste"]
new_df["Name"] = df["Name"]
new_df["URL-names"] = df["URL-names"]
new_df["timeline_position"] = df["timeline_position"]
new_df['URL-Liste-small'] = liste

new_df<jupyter_output><empty_output><jupyter_text>function to open an image-url and return it in squared format<jupyter_code># function to open url and return it in squared format 

def getsquared_image(url):
    # open image-url, get width, height
    img = Image.open(urlopen(url))
    width, height = img.size
    # store the bigger one in s
    if width < height:
        s = height
    else:
        s = width
    
    # create empty opaque image file and paste picture
    new_img = Image.new('RGBA', (s,s), (255, 0, 0, 0))
    new_img.paste(img, (s-width, 0))
    
    return new_img

img = getsquared_image('https://images.metmuseum.org/CRDImages/md/web-large/DP169508.jpg')
img
df.columns<jupyter_output><empty_output><jupyter_text>open all URLs and save the squared images names [artist-index]_[image-index].png<jupyter_code># open all the URLs and save the squared images names [artist-index]_[image-index].png

count = 0

# iterate over artists
for i in range(len(df)):
    
    if new_df['URL-Liste-small'][i] == ['nan']:
        count += 1
        print(count)
        continue
    
    else:
        
        index = 0
        # iterate over list of urls
        for j in new_df['URL-Liste-small'][i]:
    
            if j == 'nan':
                index += 1
                continue
            elif j == '':
                index += 1
                continue
            else:
                img = getsquared_image(j.replace(' ', '%20'))
                filename = './thumbnails/MET/' + str(i) + '_' + str(index) + '.png'
                img.save(filename, 'PNG')
                index += 1
        
        print(count)
        count += 1
        
# test with one file 
# open thumbnail, get width, height

img = Image.open(urlopen(new_df['URL-Liste-small'][8][0]))
width, height = img.size

# store the bigger one in s

if width < height:
    s = height
else:
    s = width
s

# create empty Image file in thumbnail size squared
index = '008'

new_img = Image.new('RGB', (s, s))

new_img.paste(img, (0, 0))
#new_img.save('./thumbnails/cables/thumbnail_' + index + '.jpg')
new_img
df
len(liste)
df.columns<jupyter_output><empty_output><jupyter_text>SAVE df as csv<jupyter_code>filename = 'artist_qid.csv'
df.to_csv(filename, index = False, header=True)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1227460711331679, 'fraction_numerical': 0.009263854425144748, 'mean_word_length': 3.1326042378673957, 'pattern_counts': {'":': 0, '<': 43, '<?xml version=': 0, '>': 41, 'https://': 3, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '351860', 'license': 'MIT', 'n_tokens_mistral': 2138, 'n_tokens_neox': 2043, 'n_words': 550, 'path': 'diverse_jupyter_notebooks/Access MET collection.ipynb', 'repo_name': 'Anna-S0phia/Cultural_Data_Sculpting', 'size': '6045'}