<jupyter_start><jupyter_text>Connect to tumblr API<jupyter_code>client = pytumblr.TumblrRestClient() #get key from https://api.tumblr.com/console

assert client.info() # Confirm connection<jupyter_output><empty_output><jupyter_text>Get "selfie" tagged posts<jupyter_code>def get_posts(n_pages, tag):
    before_start = None #increment by 20 for each page
    all_posts = []
    selfie_posts = []
    for page in range(n_pages):
        posts = client.tagged(tag, limit=20, before=before_start)
        all_posts+=posts
        before_start = posts[-1]['timestamp']
        for post in posts:
            if 'selfie' in post['tags']:
                selfie_posts.append(post) #this will be equivalent to all_posts for "selfie" tagged images, but only a subset for "enby" tagged images
    return all_posts, selfie_posts
if mode=='enby':
    all_posts, selfie_posts = get_posts(600, 'enby')
if mode=='all':
    all_posts, selfie_posts = get_posts(30, 'selfie')<jupyter_output><empty_output><jupyter_text>Extract images from selfie posts<jupyter_code>results = {}
for post in selfie_posts:
    try:
        results[post['id']] = (post, 
                               ((post['photos'][0]['alt_sizes'][2]['url']))) #extract photo using Tumblr's API
    except: 
        try:
            results[post['id']] = (post, 
                                   (bs4.BeautifulSoup(post['body']).find('img')['src'])) #parse and extract text page
        except IndexError:
            print("Can't locate photo", selfie_posts.index(post)) #can't find photo<jupyter_output><empty_output><jupyter_text>Save image information<jupyter_code>if mode =='all': base_dir = 'out_all'
if mode =='enby': base_dir = 'out_enby'
if 'out_all' not in os.listdir('./'): os.mkdir('out_all')
if 'out_enby' not in os.listdir('./'): os.mkdir('out_enby')
for result in results:
    if results[result][1].split('/')[-1] not in os.listdir(base_dir):
        write_tumblr_img(results[result][1],
                        '%s/%s' % (base_dir, results[result][1].split('/')[-1]))<jupyter_output><empty_output><jupyter_text>Save post information<jupyter_code>if mode == 'enby':
    pickle.dump(selfie_posts, open('selfie_posts_enby.pkl','wb'))
if mode == 'all':
    pickle.dump(selfie_posts, open('selfie_posts_all.pkl','wb'))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1475551709216789, 'fraction_numerical': 0.007788836001730853, 'mean_word_length': 3.867368421052632, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '66503', 'license': 'Apache-2.0', 'n_tokens_mistral': 813, 'n_tokens_neox': 782, 'n_words': 164, 'path': 'tumblr_query.ipynb', 'repo_name': 'ArjunSubramonian/twitter-algorithmic-bias-gnc', 'size': '2311'}