<jupyter_start><jupyter_text>Test notebook: Instagram CrawlerWe experiment with python to crawl an Instagram's profile, download a post and display the post's caption.  Import required modules<jupyter_code>import json
import requests
from bs4 import BeautifulSoup
from lxml.html.soupparser import fromstring<jupyter_output><empty_output><jupyter_text>Receive the Profile's Website<jupyter_code>insta_profile_url = 'https://www.instagram.com/koloot.design/'
# we download the profile's website
insta_profile = requests.get(insta_profile_url, allow_redirects=True)

# error check
if insta_profile.status_code != 200:
    print("Could not download the Instagram profile")
    print("Response code: %d" %insta_profile.status_code)
# Let's parse the website
soup = BeautifulSoup(insta_profile.text, 'lxml') 

json_data_str = ''
# iterate through all scripts and find the
# first occurence of the script containing the data
for script in soup.find_all('script'):
    script_str = script.string
    if script_str.startswith('window._sharedData = '):
        json_data_str = script_str
        break

# clean data
json_data_str = json_data_str.strip('window._sharedData = ')
json_data_str = json_data_str.strip(';')

# decode string and receive json object
json_data = json.loads(json_data_str.replace("\n","\\n"))
#
# Source: https://gist.github.com/douglasmiranda/5127251#gistcomment-2398949
#
def find_json(key, dictionary):
    """Returns the value for a key in in a (nested) iterable.
       
       Arguments:
           - key: a dictionary's entry
           - dictionary: <list> or <dictionary>
           - returned: <string> "value"
           
       Returns:
           - <iterator>
    """    
    for k, v in dictionary.items():
        if k == key:
            yield v
        elif isinstance(v, dict):
            for result in find_json(key, v):
                yield result
        elif isinstance(v, list):
            for d in v:
                if isinstance(d, dict):
                    for result in find_json(key, d):
                        yield result
insta_posts = list(find_json('shortcode', json_data))<jupyter_output><empty_output><jupyter_text>Download a Single Instagram Post<jupyter_code>insta_post = insta_posts[0]
insta_post_url = 'https://www.instagram.com/p/' + insta_post + '/'
print("Will download the post: %s" %insta_post_url)
insta_post = requests.get(insta_post_url, allow_redirects=True)
soup = BeautifulSoup(insta_post.text, 'lxml') 
# find json data
soup.find(attrs={"type": "application/ld+json"}).string<jupyter_output><empty_output><jupyter_text>Instagram Post DataThe website with the Instagram post contains json data describing the post. The `application/ld+json` typed data can be simply parsed using python's json library<jupyter_code># parse the data from the website 
post_data_str = soup.find(attrs={"type": "application/ld+json"}).string
# remove any leading and trailing whitespaces such as \n, \r, \t, \f, space.
post_data_str = post_data_str.strip()
# prep string for json parsing
post_json = json.loads(post_data_str.replace("\n","\\n"))
# access json data content
print(post_json["caption"])
post_json['uploadDate']
#
#    <meta name="medium" content="image" />
#    <meta property="og:type" content="instapp:photo" />
#
post_content_type = soup.find('meta', attrs={"name": "medium"})['content']
post_content_type
post_image_url = soup.find('meta', attrs={"property": "og:image"})['content']
post_image_url<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11537360435156026, 'fraction_numerical': 0.005439450329229888, 'mean_word_length': 4.1156661786237185, 'pattern_counts': {'":': 4, '<': 23, '<?xml version=': 0, '>': 23, 'https://': 3, 'lorem ipsum': 0, 'www.': 2, 'xml': 3}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '453365', 'license': 'MIT', 'n_tokens_mistral': 1119, 'n_tokens_neox': 1087, 'n_words': 282, 'path': 'notebooks/Test_InstaCrawler.ipynb', 'repo_name': 'cdeck3r/instablog', 'size': '3493'}