<jupyter_start><jupyter_text>Movies have form www.imdb.com/title/tt2406566/ atomic blondeTo get full credits cast, writers, etc:look for inside it will have alternating a tags linking back to the actor names and pages<a href="/name/nm0000234/?ref_=ttfc_fc_cl_t1"itemprop='url'> <NAME>  <a href="/name/nm0564215/?ref_=ttfc_fc_cl_t2"itemprop='url'> <NAME>To download full source of website:view-source:http://www.imdb.com/title/tt2406566/fullcredits<jupyter_code>from requests import Request, Session
from time import sleep
title_nums = ['tt0085244', 'tt2406566'] #The Big Chill, Atomic Blonde
'''
import requests


def print_url(r, *args, **kwargs):
    print(r.url)


hooks = dict(response=print_url)
r = requests.get('http://httpbin.org', hooks=dict(response=print_url))
print(r.status_code)'''
def write_result(response, *args, **kwargs):

    print('writing file from..{} '.format({response.url}))  
    filename =  "test" + str(next(numbers)) + ".html"
    with open(filename, 'wb') as f:
        f.write(response.content)
        f.write(response.url.encode('utf-8'))
        print('saved file %s' % filename)

def make_urls():
    base_url = "http://www.imdb.com/title/"
    urls = []
    
    for title in title_nums:
        urls.append(base_url + title + '/')
    return urls
def my_count():
    n = 10110
    while True:
        yield n
        n += 1
    
numbers = my_count()

def start(my_session = None):
    
    urls = make_urls()
    print('Urls', urls)
    
    for url in urls:
        try:
            r = my_session.get(url, hooks=dict(response=write_result), timeout=10)

            print("accessing url", url)
            print("request headers",r.request.headers)
            print("response headers",r.headers)
        except Exception as e:
            print("accessing url", url)    
            print('Exception encountered at position 1:', e)
            
    return r, urls
with Session() as s:
    try:
        start_tuple3 = start(s)
    except Exception as err:
        print('Exception encountered at position 2:', err)


make_urls()
!ls

!cat test1014.html<jupyter_output><!DOCTYPE html>
<html
xmlns:og="http://ogp.me/ns#"
xmlns:fb="http://www.facebook.com/2008/fbml">
    <head>
        <meta charset="utf-8">
        <meta http-equiv="X-UA-Compatible" content="IE=edge">

    <meta name="apple-itunes-app" content="app-id=342792525, app-argument=imdb:///title/tt2406566?src=mdot">
            <script type="text/javascript">var ue_t0=window.ue_t0||+new Date();</script>
            <script type="text/javascript">
                var ue_mid = "A1EVAM02EL8SFB"; 
                var ue_sn = "www.imdb.com";  
                var ue_furl = "fls-na.amazon.com";
                var ue_sid = "000-0000000-0000000";
                var ue_id = "0DV5ZTD18CZ7PQAKP3PT";
                (function(e){var c=e;var a=c.ue||{};a.main_scope="mainscopecsm";a.q=[];a.t0=c.ue_t0||+new Date();a.d=g;function g(h){return +new Date()-(h?0:a.t0)}function d(h){return function(){a.q.push({n:h,a:arguments,t:a.d()})}}function b(m,l,h,j,i){var k={m:m,f:l,l:h,c:""+j,err:i,fromOnError:[...]
{'fraction_non_alphanumeric': 0.15882731958762886, 'fraction_numerical': 0.03672680412371134, 'mean_word_length': 3.090909090909091, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 5, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '488873', 'license': 'MIT', 'n_tokens_mistral': 1192, 'n_tokens_neox': 1101, 'n_words': 221, 'path': 'notebooks/IMDB-Downloader.ipynb', 'repo_name': 'seanreed1111/box-office-data', 'size': '3119'}