<jupyter_start><jupyter_text>Fetching the HTML pagesTo get the data from the server, we need to issue a request for each WSD URL. The number is not enormous but at the same time non-negligible. since, in general, we want to throttle (limit the rate) our requests since we don't know if this will trigger some security response from the server, e.g. refusing connections. In addition, it's useful (especially when scraping) to be able to cache the results, so that we fetch the data once and can process it without limitations afterwards.<jupyter_code>import requests
import time

def _make_throttle_hook(timeout=1):
    """
    Returns a response hook function which sleeps for `timeout` seconds.
    """
    def hook(response, *args, **kwargs):
        time.sleep(timeout)
        return response

    return hook

def get_session(session=None, rate_max=.5, timeout=None, path_cache_db=''):
    """
    `rate_max` is in requests per second; `timeout` is in seconds.
    """
    session = session or requests.Session()
    timeout = timeout or 1 / rate_max

    session.hooks = {'response': _make_throttle_hook(timeout)}
    return session
SESSION = get_session(timeout=.1)
SESSION
from tqdm._tqdm_notebook import tqdm_notebook

from config import PATH_HTML_PAGES
PATH_HTML_PAGES
def process_row(row, session, path_base):
    status = {}

    pws_id = row.pws_id
    url = row.url
    
    status.update({'PWSID': pws_id})
    
    path = (path_base / str(pws_id)).with_suffix('.html')
    
    if path.exists():
        if path.stat().st_size > 0:
            status.update(source=path)
    else:
        resp = session.get(url)
        if resp:
            path.write_text(resp.text)
        else:
            print(f'{url}: {resp}')
        status.update(source=url)
        
    return status

def fetch_urls(_df_urls, session, path_base=None):
    df = _df_urls.reset_index()

    with tqdm_notebook(total=len(df)) as progress_bar:
        for row in df.itertuples():
            
            status = process_row(row, session, path_base)
            
            progress_bar.set_postfix(status)
            progress_bar.update(1)
fetch_urls(URLS, SESSION, path_base=PATH_HTML_PAGES)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09238395673726904, 'fraction_numerical': 0.0027039206849932404, 'mean_word_length': 3.088397790055249, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '639637', 'license': 'MIT', 'n_tokens_mistral': 696, 'n_tokens_neox': 682, 'n_words': 208, 'path': '2-fetching-html.ipynb', 'repo_name': 'fndari/sdwis-scraping', 'size': '2219'}