<jupyter_start><jupyter_text>Download data<jupyter_code>import math
import os
import requests
from tqdm import tqdm
def download_url(url, output_directory='.'):
    filename = url.split("/")[-1]
    output_directory = Path(output_directory)
    filepath = output_directory / filename
    if not filepath.exists():
        if not output_directory.is_dir():
            output_directory.mkdir()
        r = requests.get(url, stream=True)
        tsize =  int(r.headers.get('content-length', 0))
        bsize = 1024
        num_iterables = math.ceil(tsize / bsize)
        
        with open(filepath, 'wb') as file:
            for chunk in tqdm(r.iter_content(bsize), total=num_iterables, unit="KB", unit_scale=True):
                file.write(chunk)

    return filepath
Path().resolve()
download_url("https://github.com/chrischute/squad/data/train-v2.0.json", "../data/raw")
import os
import zipfile

def extract_zip(file_path, dest_path='.'):
    """Extracts all contents of a zip archive file.
    Args:
        file_path (str): Path or file to extract
        dest_path (str, optional): Destination directory. Defaults to ".".
    """
    if not os.path.exists(file_path):
        raise IOError("File doesn't exist")
    if not os.path.exists(dest_path):
        raise IOError("Destination directory doesn't exist")
    with zipfile.ZipFile(file_path) as z:
        z.extractall(dest_path, filter(lambda f: not f.endswith("\r"), z.namelist()))
def get_embedding()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12541694462975317, 'fraction_numerical': 0.00533689126084056, 'mean_word_length': 3.23728813559322, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '355254', 'license': 'MIT', 'n_tokens_mistral': 486, 'n_tokens_neox': 473, 'n_words': 114, 'path': 'BiDAF/notebooks/00-download-data.ipynb', 'repo_name': 'giwankim/squad', 'size': '1499'}