<jupyter_start><jupyter_text>Download the data<jupyter_code>article_id = 14096681  # this is the unique identifier of the article on figshare
url = f"https://api.figshare.com/v2/articles/{article_id}"
headers = {"Content-Type": "application/json"}
response = requests.request("GET", url, headers=headers)
data = json.loads(response.text)  # this contains all the articles data, feel free to check it out
files = data["files"]             # this is just the data about the files, which is what we want
%%time
for file in files:
    if file["name"] in files_to_dl:
        if (force_download or not os.path.exists(raw_folder + file["name"])):
            os.makedirs(raw_folder, exist_ok=True) # create the folder if not exists
            urlretrieve(file["download_url"], raw_folder + file["name"])
%%time
# extract the zip file
n_files = len(os.listdir(raw_folder))
if (force_download or n_files != 31): # if we must unzip the latest downloaded file or the file was not unzipped
    with zipfile.ZipFile(os.path.join(raw_folder, "data.zip"), 'r') as f:
        f.extractall(raw_folder)<jupyter_output>Wall time: 26.2 s<jupyter_text>Combine the data  Combine the data using Pandas<jupyter_code>%%time
%memit
import pandas as pd
use_cols = ["time", "lat_min", "lat_max", "lon_min", "lon_max", "rain (mm/day)"]
files = glob.glob(raw_folder + '*NSW.csv') # exclude observed_daily_rainfall_SYD

df = pd.concat((pd.read_csv(file, index_col=0, usecols=use_cols)
                .assign(model=file[max(file.rfind('/'), file.rfind('\\'))+1:file.index("_daily")])
                for file in files)
              )
df.to_csv(combined_file)
print("Size of the combined file:", os.path.getsize(combined_file)/(2**30), "GB")
%%time
df = pd.read_csv(combined_file)
print(df.shape)
df.head()<jupyter_output><empty_output><jupyter_text>Combine the data using Dask<jupyter_code># %%time
# %%memit
# dask_combined_file = processed_folder + "dask_combined_data.csv"
# ddf = dd.read_csv(raw_folder + '*NSW.csv', assume_missing=True, usecols=use_cols, include_path_column=True)
# ddf.to_csv(dask_combined_file, single_file=True)
# print("Size of the combined file:", os.path.getsize(dask_combined_file)/(2**30), "GB")
%%time
ddf = dd.read_csv(combined_file)
ddf.head()<jupyter_output>Wall time: 1.63 s
{'fraction_non_alphanumeric': 0.13941253836036827, 'fraction_numerical': 0.010960105217010083, 'mean_word_length': 4.896640826873385, 'pattern_counts': {'":': 1, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '394732', 'license': 'MIT', 'n_tokens_mistral': 832, 'n_tokens_neox': 801, 'n_words': 193, 'path': 'notebooks/Archived/data_download.ipynb', 'repo_name': 'UBC-MDS/525_group11', 'size': '2281'}