<jupyter_start><jupyter_text>DSCI 525: Web and Cloud Computing Milestone 2: Data Wrangling for Machine Learning Group 13Authors: <NAME>, <NAME>, <NAME>, <NAME>  Table of contents- [Setup EC2 instance](1)- [Setup JupyterHub](2)- [Setup the server](3)- [Data Wrangling](4)- [Setup S3 bucket and move data](5)- [Wrangle the data in preparation for machine learning](6)  Imports<jupyter_code>import re
import os
import glob
import zipfile
import requests
from urllib.request import urlretrieve
import json
import pandas as pd<jupyter_output>/opt/tljh/user/lib/python3.7/site-packages/requests/__init__.py:91: RequestsDependencyWarning: urllib3 (1.26.4) or chardet (3.0.4) doesn't match a supported version!
  RequestsDependencyWarning)<jupyter_text>1. Setup EC2 instance    2. Setup JupyterHub    3. Setup the server    4. Data Wrangling<jupyter_code># Necessary metadata
article_id = 14226968  # this is the unique identifier of the article on figshare
url = f"https://api.figshare.com/v2/articles/{article_id}"
headers = {"Content-Type": "application/json"}
output_directory = "/srv/data/figsharerainfall/"
response = requests.request("GET", url, headers=headers)
data = json.loads(response.text)  # this contains all the articles data, feel free to check it out
files = data["files"]             # this is just the data about the files, which is what we want
files
files_to_dl = ["combined_model_data_parti.parquet.zip"]  ## Please download the partitioned 
for file in files:
    if file["name"] in files_to_dl:
        os.makedirs(output_directory, exist_ok=True)
        urlretrieve(file["download_url"], output_directory + file["name"])
with zipfile.ZipFile(os.path.join(output_directory, "combined_model_data_parti.parquet.zip"), 'r') as f:
    f.extractall(output_directory)<jupyter_output><empty_output><jupyter_text>5. Setup S3 bucket and move data    6. Wrangle the data in preparation for machine learning<jupyter_code># Passing our credentials 
aws_credentials ={"key": "<hidden>","secret": "<hidden>"} 
df = pd.read_parquet("s3://mds-s3-student85/combined_model_data_parti.parquet", storage_options=aws_credentials)
df.head()
df = df.query("lat_min <= -33.86 & lat_max >= -33.86 & lon_min <= 151.21 & lon_max >= 151.21")
df = df.drop(columns = ["lat_min", "lat_max", "lon_min", "lon_max"])
df = df.set_index('time')
df.head()
df.shape
df_models = df.pivot(columns="model", values="rain (mm/day)").resample('1D').sum()
df_syd = pd.read_csv("s3://mds-s3-student85/observed_daily_rainfall_SYD.csv", storage_options=aws_credentials)
df_syd = df_syd.set_index('time')
df_syd = df_syd.rename(columns={"rain (mm/day)": "observed_rainfall"})
df_combined = df_models.merge(df_syd, on=df_models.index).set_index('key_0')
df_combined.index.name = "time"
df_combined.head()
df_combined.shape
df_combined.to_csv('s3://mds-s3-student85/output/ml_data_SYD.csv', storage_options=aws_credentials)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13321917808219177, 'fraction_numerical': 0.025684931506849314, 'mean_word_length': 5.808857808857809, 'pattern_counts': {'":': 4, '<': 20, '<?xml version=': 0, '>': 20, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '230776', 'license': 'MIT', 'n_tokens_mistral': 1060, 'n_tokens_neox': 1012, 'n_words': 239, 'path': 'notebook/milestone2.ipynb', 'repo_name': 'williamxu7/DSCI_525_Group13_Rainfall', 'size': '2936'}