<jupyter_start><jupyter_text>Pull, Process, and Prepare Data---**Project**: Masters Project **Author**: <NAME><jupyter_code># import necessary packages

import requests 
import numpy as np
import pandas as pd
import xarray as xr
from datetime import datetime
from scipy import stats
import os as os
import intake

import matplotlib.pyplot as plt
%matplotlib inline
import cartopy.crs as ccrs
import regionmask
xr.set_options(keep_attrs=True)<jupyter_output><empty_output><jupyter_text>Step 1: Pull Initial Resampled AMS data from AWS Details on how raw hourly data was pulled and resampled into an annual maximum series (AMS) from Amazon Web Services (AWS) is located in data/README.md.  Step 2: Process and Prepare Initial Resampled Data for Analysis<jupyter_code>os.listdir('./data/initial_resampled/9km')
input_path ='./data/initial_resampled/9km'
# open files

cesm2_hist = xr.open_dataset(f'{input_path}/wrf_cesm2_historical_9km.nc')
cesm2_ssp370 = xr.open_dataset(f'{input_path}/wrf_cesm2_ssp370_9km.nc')

cnrm_hist = xr.open_dataset(f'{input_path}/wrf_cnrm_esm2_1_historical_9km.nc')
cnrm_ssp370 = xr.open_dataset(f'{input_path}/wrf_cnrm_esm2_1_ssp370_9km.nc')
# convert to dataarray

cesm2_hist = cesm2_hist['T2']
cesm2_ssp370 = cesm2_ssp370['T2']
cnrm_hist = cnrm_hist['T2']
cnrm_ssp370 = cnrm_ssp370['T2']
global_attributes = {'grid mapping':'lambert conformal',
                     'unit description':'temp at 2 m',
                     'units':'kelvin',
                     'extreme value extraction method':'block maxima',
                     'extremes type':'maxima',
                     'block size':'1 year',
                     'timeseries type':'annual max series'}
# add attributes

cesm2_hist.attrs = global_attributes
cesm2_hist.attrs['global climate model'] = 'cesm2'
cesm2_hist.attrs['model observations'] = 'historical'

cesm2_ssp370.attrs = global_attributes
cesm2_ssp370.attrs['global climate model'] = 'cesm2'
cesm2_ssp370.attrs['model observations'] = 'projections'
cesm2_ssp370.attrs['pathway'] = 'ssp370'

cnrm_hist.attrs = global_attributes
cnrm_hist.attrs['global climate model'] = 'cnrm esm2-1'
cnrm_hist.attrs['model observations'] = 'historical'

cnrm_ssp370.attrs = global_attributes
cnrm_ssp370.attrs['global climate model'] = 'cnrm esm2-1'
cnrm_ssp370.attrs['model observations'] = 'projections'
cnrm_ssp370.attrs['pathway'] = 'ssp370'
# convert temperature from K to C

cesm2_hist.data = cesm2_hist.data - 273.15
cesm2_hist.attrs['units'] = 'celsius'

cesm2_ssp370.data = cesm2_ssp370.data - 273.15
cesm2_ssp370.attrs['units'] = 'celsius'

cnrm_hist.data = cnrm_hist.data - 273.15
cnrm_hist.attrs['units'] = 'celsius'

cnrm_ssp370.data = cnrm_ssp370.data - 273.15
cnrm_ssp370.attrs['units'] = 'celsius'
# set regional masks to california

cesm2_hist_mask = regionmask.defined_regions.natural_earth_v5_0_0.us_states_50.mask(cesm2_hist.lon, cesm2_hist.lat, method='shapely', wrap_lon=False)
cesm2_ssp370_mask = regionmask.defined_regions.natural_earth_v5_0_0.us_states_50.mask(cesm2_ssp370.lon, cesm2_ssp370.lat, method='shapely',wrap_lon=False)
cnrm_hist_mask = regionmask.defined_regions.natural_earth_v5_0_0.us_states_50.mask(cnrm_hist.lon, cnrm_hist.lat, method='shapely', wrap_lon=False)
cnrm_ssp370_mask = regionmask.defined_regions.natural_earth_v5_0_0.us_states_50.mask(cnrm_ssp370.lon, cnrm_ssp370.lat, method='shapely', wrap_lon=False)
# use mask to spatially subset data to california

cesm2_hist = cesm2_hist.where(cesm2_hist_mask == 4).dropna("x", how="all").dropna("y", how="all")
cesm2_ssp370 = cesm2_ssp370.where(cesm2_ssp370_mask == 4).dropna("x", how="all").dropna("y", how="all")
cnrm_hist = cnrm_hist.where(cnrm_hist_mask == 4).dropna("x", how="all").dropna("y", how="all")
cnrm_ssp370 = cnrm_ssp370.where(cnrm_ssp370_mask == 4).dropna("x", how="all").dropna("y", how="all")
# temporally subset data into intervals for analysis

cesm2_hist_1980 = cesm2_hist.sel(time=slice('1980-01-01', '2000-01-01'))

cesm2_ssp370_2020 = cesm2_ssp370.sel(time=slice('2020-01-01', '2040-01-01'))
cesm2_ssp370_2040 = cesm2_ssp370.sel(time=slice('2040-01-01', '2060-01-01'))
cesm2_ssp370_2060 = cesm2_ssp370.sel(time=slice('2060-01-01', '2080-01-01'))
cesm2_ssp370_2080 = cesm2_ssp370.sel(time=slice('2080-01-01', '2100-01-01'))

cnrm_hist_1980 = cnrm_hist.sel(time=slice('1980-01-01', '2000-01-01'))

cnrm_ssp370_2020 = cnrm_ssp370.sel(time=slice('2020-01-01', '2040-01-01'))
cnrm_ssp370_2040 = cnrm_ssp370.sel(time=slice('2040-01-01', '2060-01-01'))
cnrm_ssp370_2060 = cnrm_ssp370.sel(time=slice('2060-01-01', '2080-01-01'))
cnrm_ssp370_2080 = cnrm_ssp370.sel(time=slice('2080-01-01', '2100-01-01'))<jupyter_output><empty_output><jupyter_text>Step 3: Export Intermediate Processed Data<jupyter_code>export_path ='./data/intermediate_processed'
# export to data/intermediate_processed folder

cesm2_hist_1980.to_netcdf(f'{export_path}/cesm2_hist_1980.nc')
cesm2_ssp370_2020.to_netcdf(f'{export_path}/cesm2_ssp370_2020.nc')
cesm2_ssp370_2040.to_netcdf(f'{export_path}/cesm2_ssp370_2040.nc')
cesm2_ssp370_2060.to_netcdf(f'{export_path}/cesm2_ssp370_2060.nc')
cesm2_ssp370_2080.to_netcdf(f'{export_path}/cesm2_ssp370_2080.nc')

cnrm_hist_1980.to_netcdf(f'{export_path}/cnrm_hist_1980.nc')
cnrm_ssp370_2020.to_netcdf(f'{export_path}/cnrm_ssp370_2020.nc')
cnrm_ssp370_2040.to_netcdf(f'{export_path}/cnrm_ssp370_2040.nc')
cnrm_ssp370_2060.to_netcdf(f'{export_path}/cnrm_ssp370_2060.nc')
cnrm_ssp370_2080.to_netcdf(f'{export_path}/cnrm_ssp370_2080.nc')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1355072463768116, 'fraction_numerical': 0.10978260869565218, 'mean_word_length': 6.646814404432133, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '99832', 'license': 'MIT', 'n_tokens_mistral': 2649, 'n_tokens_neox': 2283, 'n_words': 322, 'path': '0_pull_process_data.ipynb', 'repo_name': 'nabigac/masters_project', 'size': '5528'}