<jupyter_start><jupyter_text>Water Security  Downloading all data<jupyter_code># Make sure you've run aws configure before hand
!aws s3 sync s3://s3groupegypt data
import datetime as dateformat
import pandas as pd
import numpy as np
from shapely.geometry import Point, box
import matplotlib.pyplot as plt
import geopandas
import geoplot
import h5py
import warnings
from tqdm import tqdm
import pickle
import re
from sklearn.linear_model import LinearRegression
from sklearn.model_selection import train_test_split
import torch

warnings.filterwarnings('ignore')<jupyter_output><empty_output><jupyter_text>Preprocessing This section allows you to run the full pipeline from the raw data files. The full pipeline takes around 11 minutes. Alternatively, just go to the checkpoint section below and load the final master data frame.<jupyter_code># Merge 0.5x0.5 degree cells into 1x1 degree cells.
# We overlay 1x1 degree cells on the world and intersect with co-ordinates
# and aggregate using average.
# This is pretty generic and infact works with any x degree cell.
def geodataframe_overlay_grid_cells(degrees=1):
    grid = []
    for lat in np.arange(-180, 180, degrees):
        for lon in np.arange(-360, 360, degrees):
            grid.append(box(lat, lon, lat+degrees, lon+degrees))
    return grid

def geodataframe_merge_adjacent_cells(df, degrees=1, aggfunc='mean'):
    columns = df.columns[:-1]
    grid = pd.DataFrame(data={'id': np.arange(0, 360*720)})
    crs = {'init': 'epsg:4326'}
    gf = geopandas.GeoDataFrame(grid, crs=crs, geometry=geodataframe_overlay_grid_cells(degrees=degrees))
    gf_intersect = geopandas.sjoin(gf, df, how="inner", op='intersects')
    b = gf_intersect.dissolve(by='id', aggfunc=aggfunc)
    return geopandas.GeoDataFrame(b[columns], geometry=b.geometry.centroid)

# Converts dataframe to geodataframe
# Dataframe must contain lon/lat fields!
def dataframe_to_geodataframe(df):
    crs = {'init': 'epsg:4326'}
    geo = [Point(row['lon'], row['lat']) for (_, row) in df[['lon', 'lat']].iterrows()]
    gdf = geopandas.GeoDataFrame(df, crs=crs, geometry=geo)
    return gdf

frames = []<jupyter_output><empty_output><jupyter_text>Loading SSP2 dataset Here we load the dataset and visualise the information with a head()/google map<jupyter_code># This dataset has population estimates every decade. We linearly impute 
# the intermediate years.
def ssp2_linear_impute(df, prefix, from_year, to_year):
    df[["%c2_%s" % (prefix, i) for i in np.arange(from_year, to_year) if i % 10]] = np.nan
    df_t = df[["%c2_%s" % (prefix, i) for i in np.arange(from_year, to_year)]].transpose()
    df_it = df_t.interpolate(method='linear', direction = 'forward').transpose()
    df_it[["lon", "lat"]] = df[["px", "py"]]
    return df_it

# Read, Impute, GeoDataFrame, Center on grid
def ssp2_load_population(from_year=1980, to_year=2017):
    population = pd.read_csv("data/SSP2/pop_ssp2.csv")
    population = ssp2_linear_impute(population, prefix="p", from_year=from_year, to_year=to_year)
    population = dataframe_to_geodataframe(population)
    population = geodataframe_merge_adjacent_cells(population)
    return population

def ssp2_load_gdp(from_year=1980, to_year=2017):
    gdp = pd.read_csv("data/SSP2/gdp_ssp2.csv")
    gdp = ssp2_linear_impute(gdp, prefix="g", from_year=from_year, to_year=to_year)
    gdp = dataframe_to_geodataframe(gdp)
    gdp = geodataframe_merge_adjacent_cells(gdp)
    return gdp
population = ssp2_load_population()
frames.append(population)
gdp = ssp2_load_gdp()
frames.append(gdp)<jupyter_output><empty_output><jupyter_text>Loading Water GAP dataset Here we load the dataset and visualise the information with a head()/google map<jupyter_code>#Condition- function starts from year 1980, 
# nc4_file_path -> str ,nc4_key -> str, datatype either monthly or yearly
def watergap_nc4_to_dataframe(nc4_key, nc4_file, yearN=1980):
    nc4f = h5py.File(nc4_file, 'r')
    key = nc4f[nc4_key][:]
    lat = nc4f['lat'][:]
    lon = nc4f['lon'][:]
    year = yearN
    is_monthly = 'monthly' in nc4_file

    # monthly data
    if is_monthly:
        start = (yearN - 1900 - 1)*12;
        
        #Jan of YearN to Dec of 2016
        for i in range(start, nc4f[nc4_key].shape[0]):
            data = key[i,:,:]
            if i == start:
                month = 1
                data_pd = pd.DataFrame(data = data)
                lat = nc4f['lat'][:]
                data_pd.index =  lat
                lon = nc4f['lon'][:]
                data_pd.columns = lon
                data_pd = pd.melt(data_pd, ignore_index = False, var_name='lon', value_name=str(dateformat.datetime(year, 1, month).strftime('%m/%d/%Y')))               
            else:
                data_I = pd.DataFrame(data = data)
                data_I.index =  lat
                
                if i % 12 == 0:
                    year = year+1
                    month = 1
                else:
                    month = month + 1
                    
                data_I = pd.melt(data_I,ignore_index = False, value_name= str(dateformat.datetime(year, month,1).strftime('%m/%d/%Y')))
                data_pd[str(dateformat.datetime(year,month,1).strftime('%m/%d/%Y'))] = data_I.loc[:,str(dateformat.datetime(year,month,1).strftime('%m/%d/%Y'))]
        
        # Convert to Yearly Data
        data_ion = data_pd['lon']
        data_no_ion = data_pd.drop(columns=['lon'])
        new = (data_no_ion.groupby(pd.PeriodIndex(data_no_ion.columns, freq='Y'), axis=1).mean().rename(columns=lambda c: str(c).lower()))
        data_pd=pd.concat([data_ion, new], axis=1) 
        
    # yearly data
    else:
        start = year - 1900 - 1;

        #YearN to 2016
        for i in range(start, nc4f[nc4_key].shape[0]):
            data = key[i,:,:]
            if i == start:
                data_pd = pd.DataFrame(data = data)
                lat = nc4f['lat'][:]
                data_pd.index =  lat
                lon = nc4f['lon'][:]
                data_pd.columns = lon
                data_pd = pd.melt(data_pd, ignore_index = False, var_name='lon', value_name=str(yearN))
            else:
                data_I = pd.DataFrame(data = data)
                data_I.index =  lat
                year = year+1
                data_I = pd.melt(data_I,ignore_index = False, value_name= str(year))
                data_pd[str(year)] = data_I.loc[:,str(year)]

    data_pd['lat'] = data_pd.index
    data_pd = data_pd.add_suffix('_%s' % nc4_key)
    data_pd.rename(columns={'lat_%s' % nc4_key : 'lat', 'lon_%s' % nc4_key : 'lon' }, inplace=True)
    return data_pd.reset_index(drop=True)

def watergap_load(key, period):
    filename = 'data/WaterGap/watergap_22d_WFDEI-GPCC_histsoc_%s_%s_1901_2016.nc4' % (key, period)
    d = watergap_nc4_to_dataframe(nc4_key=key, nc4_file=filename)
    return d


def watergap_load_all():
    watergap_vars = [("anag", "monthly"),
                 ("anas", "monthly"),
                 ("atotuse", "monthly"),
                 ("canopystor", "monthly"),
                 ("dis", "monthly"),
                 ("evap", "monthly"),
                 ("glolakestor", "monthly"),
                 ("glowetlandstor", "monthly"),
                 ("groundwstor", "monthly"),
                 ("loclakestor", "monthly"),
                 ("locwetlandstor", "monthly"),
                 ("ncrun", "monthly"),
                 ("pdomuse", "yearly"),
                 ("pdomww", "yearly"),
                 ("pgwuse", "monthly"),
                 ("pgwww", "monthly"),
                 ("pinduse", "yearly"),
                 ("pindww", "yearly"),
                 ("pirruse", "monthly"),
                 ("pirrww", "monthly"),
                 ("plivuse", "yearly"),
                 ("pnag", "monthly"),
                 ("pnas", "monthly"),
                 ("potevap", "monthly"),
                 ("precmon", "monthly"),
                 ("ptotuse", "monthly"),
                 ("ptotww", "monthly"),
                 ("qg", "monthly"),
                 ("ql", "monthly"),
                 ("qr", "monthly"),
                 ("qrdif", "monthly"),
                 ("qrswb", "monthly"),
                 ("qs", "monthly"),
                 ("reservoirstor", "monthly"),
                 ("riverstor", "monthly"),
                 ("soilmoist", "monthly"),
                 ("swe", "monthly"),
                 ("tws", "monthly"),
                 ("disnat", "monthly"),
                 ("ncrunnat", "monthly")]

    local_frames = []
    for (watergap_key, watergap_period) in tqdm(watergap_vars):
        local_frames.append(watergap_load(key=watergap_key, period=watergap_period))
        
    d = local_frames[0]
    for frame in local_frames[1:]:
        d = d.merge(frame)
    
    d = dataframe_to_geodataframe(d)
    d = geodataframe_merge_adjacent_cells(d)
    return d
watergap = watergap_load_all()
watergap.head()
frames.append(watergap)<jupyter_output><empty_output><jupyter_text>Loading GLDAS dataset Here we load the dataset and visualise the information with a head()/google map<jupyter_code>def gldas_hdf_to_dataframe(year, month):
    year = str(year)
    hdf_file_path = 'data/gldas/GLDAS_CLSM10_M.A%s%02d.021.hdf' % (year, month)
    A = h5py.File(hdf_file_path, 'r')
    list_1 = list(A.keys())
    list_df = pd.DataFrame()
    list_df2 = pd.DataFrame()
    list_df["keys"] = list_1
    lat = A['lat']   
    lon = A['lon'] 
    data_pd = pd.DataFrame()   
    list_df2["keys"] = list_df["keys"].loc[0:37]
    count = 0
    for keys in list_df2["keys"]:
          if keys == 'ACond_tavg':
              data_arr = A[keys][0,:,:]
              data_pd = pd.DataFrame(data = data_arr)
              data_pd.columns = lon
              data_pd.index = lat
              col_n = year + "_" + keys
              data_pd = pd.melt(data_pd, ignore_index = False, var_name='lon', value_name= col_n)
          else:
              data_arr = A[keys][0,:,:]
              data = pd.DataFrame(data = data_arr)
              data.index = lat
              col_n = year + "_" + keys
              data = pd.melt(data, ignore_index = False, value_name= col_n)
              data_pd[col_n] = data.loc[:,col_n]

    data_pd['lat'] = data_pd.index
    return data_pd.reset_index(drop=True)

def gldas_load_all():
    local_frames = []
    for year in tqdm(range(2000, 2017)):
        # We need to merge one year at a time otherwise mem swapouts hit the roof
        d = [gldas_hdf_to_dataframe(year, month) for month in range(1, 13)]
        d = pd.concat(d).groupby(['lon','lat'], group_keys=False, as_index=False).mean()
        local_frames.append(d)
        
    d = local_frames[0]
    for frame in local_frames[1:]:
        d = d.merge(frame)
    d = dataframe_to_geodataframe(d)
    return d
gldas_df = gldas_load_all()

# Use that as main dataframe since GLDAS captures only a subset of the latitudes
# from 60 to -60, which are the interesting latitudes anyways. 
frames.insert(0, gldas_df)
gldas_df<jupyter_output><empty_output><jupyter_text>Merging the dataframe Here we load the dataset and visualise the information with a head()/google map<jupyter_code>def geodataframe_spatial_merge(frames):
    main_frame = frames[0]
    for frame in frames[1:]:
        main_frame = main_frame.merge(frame)
    return main_frame
master_df = geodataframe_spatial_merge(frames)
master_df<jupyter_output><empty_output><jupyter_text>Checkpoint (save/load)<jupyter_code>def save_checkpoint(df, name):
    with open(name, 'wb') as handle:
        pickle.dump(df, handle, protocol=pickle.HIGHEST_PROTOCOL)

def load_checkpoint(name):
    with open(name, 'rb') as handle:
        return pickle.load(handle)

# Do not uncomment this unless you want to save a new version of the master dataframe
# save_checkpoint(master_df, name='data/master_df.pickle')

master_df = load_checkpoint('data/master_df.pickle')
master_df<jupyter_output><empty_output><jupyter_text>World Poverty Index<jupyter_code>from shapely import wkt

wpif = pd.read_csv("data/wpif.csv")
wpif = wpif.replace(-999.0, 0.0)
mozambique = wkt.loads(wpif.iloc[0]['the_geom'])
print(wpif.iloc[0]['NAME'])
mozambique
master_df[master_df.intersects(mozambique)].shape
# 33 1 degree cells match mozambique
def geodataframe_merge_wpi(df):
    crs = {'init': 'epsg:4326'}
    geo = [wkt.loads(mpolygons) for mpolygons in wpif['the_geom']]
    wpif_gf = geopandas.GeoDataFrame(wpif, crs=crs, geometry=geo)
    return geopandas.sjoin(df, wpif_gf, how="inner", op='intersects')

gdf_intersect = geodataframe_merge_wpi(master_df)
gdf_intersect<jupyter_output><empty_output><jupyter_text>Remove non-reporters<jupyter_code>gdf_intersect.shape
gdf_intersect = gdf_intersect[gdf_intersect['WPI'] != 0]
gdf_intersect.shape<jupyter_output><empty_output><jupyter_text>Standardizing<jupyter_code>from sklearn.preprocessing import StandardScaler, MinMaxScaler

xscaler = StandardScaler().fit(gdf_intersect.filter(regex=".*2016.*"))
yscaler = MinMaxScaler().fit(np.array(gdf_intersect['WPI']).reshape(-1, 1))<jupyter_output><empty_output><jupyter_text>Lasso<jupyter_code>from sklearn.linear_model import LassoCV, Lasso

lasso_x = xscaler.transform(gdf_intersect.filter(regex=".*2016.*"))
lasso_y = yscaler.transform(np.array(gdf_intersect['WPI']).reshape(-1, 1))

lasso = LassoCV(cv=20, random_state=42).fit(lasso_x, lasso_y)
lasso.score(lasso_x, lasso_y)
interesting_var_names = gdf_intersect.filter(regex=".*2016.*").columns[np.where(lasso.coef_ != 0)]
interesting_vars = np.where(lasso.coef_ != 0)[0]
print(len(interesting_var_names))
", ".join([str(m) for m in interesting_var_names])
predicted_y = lasso.predict(lasso_x)

lasso_results = [["Country", "Actual", "Predicted"]]
for country in list(set(gdf_intersect['ISO'])):
    actual = np.median(lasso_y[gdf_intersect['ISO'] == country])
    predicted = np.median(predicted_y[gdf_intersect['ISO'] == country].astype(float))
    lasso_results.append([country, actual, predicted])
lasso_results = np.array(lasso_results)
plt.figure(figsize=(10, 10), dpi=80)
plt.xticks(rotation=90)
plt.yticks(np.arange(-1000, 100, step=50))
plt.plot(lasso_results[1:20,0], lasso_results[1:20,2].astype(float))
plt.plot(lasso_results[1:20,0], lasso_results[1:20,1].astype(float), '.')
plt.show()<jupyter_output><empty_output><jupyter_text>LSTM<jupyter_code>def rnn_interesting_columns(frame, year=2016):
    x_scaled = xscaler.transform(frame.filter(regex=".*%s.*" % year))
    return pd.DataFrame(x_scaled)

rnn_gdf_intersect = gdf_intersect.copy()
rnn_input_size = len(rnn_interesting_columns(rnn_gdf_intersect).columns)
rnn_hidden_dim = 5
rnn_seq_len = 20
rnn_n_layers = 3
rnn_output_size = 1
rnn_batch_size = 60
class LSTMModel(torch.nn.Module):
    def __init__(self, input_size, output_size, hidden_dim, n_layers):
        super(LSTMModel, self).__init__()
        self.hidden_dim = hidden_dim
        self.n_layers = n_layers
        self.lstm = torch.nn.LSTM(input_size, hidden_dim, n_layers, batch_first=True)   
        self.fc = torch.nn.Linear(hidden_dim, output_size)
        self.sigmoid = torch.nn.Sigmoid()
    
    def forward(self, x):
        batch_size = x.size(0)
        hidden = self.init_hidden(batch_size)
        out, hidden = self.lstm(x, hidden)
        out = out.contiguous().view(-1, self.hidden_dim)
        out = self.fc(out)
        out = self.sigmoid(out)
        out = out.view(batch_size, -1)
        out = out[:,-1]
        return out, hidden
    
    def init_hidden(self, batch_size):
        hidden = torch.zeros(self.n_layers, batch_size, self.hidden_dim)
        state = torch.zeros(self.n_layers, batch_size, self.hidden_dim)
        return (hidden, state)

def sequify_countries(countries, df, interesting_columns=rnn_interesting_columns):
    seqs = np.zeros((len(countries), rnn_seq_len, rnn_input_size))
    target = np.zeros((len(countries), rnn_output_size))
    for i, country in enumerate(countries):
        country_frame = df[df['ISO'] == country]
        country_full_seq = np.zeros((rnn_seq_len, rnn_input_size))
        country_seq = interesting_columns(country_frame)
        country_full_seq[:len(country_seq),:] = country_seq[:rnn_seq_len]
        seqs[i] = country_full_seq
        target[i] = yscaler.transform(country_frame['WPI'].iloc[-1].reshape((-1, 1)))
    return (seqs, target)

def rnn_get_batch(train_seqs, train_targets, batch_size):
    sampler = np.random.choice(np.arange(train_seqs.shape[0]), size=batch_size, replace=False)
    input_seq = np.zeros((batch_size, rnn_seq_len, rnn_input_size), dtype=np.float32)
    input_seq[:,:,:] = train_seqs[sampler,:,:]
    target_seq = train_targets[sampler].squeeze()
    return torch.from_numpy(input_seq), torch.Tensor(target_seq)

def rnn_validation_loss():
    tseqs, ttargets = rnn_get_batch(rnn_test_seqs, rnn_test_targets, rnn_test_seqs.shape[0])
    tseqs.to(device)
    output, _ = model(tseqs)
    validation_loss = mse_loss(output, ttargets)
    return validation_loss.item()
countries = list(set(gdf_intersect['ISO']))
rnn_train_countries, rnn_test_countries = train_test_split(countries, test_size=0.2, random_state=42)
rnn_train_seqs, rnn_train_targets = sequify_countries(rnn_train_countries, rnn_gdf_intersect)
rnn_test_seqs, rnn_test_targets = sequify_countries(rnn_test_countries, rnn_gdf_intersect)
device = touch.device("GPU") if torch.cuda.is_available() else torch.device("cpu")
model = LSTMModel(input_size=rnn_input_size, 
                 output_size=rnn_output_size,
                 hidden_dim=rnn_hidden_dim, 
                 n_layers=rnn_n_layers)
model.to(device)

n_epochs = 2000
lr = 0.01
mse_loss = torch.nn.MSELoss()
optimizer = torch.optim.Adam(model.parameters(), lr=lr)
losses = []
vlosses = []
lowest_vloss = 1
for epoch in range(1, n_epochs + 1):
    optimizer.zero_grad()
    input_seq, target_seq = rnn_get_batch(rnn_train_seqs, rnn_train_targets, rnn_batch_size)
    input_seq.to(device)
    output, hidden = model(input_seq)
    loss = mse_loss(output, target_seq)
    loss.backward()
    optimizer.step()
    losses.append(loss.item())
    vloss = rnn_validation_loss()
    vlosses.append(vloss)
    if vloss < lowest_vloss:
        # Save model with the lowest validation loss
        torch.save(model, "data/lstm.nn")
        lowest_vloss = vloss
    if epoch%100 == 0:
        print('Epoch: {}/{}...'.format(epoch, n_epochs), end=' ')
        print('Loss: {:.4f}...'.format(loss.item()), end=' ')
        print('Validation loss: {:.4f}'.format(vloss))

print("Saved model with lowest validation loss (%s) to data/lstm.nn" % lowest_vloss)
plt.figure(figsize=(10, 10), dpi=80)
plt.plot(vlosses, 'r')
plt.plot(losses)<jupyter_output><empty_output><jupyter_text>Loading LSTM model<jupyter_code>lstm_model = torch.load("data/lstm.nn")
lstm_model.eval()
tseqs, ttargets = rnn_get_batch(rnn_test_seqs, rnn_test_targets, rnn_test_seqs.shape[0])
tseqs.to(device)
output, _ = lstm_model(tseqs)


plt.figure(figsize=(8, 8), dpi=80)
plt.xticks(rotation=90)
plt.plot(yscaler.inverse_transform(output.detach().numpy().reshape(-1, 1)), '.r')
plt.plot(yscaler.inverse_transform(ttargets.detach().numpy().reshape(-1, 1)), '.')
plt.show()<jupyter_output><empty_output><jupyter_text>Projecting into the future In this section we attempt to understand how best to model the different variables in order to predict future values.<jupyter_code>acond_tavg = master_df.filter(regex=(".*_ACond_tavg"))
plt.xticks(rotation=90)
for i in range(5):
    plt.plot(acond_tavg.iloc[i])

def glm_fit_per_cell(varname):
    v = master_df.filter(regex=(".*_%s$" % varname))
    years = [int(re.search("(.*)_%s" % varname, s).group(1)) for s in v.columns]
    min_year = min(years)
    max_year = max(years)
    x = np.arange(min_year, max_year + 1).reshape((-1, 1))
    rsqrd = []
    models = []
    for y in v.iloc:
        model = LinearRegression().fit(x, y)
        models.append(model)
        rsqrd.append(model.score(x, y))
    return (models, rsqrd)
    
acond_models, acond_rsqrd = glm_fit_per_cell("ACond_tavg")
plt.hist(acond_rsqrd)
low_scorers = np.where(np.array(acond_rsqrd) < 0.1)[0]
fig, ax = plt.subplots(10, 10, sharex='col', sharey='row', figsize=(15,15))
i = 0
for row in ax:
    for gridplt in row:
        real_idx = low_scorers[i]
        gridplt.set_xticklabels([])
        gridplt.title.set_text('index %d' % real_idx)
        gridplt.title.set_size(8)
        x = np.arange(2000, 2016).reshape((-1, 1))
        y = acond_tavg.iloc[real_idx]
        gridplt.plot(y)
        gridplt.plot([acond_models[real_idx].intercept_ + acond_models[real_idx].coef_[0] * xv for xv in x], 'r')
        i += 1<jupyter_output><empty_output><jupyter_text>It is not ideal, but the lines capture the general trend of the variable in each geographical cell.<jupyter_code>def project_all_models(df):
    all_vars = [re.search('2000_(.*)', x).group(1) for x in df.filter(regex="2000_.*").columns]
    all_var_models = []
    for v in tqdm(all_vars):
        v_models, _ = glm_fit_per_cell(v)
        all_var_models.append(v_models)
    return all_var_models

def project_cell(df, idx, year, all_models):
    all_vars = [re.search('2000_(.*)', x).group(1) for x in df.filter(regex="2000_.*").columns]
    return [all_models[i][idx].coef_[0] * year + all_models[i][idx].intercept_ for i, v in enumerate(all_vars)]

def project_ssp2_data(df, year):
    # Load GDP/population projections from SSP2
    ssp2_copy = df.copy()
    pop = ssp2_load_population(from_year=year-10, to_year=year+1)
    ssp2_copy = ssp2_copy.merge(pop)
    gdp = ssp2_load_gdp(from_year=year-10, to_year=year+1)
    ssp2_copy = ssp2_copy.merge(gdp)
    return ssp2_copy
    
def project_df(df, year, all_models):
    future_df_spec = {'lon': df['lon'], 'lat': df['lat']}
    future_array = np.array([project_cell(df, i, year, all_models) for i in range(len(df))])
    all_vars = [re.search('2000_(.*)', x).group(1) for x in df.filter(regex="2000_.*").columns]
    for i, var in enumerate(all_vars):
        future_df_spec.update({'%s_%s' % (year, var): future_array[:,i]})
    ssp2_copy = project_ssp2_data(df, year)
    pop_year = 'p2_%s' % year
    gdp_year = 'g2_%s' % year
    future_df_spec.update({gdp_year: ssp2_copy[gdp_year]})
    future_df_spec.update({pop_year: ssp2_copy[pop_year]})
    future_df = dataframe_to_geodataframe(pd.DataFrame(future_df_spec))
    new_df = df.copy()
    return new_df.merge(future_df)
all_var_models = project_all_models(master_df)
future_df = project_df(master_df, 2050, all_var_models)
future_df
future_df.filter(regex=".*2050.*")
future_df = future_df[future_df['WPI'] != 0]
future_df
# Do not uncomment this unless you want to save a new version of the FUTURE dataframe
# save_checkpoint(future_df, name='data/future_df.pickle')

future_df = load_checkpoint('data/future_df.pickle')
# long_future_df = master_df.copy()
# for i in tqdm(range(2040, 2051)):
#     long_future_df = project_df(long_future_df, i, all_var_models)
# long_future_df = geodataframe_merge_wpi(long_future_df)
# save_checkpoint(long_future_df, name='data/long_future_df.pickle')

long_future_df = load_checkpoint('data/long_future_df.pickle')<jupyter_output>100%|██████████| 11/11 [25:29<00:00, 139.05s/it]<jupyter_text>Predicting the future with Lasso<jupyter_code>def rename_year_hack(column, from_year, to_year):
    return column.replace(str(from_year), str(to_year))

future_df_2050 = future_df.filter(regex=".*2050.*").rename(columns=lambda x: rename_year_hack(x, 2050, 2016))
future_lasso_predictions = lasso.predict(xscaler.transform(future_df_2050))
lasso_dict = {}
for country in set(future_df['NAME']):
    lasso_dict[country] = {}
    lasso_dict[country]['past'] = np.median(future_df[future_df['NAME'] == country]['WPI'])
    f_val = yscaler.inverse_transform(np.median(future_lasso_predictions[future_df['NAME'] == country].astype(float)).reshape(-1,1))
    lasso_dict[country]['future'] = f_val[0][0]
    lasso_dict[country]['diff'] = lasso_dict[country]['future'] - lasso_dict[country]['past']
def top_countries_abs(d, n=5, inv=1):
    return sorted([(k, d[k]['future']) for k in d.keys()], key=lambda x: x[1]*inv)[:n]

top_losers_abs  = top_countries_abs(lasso_dict, n=10)
top_winners_abs = top_countries_abs(lasso_dict, inv=-1, n=10)
top_winners_abs
", ".join([x[0] for x in top_winners_abs])
top_losers_abs
", ".join([x[0] for x in top_losers_abs])<jupyter_output><empty_output><jupyter_text>Predicting the future with LSTM<jupyter_code>countries = list(set(future_df['ISO']))
future_seqs, future_targets = sequify_countries(countries,
                                                future_df,
                                                interesting_columns=lambda x: rnn_interesting_columns(x, year=2050))
future_seqs = torch.from_numpy(future_seqs.astype(np.float32))
future_seqs.to(device)
output, _ = lstm_model(future_seqs)
lstm_dict = {}
for i, country_iso in enumerate(countries):
    country = future_df[future_df['ISO'] == country_iso]['NAME'].iloc[0]
    lstm_dict[country] = {}
    lstm_dict[country]['past'] = np.median(future_df[future_df['NAME'] == country]['WPI'])
    f_val = yscaler.inverse_transform(output.detach().numpy().reshape(-1, 1))[i]
    lstm_dict[country]['future'] = f_val
    lstm_dict[country]['diff'] = lstm_dict[country]['future'] - lstm_dict[country]['past']
# LSTM winners
top_countries_abs(lstm_dict, inv=-1, n=10)
", ".join([x[0] for x in top_countries_abs(lstm_dict, inv=-1, n=10)])
# LSTM losers
top_countries_abs(lstm_dict, n=10)
", ".join([x[0] for x in top_countries_abs(lstm_dict, n=10)])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13842360676268003, 'fraction_numerical': 0.019724483406386977, 'mean_word_length': 3.662956204379562, 'pattern_counts': {'":': 0, '<': 71, '<?xml version=': 0, '>': 70, 'https://': 0, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '17', 'dir': 'jupyter-structured-clean-dedup', 'id': '272994', 'license': 'MIT', 'n_tokens_mistral': 9721, 'n_tokens_neox': 9288, 'n_words': 1719, 'path': '01_Main.ipynb', 'repo_name': 'venkateshIedge7/Water_Security-', 'size': '25552'}