<jupyter_start><jupyter_text>ISE-533 Project 4: Allocation Problem for COVID-19 Ventilators Forecast with Uncertainty - Simple Recourse ModelBy: <NAME>, <NAME>, <NAME>, <NAME><jupyter_code>import os
import numpy as np
import pandas as pd
import math
from pyomo.environ import *
from collections import defaultdict
import pyomo.environ as pyo
from pyomo.opt import SolverFactory
from datetime import timedelta
import matplotlib.pyplot as plt
raw = pd.read_csv('project 4 dataset/init_data.csv', header=0, index_col=0)
raw
first_date = '2020/03/25'
t = 7     # for a week
T = 14
P = 10
lbd = 0.1 # penalty

S = 51    # 50 states and SNS
states_name = raw['State']
SNS_stock = 12000

init_ratio = 0.8        # 0.6 of the stock is used for covid-19
flow_bound_ratio = 0.3  #0.2, 0.25, 0.3
stock_bound_ratio = 0.5 #0.6, 0.5
#first iteration values
data = raw.rename(columns={'Available capacity': "stock_self"})
data = data[:-1]
data['stock_self'] = init_ratio * data['stock_self']
data['stock_nb'] = None
for i in range(S):
    data.loc[i, 'stock_nb'] = str([0 for _ in eval(data.loc[i, 'nb'])])
data['stock_sns'] = [0 for _ in range(S)]
idx = 0
flow_mapping = {}
flow_mapping_rev = {}
stocks = {}
out_flow = defaultdict(list)
in_flow = defaultdict(list)

for i in range(S):
    nbs = eval(data.iloc[i, 3])
    out_flow[i] = nbs
    stocks[i ,i] = data['stock_self'][i]
    for n in nbs:
        stocks[i, n] = 0
        flow_mapping[(i, n)] = idx
        flow_mapping_rev[idx] = (i, n)
        in_flow[n].append(i)
        idx += 1

in_flow[51] = []
out_flow[51] = list(range(51))
for n in range(S):
    stocks[51, n] = 0
    flow_mapping[(51, n)] = idx
    flow_mapping_rev[idx] = (51, n)
    idx += 1
stocks[51, 51] = SNS_stock 
def first_stage(stocks:dict):
    model = ConcreteModel()

    
    #Sets
    F = idx
    model.time_set = list(range(t))
    model.state_set = list(range(S))
    model.state_p_set = list(range(S + 1))
    model.flow_set = list(range(F))
    
    # Parameters
    model.demand = Param(model.state_set, model.time_set, within=NonNegativeReals, mutable=True)

    model.U = [round(flow_bound_ratio * stocks[i, i]) for i in model.state_p_set]
    model.G = [round(stock_bound_ratio * stocks[i, i]) for i in model.state_p_set]
    
    
    # Variables
    model.s = Var(model.state_p_set, within=NonNegativeReals) # s_jj
    model.s_nb = Var(model.flow_set, within=NonNegativeReals) # s_ij
    model.x = Var(model.flow_set, within=NonNegativeReals)    # x_ij
    
    
    # Objective
    model.first = Var(within = NonNegativeReals)
    model.second = Var(within = NonNegativeReals)
    model.first_penalty = Constraint(expr = model.first == lbd * summation(model.s_nb))
    
    model.cost = Objective(expr = model.first + model.second, sense = minimize)
    
    
    # Constraints
    model.stock_self = Constraint(model.state_set, rule=lambda model, j: \
                                  model.s[j] + sum(model.x[flow_mapping[j, i]] for i in out_flow[j]) - model.x[flow_mapping[51, j]]== \
                                  stocks[j, j])
    model.stock_SNS_self = Constraint(expr = \
                                      model.s[51] + sum(model.x[flow_mapping[(51, j)]] for j in out_flow[51]) == \
                                      stocks[51, 51])

    model.stock_nb = ConstraintList()
    for j in model.state_set:
        for i in in_flow[j]:
            model.stock_nb.add(model.s_nb[flow_mapping[i,j]] - model.x[flow_mapping[i,j]] == stocks[i,j])
            
    model.s_bound = Constraint(model.state_p_set, rule=lambda model, j: model.s[j] >= model.G[j])
    
    model.x_bound = ConstraintList()
    for i in model.state_p_set:
        for j in out_flow[i]:
            model.x_bound.add(model.x[flow_mapping[i,j]] <= model.U[i])
            model.x_bound.add(-stocks[i,j] <= model.x[flow_mapping[i,j]])
            
    model.benders_cuts = ConstraintList()
    
    
    return model
def second_stage(demands:list, u:float):
    model = ConcreteModel()
    
    # Parameters
    model.time_set = list(range(t))
    
    # Variables
    model.delta = Var(model.time_set, within=NonNegativeReals)
    model.dual = Suffix(direction=Suffix.IMPORT)
    
    # Objective
    model.obj = Objective(expr = summation(model.delta), sense = minimize)
    
    # Constraint
    model.delta_bound = ConstraintList()
    for day in model.time_set:
        model.delta_bound.add(expr = model.delta[day] >= demands[day] - u)
        model.delta_bound.add(expr = model.delta[day] >= 0)
        
    return model
def benders(date, stocks, max_iteration = 200):
    opt = SolverFactory('glpk')
    master = first_stage(stocks)
    
    iter = 0
    
    lower = -float("inf")
    upper = float("inf")
    
    while iter < max_iteration:
        iter += 1
        # print("iteration", iter)
        
        opt.solve(master)
        lower = max(lower, master.cost.expr())
        # print("Lower Bound:", lower)
        
        if upper - lower <= 0.001:
            print("Converged")
            break
            
        x = master.x.get_values()   # x_ij
        s = master.s.get_values()   # s_jj
        s_nb = master.s_nb.get_values()   # s_ij
        
        sub_bound = 0
        sub_duals = defaultdict(int)
        benders_cut_constant = 0
        
        for j in range(S):
            state = states_name[j]
            state_demand = pd.read_csv("project 4 dataset/prediction/{}/{}.csv".format(date.strftime('%m%d'), state))
            
            path = [state_demand['InvVen_mean'], state_demand['InvVen_upper'], state_demand['InvVen_lower']]
            
            u = s[j] + sum(s_nb[flow_mapping[i, j]] for i in in_flow[j])

            for d in path:
                demands = d.iloc[:7]  
                
                sub = second_stage(demands, u)
                opt.solve(sub)
                
                duals = []
                
                for day in range(1, 1 + t):
                    duals.append(sub.dual[sub.delta_bound[day]])
                    
            sub_duals[j] += sum(duals)/3
            benders_cut_constant += sum([demand * dual for demand, dual in zip(demands, duals)])/3
            sub_bound += sub.obj.expr()/3

                
        sub_bound += lbd * sum(s_nb[j] for j in range(idx-S))
        upper = min(upper, sub_bound)
        # print("Upper Bound:", upper)
        
        # Add benders cuts
        master.benders_cuts.add(expr = master.cost >= lbd * sum(master.s_nb[flow] for flow in range(idx-S)) - \
                               sum(sub_duals[j] * (sum(master.s_nb[flow_mapping[i,j]] for i in in_flow[j]) + \
                                                  master.s[j]) for j in range(S)) + \
                                benders_cut_constant)
        
    return master<jupyter_output><empty_output><jupyter_text>Allocation For 10 Weeks<jupyter_code>date = pd.to_datetime("2020/03/25")
all_models = []
data_list = []
final = pd.DataFrame()

for week in range(10):
    solved_master = benders(date, stocks, max_iteration = 20)
    all_models.append(solved_master)
    s_nb = solved_master.s_nb.get_values()
    s = solved_master.s.get_values()
    
    date += timedelta(7)
    
    
    stocks = {}
    for flow, index in flow_mapping_rev.items():
        stocks[index] = s_nb[flow]
    for i in range(S+1):
        stocks[i,i] = s[i]
        
    
    temp = raw.copy()
    nb = 0
    
    for i in range(S+1):
        for index, flow in flow_mapping.items():
            if index[0] == i:
               nb += stocks[index]
        
        temp.loc[i, 'Stock'] = stocks[i,i] + nb
    
    temp = temp['Stock']
    data_list.append(temp)
    
final = pd.concat(data_list, axis = 1).set_index(states_name)
final<jupyter_output><empty_output><jupyter_text>Unmet Demand with Actual Demand<jupyter_code>states_name = raw['State']
state_demand = pd.DataFrame(columns=['date']) 

for j in range(S):
    state = states_name[j]
    temp = pd.read_csv("project 4 dataset/eval/{}.csv".format(state), header=None)
    temp.rename(columns = {0: 'date', 1: state}, inplace = True)
    state_demand = state_demand.merge(temp, on='date', how='outer') 
state_demand = state_demand.set_index('date')
eval = state_demand.groupby(np.arange(len(state_demand))//7).mean()
eval
final1 = final.copy()
final1 = final1.transpose()
final1 = final1.iloc[: , :-1]
final1.index = eval.index
final1<jupyter_output><empty_output><jupyter_text>Unmet Demand Every Week and Sum<jupyter_code>final1.to_csv('project 4 dataset/SR_supply_08_03_05.csv')
final = pd.read_csv('project 4 dataset/SR_supply_08_03_05.csv')
final = final.drop(['Unnamed: 0'], axis = 1)
final
states_name = raw['State']
state_demand = pd.DataFrame(columns=['date']) 

for j in range(S):
    state = states_name[j]
    temp = pd.read_csv("project 4 dataset/eval/{}.csv".format(state), header=None)
    temp.rename(columns = {0: 'date', 1: state}, inplace = True)
    state_demand = state_demand.merge(temp, on='date', how='outer') 

state_demand = state_demand.set_index('date')
state_demand = state_demand.reset_index()
state_demand.head()
final_repeat = final.loc[final.index.repeat(7)].reset_index(drop=True)
final_repeat
unmet_everyday = state_demand.drop(['date'], axis = 1).subtract(final_repeat)
unmet_everyday[unmet_everyday <= 0] = 0
unmet_everyweek = unmet_everyday.groupby(np.arange(len(unmet_everyday))//7).sum()

unmet_everyweek
result_08_03_05 = pd.DataFrame(unmet_everyweek.sum(axis = 1), columns = ['Unmet Demand'])
result_08_03_05.index = [1,2,3,4,5,6,7,8,9,10]
result_08_03_05 = result_08_03_05.rename_axis('Week').reset_index()
result_08_03_05.to_csv('project 4 dataset/result_08_03_05.csv')
result_08_03_05
pd.DataFrame(result_08_03_05.sum(axis = 1), columns = ['Unmet Demand']).sum(axis = 0)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13427814029363785, 'fraction_numerical': 0.023552202283849917, 'mean_word_length': 2.6751592356687897, 'pattern_counts': {'":': 0, '<': 26, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '639274', 'license': 'MIT', 'n_tokens_mistral': 3633, 'n_tokens_neox': 3442, 'n_words': 696, 'path': 'project4-simple_recourse_model.ipynb', 'repo_name': 'Yiyi-cmd/ISE533-Final-Project', 'size': '9824'}