<jupyter_start><jupyter_text>Sustainable energy transitions data model new set<jupyter_code>#initialize
%run cycler-init.py
#load ! careful, need to rebuild index if tradealpha or data changes
# NIMPEX OR IMPEX
ni='impex'
# CAREFUL HERE

predata=json.loads(file(pop_path+'trade/'+ni+'.json','r').read())
#predata=json.loads(file(pop_path+'trade/nimpex.json','r').read())
nexportmatrix=predata["nexport"]
nimportmatrix=predata["nimport"]
nrexportmatrix=predata["nrexport"]
nrimportmatrix=predata["nrimport"]
predata={}<jupyter_output><empty_output><jupyter_text>Normal<jupyter_code>sd='b1'
inf='n'
basepower=3
dendro(sd,1,basepower)
[importancematrix,influencematrix]=dendro(sd,1,basepower)
z=[np.mean(i) for i in influencematrix] #sum country influence on columns
#if you wanted weighted influence, introduce weights (by trade volume i guess) here in the above mean
s = pd.Series(1/np.array(z)) #need to 1/ to create inverse order
s=s.rank(method='dense').values.astype(int)-1 #start from 0 not one
#s is a ranked array on which country ranks where in country influence 
#we then composed the ordered vector of country influence
influencevector=[0 for i in range(len(s))]
for i in range(len(s)):
    influencevector[s[i]]=i
#create mini-world
goodcountries2=[goodcountries[i] for i in influencevector[:16]]
c=['seaGreen','royalBlue','#dd1c77']
levels=[basepower]
toplot=[cid[i] for i in goodcountries2]
tolabel=[labeler[i] for i in goodcountries2]
fig,ax=plt.subplots(1,2,figsize=(12,5))
for j in range(len(levels)):
    [importancematrix,influencematrix]=dendro(sd,1,levels[j])
    z=[np.mean(i) for i in influencematrix] #sum country influence on columns
    #if you wanted weighted influence, introduce weights (by trade volume i guess) here in the above mean
    s = pd.Series(1/np.array(z)) #need to 1/ to create inverse order
    s=s.rank(method='dense').values.astype(int)-1 #start from 0 not one
    #s is a ranked array on which country ranks where in country influence 
    #we then composed the ordered vector of country influence
    influencevector=[0 for i in range(len(s))]
    for i in range(len(s)):
        influencevector[s[i]]=i
    zplot=[]
    zplot2=[]
    for i in toplot:
        zplot.append(s[i]+1)
        zplot2.append(z[i])
    ax[0].scatter(np.array(zplot),np.arange(len(zplot))-0.2+0.2*j,40,color=c[j],label=u'$p='+str(levels[j])+'$')
    ax[1].scatter(np.array(zplot2),np.arange(len(zplot))-0.2+0.2*j,40,color=c[j],label=u'$p='+str(levels[j])+'$')
    ax[0].set_ylim(-1,len(toplot))
    ax[1].set_ylim(-1,len(toplot))
    ax[0].set_xlim(0,20)
    ax[1].set_xscale('log')
    ax[0].set_yticks(range(len(toplot)))
    ax[0].set_yticklabels(tolabel)
    ax[1].set_yticks(range(len(toplot)))
    ax[1].set_yticklabels([])
    ax[0].set_xlabel("Rank in Country Influence Vector")
    ax[1].set_xlabel("Average Country Influence")
    if levels[j]==basepower:
        civector={}
        for k in range(len(influencevector)):
            civector[k+1]={"inf":np.round(z[influencevector[k]],2),"country":labeler[goodcountries[influencevector[k]]]}
        uk=[]
        for uj in range(len(influencematrix)):
            for ui in range(len(influencematrix[uj])):
                if ui!=uj:
                    uk.append({'type':'Country influence explorer',
                               'country':labeler[goodcountries[uj]],
                               'partner':labeler[goodcountries[ui]],
                               'inf':influencematrix[uj][ui],
                               'imp':importancematrix[uj][ui]})
        #pd.DataFrame(civector).T.to_excel('../json/'+sd+'/'+ni+'_'+inf+'_c.xlsx')
        #file('../json/'+sd+'/'+ni+'_'+inf+'_uk.json','w').write(json.dumps(uk)) 
ax[1].legend(loc=1,framealpha=0)
plt.subplots_adjust(wspace=0.1)
plt.suptitle("Power Factor ($p$) Sensitivity of Normalized Country Influence",fontsize=14)
#plt.savefig('../json/'+sd+'/'+ni+'_'+inf+'_powerfactor.png',dpi=150,bbox_inches = 'tight', pad_inches = 0.1, )
plt.show()<jupyter_output><empty_output><jupyter_text>ALLINONE<jupyter_code>#initialize renewable totals for learning
total2014={'csp':0,'solar':0,'wind':0}
learning={'csp':0.04,'solar':0.04,'wind':0.02}
year=2014
for fuel in total2014:
    total2014[fuel]=np.nansum([np.nansum(data[partner][year]['energy'][fuel]['cons']['navg3'])\
                           for partner in goodcountries if fuel in data[partner][year]['energy']])
total2014
#scenario id (folder id) 
sd='0a'
#first is scenario family, then do 4 variations of scenarios (2 selfinluence, 2 power factor) as 01, 02...

#import resources

###################################
###################################
#load resources
#predata=json.loads(file(pop_path+'maps/newres.json','r').read())
predata=json.loads(file(pop_path+'maps/res.json','r').read())
res={}
for c in predata:
    res[c]={}
    for f in predata[c]:
        res[c][f]={}
        for r in predata[c][f]:
            res[c][f][r]={}
            for year in predata[c][f][r]:
                res[c][f][r][int(year)]=predata[c][f][r][year]
predata={}
print 'scenario',sd,'loaded resources',
###################################
###################################
#load demand2
predata=json.loads(file(pop_path+'demand2.json','r').read())
demand2={}
for c in predata:
    demand2[c]={}
    for year in predata[c]:
        demand2[c][int(year)]=predata[c][year]
predata={}
print 'demand',
###################################
###################################
#load tradealpha d
#predata=json.loads(file(pop_path+'/trade/traded.json','r').read())
predata=json.loads(file(pop_path+'/trade/smalltrade.json','r').read())
tradealpha={}
for c in predata:
    tradealpha[c]={}
    for year in predata[c]:
        tradealpha[c][int(year)]=predata[c][year]
predata={}
print 'tradedata',
###################################
###################################
#reload impex and normalize
predata=json.loads(file(pop_path+'trade/nimpex.json','r').read())
nexportmatrix=predata["nexport"]
nimportmatrix=predata["nimport"]
nrexportmatrix=predata["nrexport"]
nrimportmatrix=predata["nrimport"]
predata={}
print 'impex',
###################################
###################################
#load latest savedata
#we dont change the data for now, everything is handled through trade
predata=json.loads(file(pop_path+'savedata6.json','r').read())
data={}
for c in predata:
    data[c]={}
    for year in predata[c]:
        data[c][int(year)]=predata[c][year]
predata={}
print 'data'
###################################
###################################
#reset balance
ybalance={}
#recalculate balances
for year in range(2015,2101):
    balance={}
    if year not in ybalance:ybalance[year]={}
    for c in goodcountries:
        balance[c]=0
        if c in tradealpha:
            f1=0
            for fuel in tradealpha[c][year]:
                if 'Import' in tradealpha[c][year][fuel]: 
                    f1=np.nansum([f1,sum(tradealpha[c][year][fuel]['Import'].values())])
                if 'Re-Import' in tradealpha[c][year][fuel]: 
                    f1=np.nansum([f1,sum(tradealpha[c][year][fuel]['Re-Import'].values())])
                if 'Export' in tradealpha[c][year][fuel]: 
                    f1=np.nansum([f1,-sum(tradealpha[c][year][fuel]['Export'].values())])
                if 'Re-Export' in tradealpha[c][year][fuel]: 
                    f1=np.nansum([f1,-sum(tradealpha[c][year][fuel]['Re-Export'].values())])
                if fuel in data[c][year]['energy']:
                    f1=np.nansum([f1,data[c][year]['energy'][fuel]['prod']['navg3']])
            balance[c]-=f1
        balance[c]+=demand2[c][year]*8760*1e-12
        if 'balance' not in data[c][year]['energy']:
            data[c][year]['energy']['balance']={'prod':{'navg3':0},'cons':{'navg3':0}}
        data[c][year]['energy']['balance']['prod']['navg3']=max(0,balance[c])#balance can't be negative
        data[c][year]['energy']['balance']['cons']['navg3']=max(0,balance[c])
    ybalance[year]=balance
#save3('0a') #save default
#scenario id (folder id) 
sd='0b'
ni='impex'
#first is scenario family, then do 4 variations of scenarios (2 selfinluence, 2 power factor) as 01, 02...

#import resources

###################################
###################################
#load resources
#predata=json.loads(file(pop_path+'maps/newres.json','r').read())
predata=json.loads(file(pop_path+'maps/res.json','r').read())
res={}
for c in predata:
    res[c]={}
    for f in predata[c]:
        res[c][f]={}
        for r in predata[c][f]:
            res[c][f][r]={}
            for year in predata[c][f][r]:
                res[c][f][r][int(year)]=predata[c][f][r][year]
predata={}
print 'scenario',sd,'loaded resources',
###################################
###################################
#load demand2
predata=json.loads(file(pop_path+'demand2.json','r').read())
demand2={}
for c in predata:
    demand2[c]={}
    for year in predata[c]:
        demand2[c][int(year)]=predata[c][year]
predata={}
print 'demand',
###################################
###################################
#load tradealpha d
#predata=json.loads(file(pop_path+'/trade/traded.json','r').read())
predata=json.loads(file(pop_path+'/trade/smalltrade.json','r').read())
tradealpha={}
for c in predata:
    tradealpha[c]={}
    for year in predata[c]:
        tradealpha[c][int(year)]=predata[c][year]
predata={}
print 'tradedata',
###################################
###################################
#reload impex and normalize
predata=json.loads(file(pop_path+'trade/'+ni+'.json','r').read())
nexportmatrix=predata["nexport"]
nimportmatrix=predata["nimport"]
nrexportmatrix=predata["nrexport"]
nrimportmatrix=predata["nrimport"]
predata={}
print ni,
###################################
###################################
#load latest savedata
#we dont change the data for now, everything is handled through trade
predata=json.loads(file(pop_path+'savedata6.json','r').read())
data={}
for c in predata:
    data[c]={}
    for year in predata[c]:
        data[c][int(year)]=predata[c][year]
predata={}
print 'data'
###################################
###################################
[importancematrix,influencematrix]=dendro(sd,9,3) #2,5, or 4,3
z=[np.mean(i) for i in influencematrix] #sum country influence on columns
#if you wanted weighted influence, introduce weights (by trade volume i guess) here in the above mean
s = pd.Series(1/np.array(z)) #need to 1/ to create inverse order
s=s.rank(method='dense').values.astype(int)-1 #start from 0 not one
#s is a ranked array on which country ranks where in country influence 
#we then composed the ordered vector of country influence
influencevector=[0 for i in range(len(s))]
for i in range(len(s)):
    influencevector[s[i]]=i
CV={}
CV2={}
TB={}
#load data - if already saved
predata=json.loads(file(pop_path+'savedata6.json','r').read())
data={}
for c in predata:
    data[c]={}
    for year in predata[c]:
        data[c][int(year)]=predata[c][year]
predata=json.loads(file(pop_path+'/trade/smalltrade.json','r').read())
tradealpha={}
for c in predata:
    tradealpha[c]={}
    for year in predata[c]:
        tradealpha[c][int(year)]=predata[c][year]
predata={}
fc={"solar":'pv',"csp":'csp',"wind":'wind'}
divfactor=10 #min trade partners in trade diversification
divshare=0.2 #min share of the trade diversification, total
tradeway={}
lifetime=20 #base lifetime
maxrut=0.01 #for each type #max rampup total, if zero 5% of 1% 0.05 / 0.001
maxrur=1.5 #growth rate for each techno #max rampup rate 0.5
omegamin=0.1 #min share of the in-country diversification, per fuel
random.seed(2)
cs=set()

for year in range(2015,2101):
    tradeable={}
    if year not in TB:TB[year]={}
    for i in range(len(goodcountries)):
        country=goodcountries[i]
        if country not in tradeable:tradeable[country]={'solar':0,'csp':0,'wind':0}
        for fuel in {"solar","csp","wind"}:
            if fuel not in data[country][year-1]['energy']:
                tradeable[country][fuel]=nrgsum(country,year-1)*maxrut
            elif data[country][year-1]['energy'][fuel]['prod']['navg3']==0:
                tradeable[country][fuel]=nrgsum(country,year-1)*maxrut
            else: tradeable[country][fuel]=max(nrgsum(country,year-1)*maxrut,
                                               data[country][year-1]['energy'][fuel]['prod']['navg3']*maxrur)
    for i in range(len(influencevector))[:]:#4344  
        country=goodcountries[influencevector[i]]
        cs.add(country)
        #if year==2015:
        if True:
            costvector={}
            for j in range(len(goodcountries)):
                partner=goodcountries[j]
                if partner not in costvector:costvector[partner]={}
                transactioncost=gridtestimator(country,partner)
                if country not in tradeway:tradeway[country]={}
                if partner not in tradeway[country]:tradeway[country][partner]=transactioncost["tradeway"]
                for fuel in {"solar","csp","wind"}:
                    ru0=0
                    if fuel not in data[partner][year]['energy']: ru = ru0
                    elif partner not in res: ru = ru0
                    elif sum(res[partner][fc[fuel]]['res'].values())==0: ru=1
                    elif data[partner][year]['energy'][fuel]['prod']['navg3']==0: ru=ru0
                    else: ru=data[partner][year]['energy'][fuel]['prod']['navg3']*1.0/\
                                        sum(res[partner][fc[fuel]]['res'].values())
                    ru=max(ru,0)
                    ru=max(1,0.3+ru**0.1) #or 0.3
                    costvector[partner][fuel]=1.0/influencematrix[influencevector[i]][j]*\
                                        transactioncost['scaler']*\
                                        ru*\
                                        1.0/(eroei[fc[fuel]]*1.0/np.mean(eroei.values())*\
                                             res_adv(partner,fuel)*\
                                             aroei[fc[fuel]]*1.0/np.mean(aroei.values()))
            cv=costvectorranker(costvector)
        #fulfill trade diversification criterion
        balance=divshare*cbalance(year,country)
        if balance>0:
            divfill(cv,divfactor,balance)
        #fulfill in-country diversification criterion
        wasalready=set()
        balance=cbalance(year,country)*omegamin
        if balance>0:
            omegafill(cv) #fill first best source to min share
            omegafill(cv) #fill second best source to min share
        #fill up rest of trade
        balance=cbalance(year,country)
        if balance>0:
            tradefill(cv)
        #fill liquids up to min liquid level
        liquidfill(country,year)
        print i,
        #CV2[country]=cv
    print year
save3(sd,cs)
file('E:/Dropbox/Public/datarepo/netset/savedata/'+sd+'data.json','w').write(json.dumps(data)) 
file('E:/Dropbox/Public/datarepo/netset/savedata/'+sd+'trade.json','w').write(json.dumps(tradealpha))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.21667332534292183, 'fraction_numerical': 0.018377946464242907, 'mean_word_length': 3.798402555910543, 'pattern_counts': {'":': 5, '<': 13, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '470995', 'license': 'MIT', 'n_tokens_mistral': 5151, 'n_tokens_neox': 4912, 'n_words': 966, 'path': 'code/netset8barebone-cycler1.ipynb', 'repo_name': 'csaladenes/netset', 'size': '15018'}