<jupyter_start><jupyter_text>Connect to Google Drive for storage   Pydrive<jupyter_code># install PyDrive
!pip install -U -q PyDrive
from pydrive.auth import GoogleAuth
from pydrive.drive import GoogleDrive
from google.colab import auth
from oauth2client.client import GoogleCredentials

# 1. Authenticate and create the PyDrive client.
auth.authenticate_user()
gauth = GoogleAuth()
gauth.credentials = GoogleCredentials.get_application_default()
drive = GoogleDrive(gauth)

# PyDrive reference:
# https://googledrive.github.io/PyDrive/docs/build/html/index.html

'''
# 2. Create & upload a file 
uploaded = drive.CreateFile({'title': 'Sample upload.txt'})
uploaded.SetContentString('Sample upload file content')
uploaded.Upload()
print('Uploaded file with ID {}'.format(uploaded.get('id')))

# 3. Load a file by ID and print its contents.
downloaded = drive.CreateFile({'id': uploaded.get('id')})
print('Downloaded content "{}"'.format(downloaded.GetContentString()))
'''
def upload_file_to_specific_folder(folder_id, file_name):
    file_metadata = {'title': file_name, "parents": [{"id": folder_id, "kind": "drive#childList"}]}
    folder = drive.CreateFile(file_metadata)
    folder.SetContentFile(file_name) #The contents of the file
    folder.Upload()
# Auto-iterate through all files that matches this query
#NNRatioData: 1iYZSLhjTVMwaX3PVK5dzJdywjpqsDS1Z
file_list = drive.ListFile({'q': "'1SH8q5-SqxKrpBax48m8HImqqlHSHSVfN' in parents and trashed=false"}).GetList()
for file1 in file_list:
  print('title: %s, id: %s' % (file1['title'], file1['id']))

#Download to colab
# Load a file by ID and print its contents.
for file1 in file_list:
  downloaded = drive.CreateFile({'id':file1['id']})
  downloaded.GetContentFile(file1['title'])
#print('Downloaded content "{}"'.format(downloaded.GetContentString()))
print(os.getcwd())
print(os.listdir('/content'))<jupyter_output>/content
['datalab', '.cache', '.ipython', '.forever', '.config', '.keras', '.local']<jupyter_text>Main<jupyter_code>#sigmoid function
def sigmoid(x):
  return 1 / (1 + exp(-x))<jupyter_output><empty_output><jupyter_text>weight-clipping<jupyter_code>class WeightClip(Constraint):
    '''Clips the weights incident to each hidden unit to be inside a range
    '''
    def __init__(self, c=2):
        self.c = c

    def __call__(self, p):
        return K.clip(p, -self.c, self.c)

    def get_config(self):
        return {'name': self.__class__.__name__,
                'c': self.c}
#@title Glabal Parameters
epochs = 100000 #@param {type:"integer"}
batch_size = 1000 #@param {type:"integer"}
Geteps = 0.005 #@param {type:"number"}
Getmu = 0.8 #@param {type:"number"}
Getsigma = 0.02 #@param {type:"number"}
cut = 0. #@param {type:"number"}
gcut = 0. #@param {type:"number"}

'''Distribution functions'''
def SM(x):
  return exp(-8*x)

def BSM(x):
  return exp(-(x-Getmu)**2/(2*Getsigma**2))
#Normalize distribution
SM_norm = integrate.quad(lambda y :SM(y),cut,1)
BSM_norm = integrate.quad(lambda y :BSM(y),cut,1)
#SM_norm_c = integrate.quad(lambda y :SMn(y),gcut,1)

#normalized distribution
def SMn(x):
  return exp(-8*x)/SM_norm[0]

def BSMn(x):
  return (SMn(x)+Geteps*BSM(x)/BSM_norm[0])/(1+Geteps)

SM_norm_c = integrate.quad(lambda y :SM(y),gcut,1)

def SMnc(x):
  return exp(-8*x)/SM_norm_c[0]


#define probability distribution function
class P_SM(st.rv_continuous):
    def _pdf(self,x):
        return SMn(x)
      
class P_BSM(st.rv_continuous):
    def _pdf(self,x):
        return BSMn(x)        
      
class P_SMc(st.rv_continuous):
    def _pdf(self,x):
        return SMnc(x)
      
      
      
SM_gen = P_SM(a=cut,b=1,name='sm_sample')
BSM_gen = P_BSM(a=cut,b=1,name='bsm_sample')
SMc_gen = P_SMc(a=gcut,b=1,name='smc_sample')
def anglen(x):
  nm = integrate.quad(lambda y : 1*cos(y),-pi/2,pi/2)
  return (1*cos(x))/nm[0]
  #return 1

class P_angle(st.rv_continuous):
  def _pdf(self,x):
    return anglen(x)
  

#angle_gen = P_angle(a=-pi/2, b=pi/2, name='angle')
angle_gen = P_angle(a=-pi/2, b=pi/2, name='angle')
def bsmSig(x):
  return (Geteps*exp(-(x-Getmu)**2/(2*Getsigma**2)))/BSM_norm[0]

def smSig(x):
  return (exp(-8*x))/BSM_norm[0]<jupyter_output><empty_output><jupyter_text>Save samples to Google Drive<jupyter_code>#Save samples
#with open("smbsmSamples.npy",'w') as ayf:
#  ayf.write('sm_sample')

np.savez('Samples200k_2k',bsm_sample=bsm_sampleT,sm_sample=sm_sample)
#np.save('SMSamples200k.npy',sm_sample)
#np.save('BSMSamples2k.npy',bsm_sample)
#from pydrive.auth import GoogleAuth
#from pydrive.drive import GoogleDrive
#from google.colab import auth
#from oauth2client.client import GoogleCredentials

# 1. Authenticate and create the PyDrive client.
auth.authenticate_user()
gauth = GoogleAuth()
gauth.credentials = GoogleCredentials.get_application_default()
drive = GoogleDrive(gauth)

# PyDrive reference:
# https://googledrive.github.io/PyDrive/docs/build/html/index.html

# 2. Create & upload a file 
uploaded = drive.CreateFile({'title': 'Samples200k_2k.npz'})
uploaded.SetContentFile('Samples200k_2k.npz')
uploaded.Upload()
print('Uploaded file with ID {}'.format(uploaded.get('id')))

print(os.getcwd())
print(os.listdir('/content'))
#print(os.listdir('/'))<jupyter_output>/content
['datalab', '.config', 'tarray_all.npy', '.forever', 'BSMSamples2k.npy', '.ipython', '.cache', 'Samples200k_2k.npz', '.local', 'model_5_3.h5', '.keras', 'BsmSamples1500_1000.npy', 'BsmSamples', 'model_5_3_S200k_2k_R100k.h5', 'model_5_3_S1500_1000.h5', 'SMSamples200k.npy', '.rnd']<jupyter_text>Sample Histogram<jupyter_code>histg, xedges, yedges =np.histogram2d(x_train[:,0],x_train[:,1])
f = plt.figure(figsize=(7,3))
ax = f.add_subplot(131, title='imshow: square bins')
ax.hist2d(x_train[:,0],x_train[:,1])
#Plot Histogram
#plt.hist(sm_sample, bins='auto')  # arguments are passed to np.histogram
plt.hist(x_train[:,0], bins='auto') 
plt.title("Reference Dist Exp(-8x)")
plt.show()

bsm_cut = []
for x in bsm_sample:
  if x >= 0.5:
    bsm_cut.append(x)
bsm_hist=plt.hist(bsm_sample, bins=25)  # arguments are passed to np.histogram
print(len(bsm_cut),len(sm_cut))
sm_target_c = np.zeros(len(sm_cut))
bsm_target_c = np.ones(len(bsm_cut))

x_train_cut = np.append(sm_cut,bsm_cut)
y_train_cut = np.append(sm_target_c,bsm_target_c)

rfw = len(sm_cut)/len(bsm_cut)<jupyter_output><empty_output><jupyter_text>generate another sample<jupyter_code>bsm_sampleT= BSM_gen.rvs(size=2000)

bsm_cutT = []
for x in bsm_sampleT:
  if x >= 0.5:
    bsm_cutT.append(x)
test_hist=plt.hist(bsm_cutT, bins=25)
np.savez('Samples200k_2k_R2',bsm_sample=bsm_sampleT,sm_sample=sm_sample)
print(os.getcwd())
print(os.listdir('/content'))
sm_target_c = np.zeros(len(sm_cut))
bsm_target_c = np.ones(len(bsm_cutT))

x_train_cut = np.append(sm_cut,bsm_cutT)
y_train_cut = np.append(sm_target_c,bsm_target_c)

rfw = len(sm_cut)/len(bsm_cutT)
#from pydrive.auth import GoogleAuth
#from pydrive.drive import GoogleDrive
#from google.colab import auth
#from oauth2client.client import GoogleCredentials

# 1. Authenticate and create the PyDrive client.
auth.authenticate_user()
gauth = GoogleAuth()
gauth.credentials = GoogleCredentials.get_application_default()
drive = GoogleDrive(gauth)

# PyDrive reference:
# https://googledrive.github.io/PyDrive/docs/build/html/index.html

# 2. Create & upload a file 
uploaded = drive.CreateFile({'title': 'Samples200k_2k_R2.npz'})
uploaded.SetContentFile('Samples200k_2k_R2.npz')
uploaded.Upload()
print('Uploaded file with ID {}'.format(uploaded.get('id')))<jupyter_output>Uploaded file with ID 1irHGT7u8xWDZBzfAryesx0dwNI2GdcGH<jupyter_text>Build NN and Train<jupyter_code>#define custom loss function

def customloss(yTrue,yPred):
  return yTrue*K.log(1+K.exp(-yPred))+1/rfw*(1-yTrue)*K.log(1+K.exp(yPred))

rmsprop = keras.optimizers.RMSprop()

#clipvalue=0.5
sgd = optimizers.SGD()<jupyter_output><empty_output><jupyter_text>Generate samples<jupyter_code>#Data and References
NRsm=200000
Nbsm=4020
NR = 4000

Ref_sample = SM_gen.rvs(size=NRsm)
bsm_sample= BSM_gen.rvs(size=Nbsm)

#Angle distribution function
#anglesSM = np.random.uniform(low=-pi,high=pi,size=NRsm)
#anglesBSM = np.random.uniform(low=-pi,high=pi,size=Nbsm)
anglesSM = angle_gen.rvs(size=NRsm)
anglesBSM= angle_gen.rvs(size=Nbsm)

#sm_target = np.zeros(NRsm)
#bsm_target = np.ones(Nbsm)

#xmass = np.append(sm_sample,bsm_sample)
#xangle = np.append(anglesSM,anglesBSM)

#x_train = np.append(sm_sample,bsm_sample)
#x_train = np.column_stack((xmass,xangle))
#y_train = np.append(sm_target,bsm_target)

rfw = NRsm/NR
bsm_sample= BSM_gen.rvs(size=Nbsm)
anglesSM = angle_gen.rvs(size=NRsm)
anglesBSM= angle_gen.rvs(size=Nbsm)<jupyter_output><empty_output><jupyter_text>save the generated sample<jupyter_code>sfilename = 'Sample200k_4k_R1n'
#Sample200k_4k.npz
np.savez(sfilename,bsm_sample=bsm_sample,Ref_sample=Ref_sample,anglesSM=anglesSM,anglesBSM=anglesBSM)

upload_file_to_specific_folder('1SH8q5-SqxKrpBax48m8HImqqlHSHSVfN',sfilename+'.npz')<jupyter_output><empty_output><jupyter_text>Load saved samples<jupyter_code>samples=np.load('Sample200k_4k_R12.npz');
NRsm=200000
Nbsm=4020
sm_sample = samples['sm_sample']
bsm_sample= samples['bsm_sample']

#anglesSM=samples['anglesSM'];
#anglesBSM=samples['anglesBSM'];
anglesSM = angle_gen.rvs(size=NRsm)
anglesBSM= angle_gen.rvs(size=Nbsm)

#xmass = np.append(sm_sample,bsm_sample)
#xangle = np.append(anglesSM,anglesBSM)
#sm_target = np.zeros(NRsm)
#bsm_target = np.ones(Nbsm)

#x_train = np.append(sm_sample,bsm_sample)
x_train = np.column_stack((xmass,xangle))
y_train = np.append(sm_target,bsm_target)

rfw = NRsm/Nbsm<jupyter_output><empty_output><jupyter_text>Binning the Ref Sample<jupyter_code>#x_train_bsm = np.column_stack(bsm_sample,anglesBSM)
Nb_angle = 50
Nb_mass = 100

H2d,yedge,xedge = np.histogram2d(anglesSM,Ref_sample,bins=[np.linspace(-pi/2,pi/2,Nb_angle+1),np.linspace(0,1,Nb_mass+1)])
def moving_average(a, n=2) :
    ret = np.cumsum(a, dtype=float)
    ret[n:] = ret[n:] - ret[:-n]
    return ret[n - 1:] / n

xpos = moving_average(xedge,2);

ypos = moving_average(yedge,2);

wlist = []

for xidx, xval in enumerate(xpos):
  for yidx, yval in enumerate(ypos):
    wlist.append([yval,xval,H2d[yidx][xidx]])

xposf=[]; yposf=[]; zposf = [] ;

for elem in wlist:
  xposf.append(elem[1])
  yposf.append(elem[0])
  zposf.append(elem[2])
fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')

#testarray=np.array([1,2,3,4])

ax.scatter(xposf, yposf, zposf)
ax.view_init(30,50)
ax.set_xlabel('X Label')
ax.set_ylabel('Y Label')
ax.set_zlabel('Z Label')
#ax.scatter(testarray,testarray,testarray, marker='o')
Nbins = Nb_angle*Nb_mass
xmass = np.append(xposf,bsm_sample)
xangle = np.append(yposf,anglesBSM)
#sm_target = np.zeros(NRsm)
sm_target = np.zeros(Nbins)
bsm_target = np.ones(Nbsm)

#x_train = np.append(sm_sample,bsm_sample)
x_train = np.column_stack((xmass,xangle))
y_train = np.append(sm_target,bsm_target)
weightlossa = np.append(np.asarray(zposf),bsm_target)
weightloss = K.variable(value=weightlossa.reshape((Nbsm+Nbins,1)))

#weightlossdict = dict(zip(y_train,weightloss))
#weightlossdict = {'y_train':weightloss}


#rfw = NRsm/Nbsm
print(x_train.shape,
y_train.shape,
      weightloss.shape)
fig = plt.figure(figsize=(10,15))
#ax1 = fig.add_subplot(221, projection='3d'
ax = fig.add_subplot(111)
ax.set_title('pcolormesh: exact bin edges')
X, Y = np.meshgrid(xedge, yedge)
ax.pcolormesh(X, Y, H2d)
ax.set_aspect('equal')
f = plt.figure(figsize=(7,7))
ax = f.add_subplot(111, title='imshow: square bins')
ax.hist2d(x_train[:,0],x_train[:,1],bins=25)
plt.hist(bsm_sample, bins=25)
#Plot x distribution
sm_cut = []
for y in Ref_sample:
  if y >= gcut:
    sm_cut.append(y)


bsm_cut = []
for x in bsm_sample:
  if x >= 0.5:
    bsm_cut.append(x)
    
    
bsm_hist=plt.hist(bsm_cut, bins=25)
print( len(sm_cut), len(bsm_cut))
#samples after the cut
Nsm_c = len(sm_cut)
Nbsm_c = len(bsm_cut)

#angles = np.random.uniform(-pi,pi,1200)
anglesSM_c = angle_gen.rvs(size=Nsm_c)
anglesBSM_c= angle_gen.rvs(size=Nbsm_c)

sm_target_c = np.zeros(Nsm_c)
bsm_target_c = np.ones(Nbsm_c)

xmass_c = np.append(sm_cut,bsm_cut)
xangle_c = np.append(anglesSM_c,anglesBSM_c)

#x_train = np.append(sm_sample,bsm_sample)
x_train_c = np.column_stack((xmass_c,xangle_c))
y_train_c = np.append(sm_target_c,bsm_target_c)

rfw_c = Nsm_c/Nbsm_c
f = plt.figure(figsize=(7,7))
ax = f.add_subplot(111, title='imshow: square bins')
ax.hist2d(x_train_c[:,0],x_train_c[:,1],bins=25)<jupyter_output><empty_output><jupyter_text>loss functions<jupyter_code>#define custom loss function

def customlossML(yTrue,yPred):
  return -yTrue*yPred+1/rfw*(1-yTrue)*(K.exp(yPred)-1)

rmsprop = keras.optimizers.RMSprop()

#clipvalue=0.5
#Maximum likelihood loss function with binning weight
def customlossML(sw):
  #ksw = K.variable(value=sw0.reshape((len(sw0),1)))
  Nt = Nbsm+Nbins
  def lossML(yTrue,yPred):
    sw_rs = K.reshape(sw,(Nt,1))
    ytrue_rs = K.reshape(yTrue,(Nt,1))
    ypred_rs = K.reshape(yPred,(Nt,1))
    return -K.sum(ytrue_rs*sw_rs*ypred_rs)+K.sum((1-ytrue_rs)*sw_rs*(K.exp(ypred_rs)-1))/rfw
    #return (K.exp(ypred_rs)-1)
    #return K.sum(-ytrue_rs*ypred_rs*sw_rs+1/rfw*(1-ytrue_rs)*(K.exp(ypred_rs)-1)*sw_rs)
  return lossML

rmsprop = keras.optimizers.RMSprop()
samplefile='Samples800_100_s1'

np.savez(samplefile,xmass=xmass,xangle=xangle)



# 1. Authenticate and create the PyDrive client.
auth.authenticate_user()
gauth = GoogleAuth()
gauth.credentials = GoogleCredentials.get_application_default()
drive = GoogleDrive(gauth)

# PyDrive reference:
# https://googledrive.github.io/PyDrive/docs/build/html/index.html

# 2. Create & upload a file 
uploaded = drive.CreateFile({'title': samplefile+'.npz'})
uploaded.SetContentFile(samplefile+'.npz')
uploaded.Upload()
print('Uploaded file with ID {}'.format(uploaded.get('id')))<jupyter_output>Uploaded file with ID 1gDvc_ckFfSJCiXvihZPLvsHCuHMjAyYF<jupyter_text>build NN<jupyter_code># build the model
model = Sequential()
#model.add(Dense(3, activation='sigmoid',input_shape=(2,), W_constraint = WeightClip(50)))
model.add(Dense(4, activation='sigmoid',input_shape=(2,),W_constraint = WeightClip(40),kernel_regularizer=keras.regularizers.l2(0.000000)))
#model.add(Dense(5, activation='sigmoid',kernel_initializer='zeros',bias_initializer='zeros',kernel_regularizer=keras.regularizers.l2(0.0001)))
#model.add(Dense(5, activation='sigmoid'))
model.add(Dense(1,W_constraint = WeightClip(40)))


# compile the model
model.compile(loss=customlossML(weightloss),
              optimizer=rmsprop)
model.save_weights("modelML_5_2d_init.h5")
model.load_weights("modelML_5_2d_init.h5")
upload_file_to_specific_folder('1cInKkYhUABZgHm72qidU3rE6lFca3OJP',"modelML_5_2d_init.h5")
'''
#Visualize network
#from keras.utils import plot_model
#plot_model(model,to_file='model.png',show_shapes='True')
from IPython.display import SVG
from keras.utils.vis_utils import model_to_dot

SVG(model_to_dot(model).create(prog='dot', format='svg'))
'''
#load weight and continue training
#import h5py

#sample1
#model.load_weights("model_5_2d_R3M.h5")

#sample2
#model.load_weights("model_5_R2_R500k.h5")

#2d sample
#model.load_weights("modelML_3_2d_cut4_D1k_600_L2R5_MN0_sigmoid_500k.h5")
model.load_weights("modelML_5_2d_init.h5")<jupyter_output><empty_output><jupyter_text>Train NN<jupyter_code>#import time
start = time.time()
print(start)
#t = time.process_time()
model.fit(x_train, y_train,
          batch_size=len(x_train),
          epochs=100000,
          shuffle=False,
          verbose=0)

filename = 'modelML_3_2d_cut_3_D200k_4k_L20_WC40_sigmoid_400k.h5'
#model.save_weights(filename)




# 1. Authenticate and create the PyDrive client.
#auth.authenticate_user()
#gauth = GoogleAuth()
#gauth.credentials = GoogleCredentials.get_application_default()
#drive = GoogleDrive(gauth)

# PyDrive reference:
# https://googledrive.github.io/PyDrive/docs/build/html/index.html

# 2. Create & upload a file 
#uploaded = drive.CreateFile({'title': filename})
#uploaded.SetContentFile(filename)
#uploaded.Upload()
#print('Uploaded file with ID {}'.format(uploaded.get('id')))
#print(time.process_time() - t)

#upload_file_to_specific_folder('1cInKkYhUABZgHm72qidU3rE6lFca3OJP',filename)
filename = 'modelML_3_2d_cut_5_D200k_4k_L2R5E6_MW10_sigmoid_600k.h5'
model.save_weights(filename)
upload_file_to_specific_folder('1cInKkYhUABZgHm72qidU3rE6lFca3OJP',filename)


def t_vs_run(drun,ite):
    t_v_array=[]
    i=0;
    while i < ite:
      model.fit(x_train, y_train,
          batch_size=len(x_train),
          epochs=drun,
          shuffle=False,
          verbose=0);
      #tary = 2*(model.predict(np.column_stack((bsm_cut,anglesBSM_c)))).flatten();
      tary = -2*(model.evaluate(x=x_train,y=y_train,batch_size=len(x_train))).flatten();
      t_v_array = np.append(t_v_array,np.sum(tary));
      i +=1
      
    return t_v_array
#R14
#t = time.process_time()
ta = t_vs_run(10000,40)
#print(time.process_time() - t)

print(ta)
#R16 R2 clip 40
ta = t_vs_run(10000,20)

print(ta)
ta = t_vs_run(10000,50)

print(ta)
for style in plt.style.available:
  print(style)
plt.style.use('bmh')


fig, ax = plt.subplots()
ax.plot(range(1,len(ta)+1),ta)
#ax.plot(xinput,ypred,'.')
#ax.hist(bsm_sample, bins=25)
ax.set_xlabel('x 10k runs',size=15)
ax.set_ylabel('t (D)',size=15)
ax.spines['bottom'].set_linewidth(1.)
ax.spines['left'].set_linewidth(1.)
#ax.spines['bottom'].set_visible(True)
#ax.spines['left'].set_visible(True)
for tick in ax.xaxis.get_major_ticks():
    tick.label.set_fontsize(14)
    
for tick in ax.yaxis.get_major_ticks():
    tick.label.set_fontsize(14)
    
plt.title("SM: 200k, BSM: 4020, x>0.5 ", size=18)

 print(os.getcwd())
dirs=os.listdir('/content')
for f in dirs:
  print(f)<jupyter_output>/content
datalab
.cache
.ipython
.nv
.forever
.config
.keras
.local<jupyter_text>two branch architechture<jupyter_code>#load sample

samples=np.load('Samples3k_500.npz');
pt_branch = Sequential()
pt_branch.add(Dense(4, activation='sigmoid',input_dim=1,W_constraint = WeightClip(40),b_constraint = WeightClip(40)))
#pt_branch.add(Dense(3, activation='sigmoid'))

angle_branch = Sequential()
angle_branch.add(Dense(4, activation='sigmoid',input_dim=1,W_constraint = WeightClip(40),b_constraint = WeightClip(40)))
#angle_branch.add(Dense(3, activation='sigmoid'))

merged = Merge([pt_branch, angle_branch], mode='concat')

model2b = Sequential()
model2b.add(merged)
model2b.add(Dense(3, activation='sigmoid',W_constraint = WeightClip(40)))
model2b.add(Dense(1,W_constraint = WeightClip(40),b_constraint = WeightClip(40)))

model2b.compile(loss=customlossML(weightloss),
              optimizer=rmsprop)
model2b.load_weights("model2b_5_cut5_2d_R3M.h5")
import time
start = time.time()
print(start)
t = time.process_time()
model2b.fit([xmass,xangle], y_train,
          batch_size=len(y_train),
          epochs=100000,
          shuffle=False,
          verbose=0)

filename = 'model2b_5_cut6_2d_S200_200_R100k.h5'
#model2b.save_weights(filename)
'''
#from google.colab import files
#files.download("/content/model1.json")
#files.download("/content/model_5_3_S1500_1000.h5")
#from pydrive.auth import GoogleAuth
#from pydrive.drive import GoogleDrive
#from google.colab import auth
#from oauth2client.client import GoogleCredentials

# 1. Authenticate and create the PyDrive client.
auth.authenticate_user()
gauth = GoogleAuth()
gauth.credentials = GoogleCredentials.get_application_default()
drive = GoogleDrive(gauth)

# PyDrive reference:
# https://googledrive.github.io/PyDrive/docs/build/html/index.html

# 2. Create & upload a file 
uploaded = drive.CreateFile({'title': filename})
uploaded.SetContentFile(filename)
uploaded.Upload()
print('Uploaded file with ID {}'.format(uploaded.get('id')))
print(time.process_time() - t)
'''
weightBrA = model2b.layers[0].get_weights()

print(weightBrA)
print(os.getcwd())
print(os.listdir('/content'))
#load weight and continue training
model2b.load_weights('model2b_5_2d_R200k.h5')<jupyter_output><empty_output><jupyter_text>Save and Continue training<jupyter_code>#load weight and continue training
model.load_weights("model_5_3.h5")
model.fit(x_train, y_train,
          batch_size=batch_size,
          epochs=2000,
          verbose=0)
#Save weight
model_json = model.to_json()
with open("model1.json","w") as json_file:
  json_file.write(model_json)
#Save weight to HDF5
model.save_weights("model2.h5")
print("Save model to disk")
print(os.getcwd())
print(os.listdir('/content'))
#download weight
from google.colab import files
files.download("/content/model1.json")
files.download("/content/model1.h5")
#upload saved data
from google.colab import files
uploaded = files.upload()

for fn in uploaded.keys():
  print('User uploaded file "{name}" with length {length} bytes'.format(
      name=fn, length=len(uploaded[fn])))<jupyter_output><empty_output><jupyter_text>Export NN function and Plot Result<jupyter_code>def BSM2d(x,y):
  return BSMn(x)*(1*cos(y))

def SM2d(x,y):
  return SMn(x)*(1*cos(y))

def BSM2d_c(x,y,cutv):
  norm1 = integrate.quad(lambda a :BSMn(a),cutv,1)
  return BSMn(x)/norm1[0]*anglen(y)

def SM2d_c(x,y,cutv):
  norm2 = integrate.quad(lambda a :SMn(a),cutv,1)
  return SMn(x)/norm2[0]*anglen(y)
model.load_weights("modelML_3_2d_cut_5_D200k_4k_L2R5E6_MW10_sigmoid_600k.h5")
x = np.arange(0.5,1,0.01)
#y = np.arange(-pi,pi,0.01)
y = np.arange(-pi/2,pi/2,0.025)

tseq = []
for j in y:
  for i in x:
    tseq = np.append(tseq,exp(model.predict(np.asarray([[i,j]])))*SM2d_c(i,j,0.5))
    #tseq = np.append(tseq,exp(model2b.predict([np.asarray([[i]]),np.asarray([[j]])]))*SM2d_c(i,j,0.5))
    
    
    
tseqr = np.reshape(tseq,(len(y),len(x)))<jupyter_output><empty_output><jupyter_text>print weight<jupyter_code>weightL1 = model.layers[0].get_weights()

print(weightL1)
weightL2 = model.layers[1].get_weights()

print(weightL2)
weightL3 = model.layers[2].get_weights()

print(weightL3)
#hfont = {'fontname':'Helvetica'}

fig = plt.figure(figsize=(15,15))
ax1 = fig.add_subplot(221, projection='3d')

#x = np.arange(0.5,1,0.01)
#y = np.arange(-pi,pi,0.01)

#xx, yy = np.meshgrid(x,y, sparse = True)
xx,yy = np.meshgrid(x,y, sparse = True)

#BSM2d(r)
z=np.vectorize(BSM2d_c)(xx,yy,0.5)

ax1.plot_surface(xx,yy,z)
ax1.set_xlabel('x')
ax1.set_ylabel(r'$\theta$')
#ax1.zscale('log', nonposy='clip')
plt.title(r'Generator: $\left( N(e^{-8x})+\epsilon N(e^{(-(x-\mu)^2)/(2 \sigma^2)}) \right) \cdot (1-\cos \theta) $',fontsize=14)
for tick in ax1.xaxis.get_major_ticks():
    tick.label.set_fontsize(11)
    
for tick in ax1.yaxis.get_major_ticks():
    tick.label.set_fontsize(11)


ax2 = fig.add_subplot(222, projection = '3d')
ax2.plot_surface(xx,yy,tseqr)
ax2.set_xlabel('x')
ax2.set_ylabel(r'$\theta$')
plt.title("Trained Network Output",fontsize=14)

ax3 = fig.add_subplot(223, projection = '3d')
ax3.plot_surface(xx,yy,z)
ax3.set_xlabel('x')
ax3.set_ylabel(r'$\theta \;\;$' 'angle')
ax3.view_init(20,60)
plt.title(r'$\mu=0.8,\;\sigma=0.02,\;\epsilon =0.005$',fontsize=14)

ax4 = fig.add_subplot(224, projection = '3d')
ax4.plot_surface(xx,yy,tseqr)
ax4.set_xlabel('x')
ax4.set_ylabel(r'$\theta$')
ax4.view_init(20,60)
#plt.title("Arch: 1 layer 3 neurons, sigmoid activation, L2 regul.=5E-6",fontsize=14)

plt.suptitle("SM: 200k, BSM: 4020, 600k Rounds \n Arch: 1 layer 3 neurons, sigmoid activation, L2 regul.=5E-6", fontsize=20)

plotfilename = 'NN_3N_cut5_D200k_4020_L2_5E6_WC10_sigmoid_600k.pdf'

fig.savefig(plotfilename)

# 1. Authenticate and create the PyDrive client.
auth.authenticate_user()
gauth = GoogleAuth()
gauth.credentials = GoogleCredentials.get_application_default()
drive = GoogleDrive(gauth)

#plotfile = '2d_train_7_2d_cut5_D4k_1k_clipv5_R5000k.pdf'
# PyDrive reference:
# https://googledrive.github.io/PyDrive/docs/build/html/index.html

# 2. Create & upload a file 
#uploaded = drive.CreateFile({'title': plotfilename})
#uploaded.SetContentFile(plotfilename)
#uploaded.Upload()
#print('Uploaded file with ID {}'.format(uploaded.get('id')))

upload_file_to_specific_folder('1cInKkYhUABZgHm72qidU3rE6lFca3OJP',plotfilename)

tary = 2*(model.predict(np.column_stack((bsm_cut,anglesBSM_c)))).flatten()
np.sum(tary)
fig, ax = plt.subplots()
ax.plot(xinput,yorig,'*')
ax.plot(xinput,ypred,'.')
ax.hist(bsm_sample, bins=25)
plt.yscale('log', nonposy='clip')
plt.title("SM: 1k, BSM: 100, Sample 1, 1M Rounds")
#compare NN output and generating function
xinput_all = np.linspace(0.,1,1000)
y=[]
for xi in xinput_all:
  #y.append(exp(NNfunction2(xi,weightL1,weightL2)[0][0])*SMn(xi))
  #y.append(exp(NNfunction(xi,weightL1,weightL2)[0][0]))
  y.append(BSMn(xi))
  
  
ypred_all = np.vectorize(exp)(model.predict(xinput_all));
ypred_all = ypred_all.flatten()*np.vectorize(SMn)(xinput_all);<jupyter_output><empty_output><jupyter_text>Compute test statistics<jupyter_code>#Likelihood Ratio Test
tlist = []
for xii in bsm_sample:
  tlist.append(NNfunction2(xii,weightL1,weightL2)[0][0])
  
print(np.sum(tlist))


tarray = 2*(model.predict(bsm_sample)).flatten();
print(np.sum(tarray))
#distribution of t
Nsmb = 4000;
rfw_tc = len(sm_cut)/Nsmb;

def customlossMLc(yTrue,yPred):
  return -yTrue*yPred+1/rfw_tc*(1-yTrue)*(K.exp(yPred)-1)

class test_statistics:
  '''
  Compute test statistics given data sample(bsm or sm) 
  '''
  def __init__(self,NNmodel,lossfunc,RefSample,TrainSample):
    self.NNmodel = NNmodel
    self.lossfunc = lossfunc
    self.RefSample = RefSample
    self.sm_sample_t = SMc_gen.rvs(size=Nsmb)
    self.anglesSM_t = angle_gen.rvs(size=Nsmb)
    
  #train NN with data(input) sample and ref sample
  #training variables
  
  #generate sm sample
  
  def train_t(self,epoch,lossfunc):
    self.sm_sample_t = SMc_gen.rvs(size=Nsmb)
    self.anglesSM_t = angle_gen.rvs(size=Nsmb)
    bsm_target_tc = np.ones(Nsmb)
    xmass_t = np.append(sm_cut,self.sm_sample_t)
    xangle_t = np.append(anglesSM_c,self.anglesSM_t)
    
    #bsm_target_t = np.ones(Nbsm_c)

    x_train_t = np.column_stack((xmass_t,xangle_t))
    y_train_t = np.append(sm_target_c,bsm_target_tc)
    self.NNmodel.compile(loss=lossfunc,optimizer=rmsprop)
    #self.NNmodel.load_weights("modelML_5_2d_init.h5")
    self.NNmodel.fit(x_train_t, y_train_t,
          batch_size=len(x_train_t),
          epochs=epoch,
          verbose=0);
    tary = 2*(self.NNmodel.predict(np.column_stack((self.sm_sample_t,self.anglesSM_t)))).flatten();
    return (np.sum(tary))
    
  def train_result(self,cutv):
    x = np.arange(cutv,1,0.01)
    y = np.arange(-pi/2,pi/2,0.025)

    tseq = []
    for j in y:
      for i in x:
        tseq = np.append(tseq,exp(model.predict(np.asarray([[i,j]])))*SM2d_c(i,j,cutv))
    
    tseqr = np.reshape(tseq,(len(y),len(x)))
    #fig, ax = plt.subplots(projection='3d')
    
    fig1 = plt.figure(figsize=(7,7))
    xx,yy = np.meshgrid(x,y, sparse = True)
    ax = fig1.add_subplot(111, projection = '3d')
    ax.plot_surface(xx,yy,tseqr)
    ax.set_xlabel('x')
    ax.set_ylabel(r'$\theta$')
    plt.title("SM: 2k, BSM: 200, Sample 1, 1.2M Rounds",fontsize=14)
    plt.show()
    
    
  def t_vs_run(self,drun,ite):
    xmass_t = np.append(sm_cut,self.sm_sample_t)
    xangle_t = np.append(anglesSM_c,self.anglesSM_t)
    bsm_target_tc = np.ones(Nsmb)
    x_train_t = np.column_stack((xmass_t,xangle_t))
    y_train_t = np.append(sm_target_c,bsm_target_tc)
    self.NNmodel.compile(loss=self.lossfunc,optimizer=rmsprop)
    
    t_v_array=[]
    i=0;
    while i < ite:
      self.NNmodel.fit(x_train_t, y_train_t,
          batch_size=len(x_train_t),
          epochs=drun,
          verbose=0);
      tary = 2*(self.NNmodel.predict(np.column_stack((self.sm_sample_t,self.anglesSM_t)))).flatten();
      t_v_array = np.append(t_v_array,np.sum(tary));
      i +=1
      
    return t_v_array
  
  def t_value(self):
    tary = 2*(self.NNmodel.predict(np.column_stack((sm_sample_t,anglesSM_t)))).flatten();
    return (np.sum(tary)) 
len(anglesSM_c)
len(sm_cut)
smtrain1 = test_statistics(model,customlossMLc)
tarray = smtrain1.t_vs_run(10000,100)
print(tarray)
fig, ax = plt.subplots(figsize = (8,6))
#fig1 = plt.figure(facecolor='white')
#ax = plt.axes(frameon=True)
#ax.set_facecolor('white')
ax.plot(range(1,len(tarray)+1),tarray)
#ax.plot(xinput,ypred,'.')
#ax.hist(bsm_sample, bins=25)
ax.axis('auto')
#ax.set_anchor((0,0))
ax.grid(True)
ax.set_xlabel('x 10k runs',size=15)
ax.set_ylabel('t (R)',size=15)
#ax.spines['bottom'].set_linewidth(1)
#ax.spines['left'].set_linewidth(1)
#ax.spines['bottom'].set_visible(True)
ax.spines['left'].set_visible(True)
#plt.yscale('log', nonposy='clip')
#plt.xticks(range(1,10,10),range(1,10,10))
ax.set_title("SM: 200k, BSM: 4020, x>0.5",size=18)
plt.subplots_adjust(bottom=0.15)


for label in ax.get_ticklabels():
  label.set_rotation(45)
  label.set_fontsize(16)

#ax.draw()
tarray10k50R1=np.array([6.51207161  7.2483387   7.71594048  8.07175255  8.3565979   8.59105492
  8.7889576   8.95766449  9.10435104  9.2323494   9.34585571  9.44564629
  9.53365326  9.61269951  9.68392372  9.74649429  9.8041153   9.85485649
  9.90026665  9.94205093  9.9802475  10.01398277 10.04444122 10.07260323
 10.09846115 10.12013817 10.14096451 10.16246414 10.17786789 10.19262123
 10.20655823 10.22085571 10.23260307 10.24283028 10.25219917 10.26178169
 10.26958847 10.27536583 10.28084564 10.28681183 10.29360199 10.29886055
 10.30295563 10.30595207 10.30957222 10.31306839 10.31357193 10.3149128
 10.31832123 10.31690598])
len(tarray)
smtrain1.train_t(20000,customlossML)
smtrain1.train_result(0.5)
def get_tsm(Nsample):
  tvalue_array = []
  smtrain = test_statistics(model,customlossMLc)
  i=0
  while i < Nsample:
    tvalue_array.append(smtrain.train_t(200000,customlossMLc))
    #tvalue_array.append(smtrain.t_value())
    i += 1
    
  return tvalue_array
t = time.process_time()
tarray2 = get_tsm(10)
print(tarray2)
print(time.process_time() - t)
t = time.process_time()
tarray2 = get_tsm(25)
print(tarray2)
print(time.process_time() - t)
tary200kR2cut=np.array([6.280324, 3.935751,3.002756, 3.428236, 6.76935, 7.586645, 21.619804, 4.3351784, 9.306698, 9.7299, 19.548218, 5.6026573,
                       14.930061, 9.929951, 4.20903, 2.857113, 8.879791, 18.498081, 10.508842, 5.499588, 5.8298645, 1.9355392,27.536953, 11.842777,
                        9.680717, 6.4893913, 6.888027,11.9573555, 7.1428375, 15.580963, 11.086571, 12.127045, 8.163109, 35.205555, 4.7552986, 4.13097, 
                        21.240387,12.448734, 8.343151, 2.999569, 11.258251, 10.019241, 4.69812, 14.007263, 0.9439087, -1.7368622, -0.45786285, 1.764965,
                        3.4663277, 2.0784607, 7.968746, 12.509335, 12.732555, 11.15712, 8.878368, 10.126976, 3.58535, 8.5193405, 4.748558, 12.377155, 4.23851, 2.246849])
tary200kR1cut=np.array([9.198353, 22.413506, 18.658892, 3.3231735, 12.6468315, 19.533606, 7.0150704, 7.973974, 15.004824,
                       1.8119359, 17.662802, 9.209027, 8.491543, 6.6254063, 7.361366, 20.885723, 2.8854532, 15.239798, 12.919951,
                       15.254736, 49.6819, 6.9491396, 9.390984, 8.531012, 5.195326, 7.3243885, 10.206685, 13.672317, 10.485491,
                       11.436906, 13.185726, 10.5280075, 13.9951, 10.375193, 23.16745, 7.866772, 9.366356, 15.060921, 24.468105,
                       2.1078587, 12.849054, 7.8117046, 1.1931639, 9.656768, 8.3884945, 10.9362335, 19.04181, 5.602028, 6.96894,
                       13.527407, 9.009592, 9.232324, 13.798014, 26.165163, 17.853573, 5.0918174, 0.0597561, 4.3211384, 
                       16.246407, 15.064257, 11.282628, 5.164405, 17.900526, 10.412801, 12.343819, 18.298119, 4.579193, 6.9842525,
                       2.6973543, 9.363689, 8.552435, 11.587778, 16.519943,16.553617, 12.130791, 4.799429, 5.813991, 25.142912, 
                       10.338545, 22.89791, 16.821117, 6.818872, 13.331888,14.899183, 6.5405235, 17.230661, 4.3786163, 16.537035, 7.003011, 
                       21.634886, 6.693706, 2.0629637, 17.045473,10.030758, 10.766983, 8.218758, 4.9844117, 9.099534, 12.161718, 18.165085, 
                       14.922871, 9.762325, 17.848711,11.368221, 13.5494585, 10.52153, 10.173314, 15.988689, 8.45729, 11.216583, 9.845869, 10.222246, 13.011869])
tary200kAll=np.array([9.198353, 22.413506, 18.658892, 3.3231735, 12.6468315, 19.533606, 7.0150704, 7.973974, 15.004824,
                       1.8119359, 17.662802, 9.209027, 8.491543, 6.6254063, 7.361366, 20.885723, 2.8854532, 15.239798, 12.919951,
                       15.254736, 49.6819, 6.9491396, 9.390984, 8.531012, 5.195326, 7.3243885, 10.206685, 13.672317, 10.485491,
                       11.436906, 13.185726, 10.5280075, 13.9951, 10.375193, 23.16745, 7.866772, 9.366356, 15.060921, 24.468105,
                       2.1078587, 12.849054, 7.8117046, 1.1931639, 9.656768, 8.3884945, 10.9362335, 19.04181, 5.602028, 6.96894,
                       13.527407, 9.009592, 9.232324, 13.798014, 26.165163, 17.853573, 5.0918174, 0.0597561, 4.3211384, 
                       16.246407, 15.064257, 11.282628, 5.164405, 17.900526, 10.412801, 12.343819, 18.298119, 4.579193, 6.9842525,
                       2.6973543, 9.363689, 8.552435, 11.587778, 16.519943,16.553617, 12.130791, 4.799429, 5.813991, 25.142912, 
                       10.338545, 22.89791, 16.821117, 6.818872, 13.331888,14.899183, 6.5405235, 17.230661, 4.3786163, 16.537035, 7.003011, 
                       21.634886, 6.693706, 2.0629637, 17.045473,10.030758, 10.766983, 8.218758, 4.9844117, 9.099534, 12.161718, 18.165085, 
                       14.922871, 9.762325, 17.848711,11.368221, 13.5494585, 10.52153, 10.173314, 15.988689, 8.45729, 11.216583, 9.845869, 10.222246, 13.011869,
                     6.280324, 3.935751,3.002756, 3.428236, 6.76935, 7.586645, 21.619804, 4.3351784, 9.306698, 9.7299, 19.548218, 5.6026573,
                       14.930061, 9.929951, 4.20903, 2.857113, 8.879791, 18.498081, 10.508842, 5.499588, 5.8298645, 1.9355392,27.536953, 11.842777,
                        9.680717, 6.4893913, 6.888027,11.9573555, 7.1428375, 15.580963, 11.086571, 12.127045, 8.163109, 35.205555, 4.7552986, 4.13097, 
                        21.240387,12.448734, 8.343151, 2.999569, 11.258251, 10.019241, 4.69812, 14.007263, 0.9439087, -1.7368622, -0.45786285, 1.764965,
                        3.4663277, 2.0784607, 7.968746, 12.509335, 12.732555, 11.15712, 8.878368, 10.126976, 3.58535, 8.5193405, 4.748558, 12.377155, 4.23851, 2.246849])
t = time.process_time()
tarray2 = get_tsm(10)
print(tarray2)
print(time.process_time() - t)
tarray50kR1=np.array([17.859928, 21.576794, 22.648262, 4.7346706, 0.041297942,9.307671, 7.986084, 2.9581623, 11.567207, 7.4215584,19.745886, 3.8535743, 7.994503, 11.461901, 19.316307,18.834127, 10.514814, 9.93902, 21.024137, 6.800843,13.896572, 7.2651596, 15.426065, 14.872274, 34.841877])
tarrayR1 = np.array([13.871891, 9.872658, 20.530746, 12.656269,12.109566, 16.218359, 5.0516987])
plt.hist(tary200kAll, bins=20,normed=True)
np.save('tarray1',tarray1)
print(os.getcwd())
print(os.listdir('/content'))
#from pydrive.auth import GoogleAuth
#from pydrive.drive import GoogleDrive
#from google.colab import auth
#from oauth2client.client import GoogleCredentials

# 1. Authenticate and create the PyDrive client.
auth.authenticate_user()
gauth = GoogleAuth()
gauth.credentials = GoogleCredentials.get_application_default()
drive = GoogleDrive(gauth)

# PyDrive reference:
# https://googledrive.github.io/PyDrive/docs/build/html/index.html

# 2. Create & upload a file 
uploaded = drive.CreateFile({'title': 'tarray_all.npy'})
uploaded.SetContentString('all t-value array')
uploaded.Upload()
print('Uploaded file with ID {}'.format(uploaded.get('id')))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15824953654289084, 'fraction_numerical': 0.12378518060783102, 'mean_word_length': 5.0631811989100814, 'pattern_counts': {'":': 3, '<': 75, '<?xml version=': 0, '>': 78, 'https://': 8, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 13, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '19', 'dir': 'jupyter-structured-clean-dedup', 'id': '584165', 'license': 'MIT', 'n_tokens_mistral': 17816, 'n_tokens_neox': 15102, 'n_words': 2268, 'path': 'NNratio_2D_BinnedRef.ipynb', 'repo_name': 'jameswang0619/AnomalyDetectNN', 'size': '35602'}