<jupyter_start><jupyter_text>Regression  Train model<jupyter_code>import torch
import torch.nn.functional as F
import matplotlib.pyplot as plt
import numpy as np
import torch.utils.data as Data
import itertools
import pickle

fo = open("log.txt", "w")

tasktype='regression'# classification
torch.manual_seed(1)    # reproducible
p=10
x=torch.rand(20000,p)*2-1
X=x.numpy()

### you may select the following groundtruth as you like
# multi 1
# y = X[:,0]*X[:,1] + X[:,2]*X[:,3]*X[:,4]+X[:,5]*X[:,6]*X[:,7]+ X[:,8]*X[:,9] 
# multi 2
# y = np.exp(np.abs(X[:,0]-X[:,1]))*X[:,9]+np.abs(X[:,2]*X[:,3])-(X[:,4]**2)*np.abs(X[:,5])+(X[:,6]*X[:,7])**2+X[:,8]
# multi 3
# y = X[:,0]*X[:,1] + np.abs(X[:,1]+X[:,2]*X[:,3])+X[:,4]*X[:,5]-X[:,5]**2*X[:,6]-np.exp(X[:,7]+ X[:,8]*X[:,9]) 
# F8 (example of node overlapping)
y=X[:,0]*X[:,1]+2**(X[:,2]+X[:,4]+X[:,5])+2**(X[:,2]+X[:,3]+X[:,4]+X[:,6])+np.sin(X[:,6]*np.sin(X[:,7]+X[:,8]))+np.arccos(0.9*X[:,9])

y=torch.from_numpy(y)
if tasktype is 'regression':
    print('task:',tasktype)
    loss_func = torch.nn.MSELoss()  # this is for regression mean squared loss


y=y.reshape(len(y),1)
x_tr,y_tr,x_te,y_te=x[:10000,:],y[:10000,:],x[10000:,:],y[10000:,:]
torch_dataset = Data.TensorDataset(x, y)
BATCH_SIZE=400
loader = Data.DataLoader(
    dataset=torch_dataset,      
    batch_size=BATCH_SIZE,      
    shuffle=True,               
    num_workers=2,              
)


class Net(torch.nn.Module):
    def __init__(self, n_feature, n_hidden, n_output):
        super(Net, self).__init__()
        self.hidden1 = torch.nn.Linear(n_feature, n_hidden)   # hidden layer
        self.hidden2 = torch.nn.Linear(n_hidden, n_hidden)
        self.hidden3 = torch.nn.Linear(n_hidden, n_hidden)
        self.predict = torch.nn.Linear(n_hidden, n_output)   # output layer

    def forward(self, x):
        x = F.relu(self.hidden1(x))      # activation function for hidden layer sigmoid
        x = F.relu(self.hidden2(x))
        x = F.relu(self.hidden3(x))
        x = self.predict(x)             # linear output
        return x

net = Net(n_feature=p, n_hidden=400, n_output=1)     # define the network #400 1200
print(net)  # net architecture

optimizer= torch.optim.Adam(net.parameters(), betas=(0.9, 0.99))
for t in range(1000):
    prediction = net(x_tr)     # input x and predict based on x
    loss = loss_func(prediction, y_tr)     # must be (1. nn output, 2. target)

    optimizer.zero_grad()   # clear gradients for next train
    loss.backward()         # backpropagation, compute gradients
    optimizer.step()        # apply gradients
    #print(t,'loss:',loss)

    if t % 10 == 0:
        teloss = loss_func(net(x_te), y_te)
        print(t,'training loss:',loss,'test loss:',teloss)
        #torch.save(net.state_dict(), 'models/net_params_'+str(t)+'.pkl') 
        string='epoch:'+str(t)+'training loss:'+str(loss)+'test loss:'+str(teloss)+'\n'
        fo.write(string)

fo.close()<jupyter_output><>:28: SyntaxWarning: "is" with a literal. Did you mean "=="?
<>:28: SyntaxWarning: "is" with a literal. Did you mean "=="?
<ipython-input-1-36e87a054ba9>:28: SyntaxWarning: "is" with a literal. Did you mean "=="?
  if tasktype is 'regression':<jupyter_text>Detect interactions analytically. (Hessian matrix is a zero matrix)<jupyter_code>import torch
from torch import Tensor
from torch.autograd import Variable
from torch.autograd import grad
from torch import nn
import time
st=time.time()
torch.manual_seed(623)
cnt=0
aa=0
Hessian=torch.zeros(p,p)
grad1=torch.zeros(1,p)

for j in range(100):
        xeval = Variable(x_tr[j,:], requires_grad=True)
        xeval=xeval.reshape(1,p)
        f = net(xeval)
        x_1grad, = grad(f, xeval, create_graph=True)
        grad1=grad1+(x_1grad)**2
        x_2grad0, = grad(x_1grad[0,0], xeval, create_graph=True)

        for i in range(p-1):
            x_2grad1, = grad(x_1grad[0,i+1], xeval, create_graph=True)
            x_2grad0 = torch.cat((x_2grad0, x_2grad1), dim=0)
            cnt=cnt+1
        Hessian=Hessian+torch.abs(x_2grad0) 
        aa=Hessian/cnt

print('detection time:',time.time()-st)
plt.matshow(np.abs(aa.detach().numpy()))<jupyter_output>detection time: 51.777947425842285<jupyter_text>Detect interactions Numerically (brute force, pull each arm 100 times)<jupyter_code>def one_hot(i,p):
    '''
    generate a p-dimensional one_hot vector, i-th element is 1
    '''
    
    batch_size=1
    # Dummy input that HAS to be 2D for the scatter (you can use view(-1,1) if needed)
    y = torch.LongTensor([[i]])
    # One hot encoding buffer that you create out of the loop and just keep reusing
    y_onehot = torch.FloatTensor(batch_size, p)
    #print(y_onehot)
    # In your for loop
    y_onehot.zero_()
    y_onehot.scatter_(1, y, 1)
    return y_onehot
import torch
from torch import Tensor
from torch.autograd import Variable
from torch.autograd import grad
from torch import nn
import time
st=time.time()
torch.manual_seed(623)
cnt=0
aa=0
Hessian=torch.zeros(p,p)
Firstorder=torch.zeros(1,p)
Firstordertemp=torch.zeros(1,p)
Hessiantemp=torch.zeros(p,p)
grad1=torch.zeros(1,p)
delta=1 # h=k=1 

for j in range(100):

        xeval = Variable(x_tr[j,:], requires_grad=True)
        xeval=xeval.reshape(1,p)

        
        for i in range(p):                        
            for k in range(i):
                f0 = net(xeval-one_hot(i,p)*delta-one_hot(k,p)*delta)
                fi =net(xeval+one_hot(i,p)*delta-one_hot(k,p)*delta)
                fik =net(xeval+one_hot(i,p)*delta+one_hot(k,p)*delta)
                fk=net(xeval+one_hot(k,p)*delta-one_hot(i,p)*delta)
                
                Hessiantemp[i,k]=((fik-fi-fk+f0)/delta**2/4)**2
                Hessiantemp[k,i]=Hessiantemp[i,k]

        cnt=cnt+1
        Hessian=Hessian+torch.abs(Hessiantemp) 
        aa=Hessian/cnt

print(time.time()-st)
plt.matshow(np.abs(aa.detach().numpy()))<jupyter_output>6.42970871925354<jupyter_text>Graph and spectral clustering<jupyter_code>import networkx as nx
score=np.abs(aa.detach().numpy())
G=nx.from_numpy_matrix(score)
color=[]
for i in G.edges:
    color.append(score[i[0],i[1]])
plt.figure(1)
nx.draw(G, 
        with_labels=True,
        node_size=400, 
        edge_color=color,        
        width=3, 
        cmap=plt.cm.Dark2, 
        edge_cmap=plt.cm.Blues
       )  
plt.show()


#np.max(score)*0.1
#score[score<np.max(score)*0.1]=0
adMatrix=score


from sklearn.cluster import KMeans
p=10
k=3

#score[score<np.max(score)*0.1]=0
adMatrix=score
aaaa=np.any(adMatrix,axis=1)
def calLaplacianMatrix(adjacentMatrix):

    # compute the Degree Matrix: D=sum(A)
    degreeMatrix = np.sum(adjacentMatrix, axis=1)

    # compute the Laplacian Matrix: L=D-A
    laplacianMatrix = np.diag(degreeMatrix) - adjacentMatrix

    # normailze
    # D^(-1/2) L D^(-1/2)
    sqrtDegreeMatrix = np.diag(1.0 / (degreeMatrix ** (0.5)))
    return np.dot(np.dot(sqrtDegreeMatrix, laplacianMatrix), sqrtDegreeMatrix)

def spKmeans(H,k):
    sp_kmeans = KMeans(n_clusters=k).fit(H)
    return sp_kmeans.labels_

Laplacian=calLaplacianMatrix(adMatrix[aaaa][:,aaaa])
x, V = np.linalg.eig(Laplacian)

x = zip(x, range(len(x)))
x = sorted(x, key=lambda x:x[0])

H = np.vstack([V[:,i] for (v, i) in x[:k]]).T

label111=spKmeans(H,k)

nodecolormap=np.ones(p)
nodecolormap[np.where(aaaa==False)]=range(k,k+len(np.where(aaaa==False)[0])) #nodes with degree=1
nodecolormap[np.where(aaaa==True)]=label111
#
plt.figure(2)
nx.draw(G, 
        with_labels=True,
        node_size=400, 
        node_color=nodecolormap,
        edge_color=color,        
        width=3, 
        cmap=plt.cm.Dark2, 
        edge_cmap=plt.cm.Blues
       )  
plt.savefig('figure_name.pdf',bbox_inches='tight',dpi='figure',pad_inches=0.1) # figure_name could be multi_1,2,3

plt.show()<jupyter_output><empty_output><jupyter_text>Detecting 2-way/3-way/4-way interactions via UCB<jupyter_code>adMatrix # cut off
def cutoff(adj,k):
    adjflatten=adj.flatten()
    return adj>=adjflatten[np.argsort(adjflatten)[-k*2]]
Gcut=nx.from_numpy_matrix(cutoff(adMatrix,20))
pos=nx.shell_layout(Gcut)
nx.draw(Gcut,pos,with_labels=True,node_size=800,node_color='g',alpha=0.8,font_size=20)
enum=nx.enumerate_all_cliques(Gcut)

Tway_candid=[]  # save all 3-cliques
Fway_candid=[]  # save all 4-cliques
for i in enum:
    print(i)
    if len(i)==3:
        Tway_candid.append(i)
    if len(i)==4:
        Fway_candid.append(i)
print('All 3-cliques: ', Tway_candid) 
print('All 4-cliques: ', Fway_candid)
from UCBtools import *
detect_Hessian_UCB(net,x_tr,20) # detected top 20 pairwise (2-way) interactions sorted in order of detection
# Then we check if those candidates are 3-way interactions
detect_3rd_UCB(net,x_tr,Tway_candid,6)
Fway_candid<jupyter_output><empty_output><jupyter_text>You may see {2,3,4,6} is the only 4-clique such that {2,3,6},{2,4,6},{3,4,6},{2,3,4} all exist.<jupyter_code>detect_4th_UCB(net,x_tr,Fway_candid,4)<jupyter_output>start dectecting
Initialization done! initial try3times
chosen arm: 3 strength: 0.7589151919118982 iteration: 164
chosen arm: 2 strength: 0.7992090514843585 iteration: 205
chosen arm: 5 strength: 0.4865295314873218 iteration: 300
chosen arm: 4 strength: 0.06410021153935304 iteration: 396
time: 3.0254740715026855
{'fraction_non_alphanumeric': 0.161454388053241, 'fraction_numerical': 0.04707282761605887, 'mean_word_length': 3.897721250662427, 'pattern_counts': {'":': 0, '<': 26, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '576671', 'license': 'Apache-2.0', 'n_tokens_mistral': 3908, 'n_tokens_neox': 3691, 'n_words': 662, 'path': 'demo_pytorch/1-demo.ipynb', 'repo_name': 'zhangtj1996/ParaACE', 'size': '9241'}