<jupyter_start><jupyter_text>Graph Neural Network<jupyter_code>class GCN(torch.nn.Module):
    def __init__(self, c_in, c_out):
        super().__init__()
        self.conv1 = GCNConv(c_in, 32)
        self.conv2 = GCNConv(32, c_out)

    def forward(self, data):
        x, edge_index = data.x, data.edge_index

        x = self.conv1(x, edge_index)
        x = F.relu(x)
        x = F.dropout(x, training=self.training)
        x = self.conv2(x, edge_index)

        return x
def train_graph_classifier():
  batch_size = df_train.shape[0] // 90
  num_iterations = df_train.shape[0] // batch_size

  model = GCN(c_in=X_train.shape[1], c_out=1)
        
  model = model.to(device)
  optimizer = torch.optim.Adam(model.parameters(), lr=0.01, weight_decay=0.0005)
  criterion = nn.L1Loss()
  criterion = criterion.to(device)

  model.train()
  for epoch in range(200):
    running_loss = 0.

    for iteration in range(num_iterations):
      dict_author = dict(zip(df_train.author[iteration*batch_size:(iteration+1)*batch_size], 
                             df_train.index[:batch_size]))
      G_train = G.subgraph(df_train.author.to_list()[iteration*batch_size:(iteration+1)*batch_size])

      G_train = G_train.to_directed() if not nx.is_directed(G_train) else G_train
      edges = list(itertools.chain(*list(G_train.edges)))
      edge_index = torch.tensor(np.array(list(map(lambda x: dict_author[x], edges)))).reshape(2, -1)

      train_data = Data(x=torch.tensor(X_train.astype(np.float32)[iteration*batch_size:(iteration+1)*batch_size]), 
                        edge_index=edge_index.long(), 
                        y=torch.tensor(y_train[iteration*batch_size:(iteration+1)*batch_size, :]))
      data = train_data.to(device)

      optimizer.zero_grad()
      out = model(data)
      loss = criterion(out, data.y.float())
      loss.backward()
      optimizer.step()

      running_loss += loss.item()
    if epoch % 10 == 9:
        print('Epoch %d, loss: %.3f' %
              (epoch + 1, running_loss / (10 * num_iterations)))
        running_loss = 0.0

  print('Finished Training')

  return model
model = train_graph_classifier()
dict_author_test = dict(zip(df_test.author, df_test.index))
G_test = G.subgraph(df_test.author.to_list())

G_test = G_test.to_directed() if not nx.is_directed(G_test) else G_test
edges = list(itertools.chain(*list(G_test.edges)))
edge_index = torch.tensor(np.array(list(map(lambda x: dict_author_test[x], edges)))).reshape(2, -1)

test_data = Data(x=torch.tensor(X_test.astype(np.float32)), edge_index=edge_index.long())
torch.save(test_data, DATA_DIR  + 'graph_data_test.pt')
test_data
model.eval()
test_data = test_data.to(device)
pred = model(test_data)
df_test['hindex'] = pd.Series(np.round_(pred.cpu().detach().numpy().squeeze(), decimals=3))
df_test.hindex = df_test.hindex.apply(lambda x: np.round(x))
df_test
df_test.loc[:,["author","hindex"]].to_csv(DATA_DIR + 'inf554-2021/submission_6.csv', index=False)<jupyter_output><empty_output><jupyter_text>Multilayer Perceptron (MLP)<jupyter_code>y_train = np.array(df_train.hindex.values).reshape(-1,1)
train_ratio = 0.9

n_train_examples = int(len(X_train) * train_ratio)
n_valid_examples = len(X_train) - n_train_examples
class MyDataset(Dataset):
    def __init__(self, data, labels):
        self.data = torch.tensor(data).to(torch.float32)
        self.labels = torch.tensor(labels).reshape(-1, 1).to(torch.float32)

    def __len__(self):
        return len(self.data)

    def __getitem__(self, idx):
        return self.data[idx], self.labels[idx]
#train_data = MyDataset(np.delete(X_train, 5, 1), y_train)
train_data = MyDataset(X_train, y_train)

train_data, valid_data = data.random_split(train_data, 
                                           [n_train_examples, n_valid_examples])
print(f'Number of training examples: {len(train_data)}')
print(f'Number of validation examples: {len(valid_data)}')
batch_size = 32

train_loader = data.DataLoader(train_data, shuffle=True, batch_size=batch_size)

valid_loader = data.DataLoader(valid_data, batch_size=batch_size)

test_loader = data.DataLoader(X_test, batch_size=batch_size)
class MLP(nn.Module):
    def __init__(self, input_dim, output_dim):
        super().__init__()
                
        self.input_fc = nn.Linear(input_dim, 64)
        #self.batch_norm1 = nn.BatchNorm1d(32)
        #self.hidden_fc1 = nn.Linear(32, 64)
        #self.batch_norm2 = nn.BatchNorm1d(64)
        #self.hidden_fc2 = nn.Linear(128, 64)
        #self.hidden_fc3 = nn.Linear(64, 32)
        self.output_fc = nn.Linear(64, output_dim)

        self.dropout = nn.Dropout(.1)
        
    def forward(self, x):

        batch_size = x.shape[0]

        x = x.view(batch_size, -1)

        x = self.dropout(x)

        #h_1 = F.relu(self.batch_norm1(self.input_fc(x)))
        h_1 = F.relu(self.input_fc(x))
        #h_1 = self.dropout(h_1)

        #h_2 = F.relu(self.batch_norm2(self.hidden_fc1(h_1)))
        #h_2 = F.relu(self.hidden_fc1(h_1))
        #h_2 = self.dropout(h_2)
        #h_3 = F.relu(self.hidden_fc2(h_2))
        #h_4 = F.relu(self.hidden_fc3(h_3))
        y_pred = self.output_fc(h_1)
        
        return y_pred
input_dim = X_train.shape[1]
#input_dim = np.delete(X_train, 5, 1).shape[1]
output_dim = 1

model = MLP(input_dim, output_dim)
learning_rate = 0.01

optimizer = optim.Adam(model.parameters(), lr=learning_rate)
criterion = nn.L1Loss()

model = model.to(device)
criterion = criterion.to(device)
def train(model, dataloader, optimizer, criterion, device):
    
    epoch_loss = 0.
    epoch_error = 0.
    
    model.train()
    
    for x, y in dataloader:
        
        x = x.to(device)
        y = y.to(device)
        
        optimizer.zero_grad()
                
        y_pred = model(x)
        
        loss = criterion(y_pred, y)
        
        loss.backward()
        
        optimizer.step()
        
        epoch_loss += loss.item()
        epoch_error += F.mse_loss(y_pred, y).item()
        
    return epoch_loss / len(dataloader), epoch_error / len(dataloader)
def evaluate(model, dataloader, criterion, device):
    
    epoch_loss = 0.
    epoch_error = 0.
    
    model.eval()
    
    with torch.no_grad():
        
        for x, y in dataloader:

            x = x.to(device)
            y = y.to(device)

            y_pred = model(x)

            loss = criterion(y_pred, y)

            epoch_loss += loss.item()
            epoch_error += F.mse_loss(y_pred, y).item()
        
    return epoch_loss / len(dataloader), epoch_error / len(dataloader)
num_epochs = 300

best_valid_loss = float('inf')

train_losses = []
valid_losses = []

train_errors = []
valid_errors = []

for epoch in range(num_epochs):
    
    train_loss, train_error = train(model, train_loader, optimizer, criterion, device)
    valid_loss, valid_error = evaluate(model, valid_loader, criterion, device)

    train_losses.append(train_loss)
    valid_losses.append(valid_loss)

    train_errors.append(train_error)
    valid_errors.append(valid_error)
    
    if valid_loss < best_valid_loss:
        best_valid_loss = valid_loss
        torch.save(model.state_dict(), DATA_DIR + 'mlp-model_' + str(learning_rate) + '.pt')

    if epoch % 10 == 0:
      print(f'Epoch: {epoch:02}')
      print(f'\tTrain Loss: {train_loss:.3f}')
      print(f'\t Val. Loss: {valid_loss:.3f} ')
      print()
      print(f'\tTrain MSE: {train_error:.3f}')
      print(f'\t Val. MSE: {valid_error:.3f} ')
import matplotlib.pyplot as plt

plt.plot(np.arange(num_epochs), train_losses, label='Train MSE')
plt.plot(np.arange(num_epochs), valid_losses, label='Validation MSE')
plt.legend(loc='upper right')
model.load_state_dict(torch.load(DATA_DIR + 'mlp-model_' + str(learning_rate) + '.pt'))

y_pred = model(torch.tensor(X_test).to(torch.float32).to(device))
df_test['hindex'] = pd.Series(np.round_(y_pred.cpu().detach().numpy().squeeze(), decimals=3))


df_test.loc[:,["author","hindex"]].to_csv(DATA_DIR + 'inf554-2021/submission_11.csv', index=False)<jupyter_output><empty_output><jupyter_text>Node2Vec Embeddings<jupyter_code>f = open(DATA_DIR + 'coauthorship.emb', 'r')

print(f.readline())
embeddings = dict()
for x in f:
  emb = x.split(' ')
  embeddings[int(emb[0])] = list(map(float, emb[1:]))
emb_train = [embeddings[x] for x in df_train.author.values]
emb_train = np.array(emb_train)
y_train = df_train.hindex.values.reshape(-1, 1)

emb_test = [embeddings[x] for x in df_test.author.values]
emb_test = np.array(emb_test)
mean_train = emb_train.mean(axis=0)
std_train = emb_train.std(axis=0)

emb_train = (emb_train - mean_train) / std_train
emb_test = (emb_test - mean_train) / std_train
train_ratio = 0.9

n_train_examples = int(len(emb_train) * train_ratio)
n_valid_examples = len(emb_train) - n_train_examples

train_data = MyDataset(emb_train, y_train)

train_data, valid_data = data.random_split(train_data, 
                                           [n_train_examples, n_valid_examples])
batch_size = 64

train_loader = data.DataLoader(train_data, shuffle=True, batch_size=batch_size)

valid_loader = data.DataLoader(valid_data, batch_size=batch_size)

test_loader = data.DataLoader(emb_test, batch_size=batch_size)
input_dim = emb_train.shape[1]
output_dim = 1

model = MLP(input_dim, output_dim)
learning_rate = 0.01

optimizer = optim.Adam(model.parameters(), lr=learning_rate)
criterion = nn.L1Loss()

model = model.to(device)
criterion = criterion.to(device)

num_epochs = 300

best_valid_loss = float('inf')

train_losses = []
valid_losses = []

train_errors = []
valid_errors = []

for epoch in range(num_epochs):
    
    train_loss, train_error = train(model, train_loader, optimizer, criterion, device)
    valid_loss, valid_error = evaluate(model, valid_loader, criterion, device)

    train_losses.append(train_loss)
    valid_losses.append(valid_loss)

    train_errors.append(train_error)
    valid_errors.append(valid_error)
    
    if valid_loss < best_valid_loss:
        best_valid_loss = valid_loss
        torch.save(model.state_dict(), DATA_DIR + 'mlp-model_' + str(learning_rate) + '.pt')

    if epoch % 10 == 0:
      print(f'Epoch: {epoch:02}')
      print(f'\tTrain Loss: {train_loss:.3f}')
      print(f'\t Val. Loss: {valid_loss:.3f} ')
      print()
      print(f'\tTrain MSE: {train_error:.3f}')
      print(f'\t Val. MSE: {valid_error:.3f} ')
plt.plot(np.arange(len(train_losses)), train_losses, label='Train MSE')
plt.plot(np.arange(len(valid_losses)), valid_losses, label='Validation MSE')
plt.legend(loc='upper right')
model.load_state_dict(torch.load(DATA_DIR + 'mlp-model_' + str(learning_rate) + '.pt'))

y_pred = model(torch.tensor(emb_test).to(torch.float32).to(device))
df_test['hindex'] = pd.Series(np.round_(y_pred.cpu().detach().numpy().squeeze(), decimals=3))


df_test.loc[:,["author","hindex"]].to_csv(DATA_DIR + 'inf554-2021/submission_12.csv', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14069799395438307, 'fraction_numerical': 0.018411651552624347, 'mean_word_length': 3.5415973377703827, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '93658', 'license': 'MIT', 'n_tokens_mistral': 4211, 'n_tokens_neox': 4041, 'n_words': 649, 'path': 'Graph.ipynb', 'repo_name': 'oublalkhalid/artificial-intelligence-to-a-real-world', 'size': '10917'}