<commit_before>import os, sys

import argparse
import random

import time, pytz
from datetime import datetime, timezone

import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F

import h5py
from torch.utils.data import Dataset
from torch.utils.data import DataLoader

#from text_utils import TextEncoder   # This is my version

sys.path.append('orig/pytorch-openai-transformer-lm')
from model_pytorch import TransformerModel, load_openai_pretrained_model, DEFAULT_CONFIG
from model_pytorch import Conv1D
from opt import OpenAIAdam
from utils import ResultLogger

pretrained_model_path = os.path.join('.', 'orig', 'finetune-transformer-lm', 'model')

# So, let's read in a text file
relation_splits_path = os.path.join('.', 'orig', 'omerlevy-bidaf_no_answer-2e9868b224e4', 'relation_splits', )

#   840000  31087632 191519344 orig/omerlevy-bidaf_no_answer-2e9868b224e4/relation_splits/train.1
#      600     21854    136415 orig/omerlevy-bidaf_no_answer-2e9868b224e4/relation_splits/dev.1
#    12000    427110   2688895 orig/omerlevy-bidaf_no_answer-2e9868b224e4/relation_splits/test.1
#   852600  31536596 194344654 total

# TODO : Fn to get list of relationship_types and relationship_templates for each type


# Props to : https://github.com/rasbt/deep-learning-book/blob/master/code/model_zoo/pytorch_ipynb/custom-data-loader-csv.ipynb

class Hdf5Dataset(Dataset):
  """Custom Dataset for loading entries from HDF5 databases"""

  def __init__(self, h5_path, vocab_count, valid_indices=None): # transform=None, 
    self.h5f = h5py.File(h5_path, 'r')
    features = self.h5f['features']
    
    self.valid_indices=valid_indices          
    if valid_indices is None:
      self.num_entries = features.shape[0]
    else:
      self.num_entries = len(valid_indices)
    #self.transform = transform
    
    self.n_ctx = features.shape[1]

    self.postitional_encoder = np.arange(vocab_count, vocab_count + self.n_ctx)
      
  def __getitem__(self, index):
    if self.valid_indices is not None:  # find on-disk index
      index = self.valid_indices[index]
      
    features = self.h5f['features'][index]
    labels   = self.h5f['labels'][index].astype(np.int64)
    deps     = self.h5f['deps'][index].astype(np.int64)
    
    # Find the token_clf
    token_clf_pos = np.nonzero( features==token_clf )[-1].sum()  # This is zero if it is not found
    if token_clf_pos>=features.shape[0]-1:  
      #print("token_clf_pos right at end, index=", index, token_clf_pos, features.shape[0]-1)
      token_clf_pos=features.shape[0]-2 # Need to have this location, and the next one
    
    #if self.transform is not None:
    #  features = self.transform(features)
      
    #xmb[:, :, :, 1] = np.arange(n_vocab + n_special, n_vocab + n_special + n_ctx)
    #xmb[:, 1] = np.arange(n_vocab + n_special, n_vocab + n_special + n_ctx) # This is a single row, of batch=1

    #features_with_positions = np.stack( [ features, self.postitional_encoder ], axis=1 )
    features_with_positions = np.stack( [ features, self.postitional_encoder.copy() ], axis=1 )  # May be safer when multithreaded?
    #print(features.shape, features_with_positions.shape)  # (128,) (128, 2)

    unanswerable=False
    if 3 not in list(labels):  # There is no answer to this question
      unanswerable=True
    if 4 not in list(labels):  # There is no answer to this question
      unanswerable=True
    
    print(token_clf_pos, unanswerable)
    if unanswerable:
      if False:
        labels[0]=4 # end is before start
        labels[1]=3
      if True:
        labels[token_clf_pos  ] = 4 # end is before start
        labels[token_clf_pos+1] = 3
      
    # https://docs.scipy.org/doc/numpy-1.13.0/reference/generated/numpy.clip.html
    np.clip(deps, 0, self.n_ctx-1, out=deps)
    
    return features_with_positions, labels, deps

  def __len__(self):
    return self.num_entries

  def close(self):
    self.h5f.close()
    

class StepwiseClassifierModel(nn.Module):
    """ Transformer with stepwise classifier(s) """
    def __init__(self, cfg, n_classifier=None, one_hot=True, vocab_count=None, n_ctx=128): # 40990
        super(StepwiseClassifierModel, self).__init__()
        self.n_embd = cfg.n_embd
        self.n_ctx = n_ctx
        self.n_classifier = n_classifier
        
        self.transformer = TransformerModel(cfg, vocab=vocab_count+n_ctx, n_ctx=n_ctx)
        <commit_msg>Added code to deal with non-answers affection token_clf instead of 0,1 - remove print()<commit_after>import os, sys

import argparse
import random

import time, pytz
from datetime import datetime, timezone

import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F

import h5py
from torch.utils.data import Dataset
from torch.utils.data import DataLoader

#from text_utils import TextEncoder   # This is my version

sys.path.append('orig/pytorch-openai-transformer-lm')
from model_pytorch import TransformerModel, load_openai_pretrained_model, DEFAULT_CONFIG
from model_pytorch import Conv1D
from opt import OpenAIAdam
from utils import ResultLogger

pretrained_model_path = os.path.join('.', 'orig', 'finetune-transformer-lm', 'model')

# So, let's read in a text file
relation_splits_path = os.path.join('.', 'orig', 'omerlevy-bidaf_no_answer-2e9868b224e4', 'relation_splits', )

#   840000  31087632 191519344 orig/omerlevy-bidaf_no_answer-2e9868b224e4/relation_splits/train.1
#      600     21854    136415 orig/omerlevy-bidaf_no_answer-2e9868b224e4/relation_splits/dev.1
#    12000    427110   2688895 orig/omerlevy-bidaf_no_answer-2e9868b224e4/relation_splits/test.1
#   852600  31536596 194344654 total

# TODO : Fn to get list of relationship_types and relationship_templates for each type


# Props to : https://github.com/rasbt/deep-learning-book/blob/master/code/model_zoo/pytorch_ipynb/custom-data-loader-csv.ipynb

class Hdf5Dataset(Dataset):
  """Custom Dataset for loading entries from HDF5 databases"""

  def __init__(self, h5_path, vocab_count, valid_indices=None): # transform=None, 
    self.h5f = h5py.File(h5_path, 'r')
    features = self.h5f['features']
    
    self.valid_indices=valid_indices          
    if valid_indices is None:
      self.num_entries = features.shape[0]
    else:
      self.num_entries = len(valid_indices)
    #self.transform = transform
    
    self.n_ctx = features.shape[1]

    self.postitional_encoder = np.arange(vocab_count, vocab_count + self.n_ctx)
      
  def __getitem__(self, index):
    if self.valid_indices is not None:  # find on-disk index
      index = self.valid_indices[index]
      
    features = self.h5f['features'][index]
    labels   = self.h5f['labels'][index].astype(np.int64)
    deps     = self.h5f['deps'][index].astype(np.int64)
    
    # Find the token_clf
    token_clf_pos = np.nonzero( features==token_clf )[-1].sum()  # This is zero if it is not found
    if token_clf_pos>=features.shape[0]-1:  
      #print("token_clf_pos right at end, index=", index, token_clf_pos, features.shape[0]-1)
      token_clf_pos=features.shape[0]-2 # Need to have this location, and the next one
    
    #if self.transform is not None:
    #  features = self.transform(features)
      
    #xmb[:, :, :, 1] = np.arange(n_vocab + n_special, n_vocab + n_special + n_ctx)
    #xmb[:, 1] = np.arange(n_vocab + n_special, n_vocab + n_special + n_ctx) # This is a single row, of batch=1

    #features_with_positions = np.stack( [ features, self.postitional_encoder ], axis=1 )
    features_with_positions = np.stack( [ features, self.postitional_encoder.copy() ], axis=1 )  # May be safer when multithreaded?
    #print(features.shape, features_with_positions.shape)  # (128,) (128, 2)

    unanswerable=False
    if 3 not in list(labels):  # There is no answer to this question
      unanswerable=True
    if 4 not in list(labels):  # There is no answer to this question
      unanswerable=True
    
    #print(token_clf_pos, unanswerable)
    if unanswerable:
      if False:
        labels[0]=4 # end is before start
        labels[1]=3
      if True:
        labels[token_clf_pos  ] = 4 # end is before start
        labels[token_clf_pos+1] = 3
      
    # https://docs.scipy.org/doc/numpy-1.13.0/reference/generated/numpy.clip.html
    np.clip(deps, 0, self.n_ctx-1, out=deps)
    
    return features_with_positions, labels, deps

  def __len__(self):
    return self.num_entries

  def close(self):
    self.h5f.close()
    

class StepwiseClassifierModel(nn.Module):
    """ Transformer with stepwise classifier(s) """
    def __init__(self, cfg, n_classifier=None, one_hot=True, vocab_count=None, n_ctx=128): # 40990
        super(StepwiseClassifierModel, self).__init__()
        self.n_embd = cfg.n_embd
        self.n_ctx = n_ctx
        self.n_classifier = n_classifier
        
        self.transformer = TransformerModel(cfg, vocab=vocab_count+n_ctx, n_ctx=n_ctx)
        
{'fraction_non_alphanumeric': 0.10365373805508712, 'fraction_numerical': 0.04024732996065205, 'mean_word_length': 3.5480572597137012, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5109628', 'n_tokens_mistral': 3310, 'n_tokens_neox': 3121, 'n_words': 704}