parameters: (text_file,
               encoded_file=None,
               vocab_file=None,
               ignore_whitespace=True)
return_statement: return encoded_data, vocab_size
docstring: Convert text file to sequence of token IDs.
  
      Tokenization is performed with BERT tokenizer.
  
      Args:
          text_file (str): Text file to be encoded.
          encoded_file (str, optional): If provided, path where the
             encoded data will be saved as an .npz file. The sequence of
             token IDs is saved as 'encoded_data' and the vocabulary
             size is saved as 'vocab_size'.
          vocab_file (str, optional): If provided, path where the
              vocabulary will be saved as a text file.
          ignore_whitespace (bool, optional): Whether to ignore text
              lines that are purely made of whitespace (default: True).
  
      Returns:
          array of int: Sequence of token IDs.
          int: Number of tokens in vocabulary.
docstring_summary: Convert text file to sequence of token IDs.
docstring_tokens: Convert text file to sequence of token IDs .
function: def tokenize(text_file,
               encoded_file=None,
               vocab_file=None,
               ignore_whitespace=True):
      """Convert text file to sequence of token IDs.
  
      Tokenization is performed with BERT tokenizer.
  
      Args:
          text_file (str): Text file to be encoded.
          encoded_file (str, optional): If provided, path where the
             encoded data will be saved as an .npz file. The sequence of
             token IDs is saved as 'encoded_data' and the vocabulary
             size is saved as 'vocab_size'.
          vocab_file (str, optional): If provided, path where the
              vocabulary will be saved as a text file.
          ignore_whitespace (bool, optional): Whether to ignore text
              lines that are purely made of whitespace (default: True).
  
      Returns:
          array of int: Sequence of token IDs.
          int: Number of tokens in vocabulary.
  
      """
  
      # Get BERT tokenizer from Transformers
      import transformers
      tokenizer = transformers.BertTokenizer.from_pretrained('bert-base-uncased')
      vocab_size = tokenizer.vocab_size
      if vocab_file:
          tokenizer.save_vocabulary(vocab_file)
  
      # Apply tokenizer to text file
      encoded_data = array.array('l')
      with open(text_file) as f:
          for line in f:
              if ignore_whitespace and line.isspace():
                  continue
              encoded_data.extend(tokenizer.encode(line))
      if encoded_file:
          np.savez_compressed(encoded_file,
                              encoded_data=encoded_data,
                              vocab_size=vocab_size)
      return encoded_data, vocab_size
function_tokens: def tokenize ( text_file , encoded_file = None , vocab_file = None , ignore_whitespace = True ) : # Get BERT tokenizer from Transformers import transformers tokenizer = transformers . BertTokenizer . from_pretrained ( 'bert-base-uncased' ) vocab_size = tokenizer . vocab_size if vocab_file : tokenizer . save_vocabulary ( vocab_file ) # Apply tokenizer to text file encoded_data = array . array ( 'l' ) with open ( text_file ) as f : for line in f : if ignore_whitespace and line . isspace ( ) : continue encoded_data . extend ( tokenizer . encode ( line ) ) if encoded_file : np . savez_compressed ( encoded_file , encoded_data = encoded_data , vocab_size = vocab_size ) return encoded_data , vocab_size