<jupyter_start><jupyter_text>Copyright 2021 Google LLC<jupyter_code>#@title Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.<jupyter_output><empty_output><jupyter_text>Graph regularization for Twitter rumour veracity classification using natural graphs  OverviewThis tutorial uses the PHEME dataset for veracity classification of Twitter rumours, a binary classification task.Tweet texts are used as input for computing embeddings, e.g. with ALBERT. Those representations are then used as features for the baseline MLP classification model, as well as for the graph regularized version, which uses the structure defined by the tweet replies as a natural graph.  Install and import dependencies<jupyter_code>!pip install --quiet neural-structured-learning
!pip install --quiet transformers
!pip install --quiet tokenizers
!pip install --quiet sentencepiece
import os
import json
import random
import pprint

import numpy as np

import neural_structured_learning as nsl

import tensorflow as tf
import tensorflow_datasets as tfds

import sentencepiece

from tokenizers import BertWordPieceTokenizer

from transformers import BertConfig
from transformers import BertTokenizer, TFBertModel
from transformers import XLNetTokenizer, TFXLNetModel
from transformers import RobertaTokenizer, TFRobertaModel
from transformers import AlbertTokenizer, TFAlbertModel
from transformers import T5Tokenizer, TFT5Model
from transformers import ElectraTokenizer, TFElectraModel

# Resets notebook state
tf.keras.backend.clear_session()

print("Version: ", tf.__version__)
print("Eager mode: ", tf.executing_eagerly())
print(
    "GPU is",
    "available" if tf.config.list_physical_devices("GPU") else "NOT AVAILABLE")<jupyter_output><empty_output><jupyter_text>Dataset descriptionThe PHEME dataset includes the following, which are used in this tutorial. * Twitter rumours associated with diverse news events* Source tweets, each of which is accompanied by reaction tweets* The reply structure for each source and corresponding reaction tweets* Veracity label annotations (by professional journalists) for each source tweet.  Download and extract the dataset AttributionWe use the [PHEME dataset for Rumour Detection and Veracity Classification](https://figshare.com/articles/dataset/PHEME_dataset_for_Rumour_Detection_and_Veracity_Classification/6392078)created by <NAME>, <NAME>, and <NAME>.This data set is licensed under [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/).The underlying data is used without modifications as labels or for computing model features.<jupyter_code>!mkdir /tmp/PHEME
!wget --quiet -P /tmp/PHEME https://ndownloader.figshare.com/files/11767817
!ls -l /tmp/PHEME
!tar -C /tmp/PHEME -xvzf /tmp/PHEME/11767817
!ls /tmp/PHEME/all-rnr-annotated-threads/<jupyter_output><empty_output><jupyter_text>Convert rumor annotations to labelsCode available in `convert_veracity_annotations.py` together with the [PHEME dataset](https://figshare.com/articles/PHEME_dataset_for_Rumour_Detection_and_Veracity_Classification/6392078)<jupyter_code>"""
Python 3 function to convert rumour annotations into True, False, Unverified
"""

def convert_annotations(annotation, string = True):
    if 'misinformation' in annotation.keys() and 'true'in annotation.keys():
        if int(annotation['misinformation'])==0 and int(annotation['true'])==0:
            if string:
                label = "unverified"
            else:
                label = 2
        elif int(annotation['misinformation'])==0 and int(annotation['true'])==1 :
            if string:
                label = "true"
            else:
                label = 1
        elif int(annotation['misinformation'])==1 and int(annotation['true'])==0 :
            if string:
                label = "false"
            else:
                label = 0
        elif int(annotation['misinformation'])==1 and int(annotation['true'])==1:
            print ("OMG! They both are 1!")
            print(annotation['misinformation'])
            print(annotation['true'])
            label = None
            
    elif 'misinformation' in annotation.keys() and 'true' not in annotation.keys():
        # all instances have misinfo label but don't have true label
        if int(annotation['misinformation'])==0:
            if string:
                label = "unverified"
            else:
                label = 2
        elif int(annotation['misinformation'])==1:
            if string:
                label = "false"
            else:
                label = 0
                
    elif 'true' in annotation.keys() and 'misinformation' not in annotation.keys():
        print ('Has true not misinformation')
        label = None
    else:
        print('No annotations')
        label = None
           
    return label<jupyter_output><empty_output><jupyter_text>Load annotated rumours from dataset<jupyter_code>def load_pheme_data(parent_dir):
  """ Loads and returns tweets and their replies from input `parent_dir`.
  
  Args:
    parent_dir: A string with full path to directory where the data lies in.

  Returns:
    A tuple T (annotation, structure, source_tweets, reactions) such that:
      Each of them is a dictionary directly read from the underlying JSON
      structure provided in the PHEME dataset.
  """
  with open(parent_dir + '/annotation.json') as f:
    annotation = json.load(f)
  with open(parent_dir + '/structure.json') as f:
    structure = json.load(f)
  source_tweets = {}
  for f in os.scandir(parent_dir + '/source-tweets'):
    if f.name[0] != '.':
      with open(f.path) as json_file:
        source_tweets[f.name.split('.json')[0]] = json.load(json_file)
  reactions = {}
  for f in os.scandir(parent_dir + '/reactions'):
    if f.name[0] != '.':
      with open(f.path) as json_file:
        reactions[f.name.split('.json')[0]] = json.load(json_file)
  return annotation, structure, source_tweets, reactions
def load_labels_and_texts(topics):
  """Reads verified rumour tweets, replies and labels for input `topics`.
  Non-rumours or unverified tweet threads aren't included in returned dataset.

  Args:
    topics: A List of strings, each containing the full path to a topic
    to be read.

  Returns:
    A List of dictionaries such that each entry E contains:
      E['label']: (integer) the rumour veracity annotation.
      E['source_text']: (string) the source tweet text.
      E['reactions']: (List of strings) the texts from the tweet replies.
  """
  labels_and_texts = []
  for t in topics:
    rumours = [
      f.path for f in os.scandir(t + '/rumours') if f.is_dir()
    ]
    for r in rumours:
      annotation, structure, source_tweets, reactions = load_pheme_data(r)
      for source_tweet in source_tweets.values():
        labels_and_texts.append({
            'label' : convert_annotations(annotation, string = False),
            'source_text' : source_tweet['text'],
            'reactions' : [reaction['text'] for reaction in reactions.values()]
        })

  print('Read', len(labels_and_texts), 'annotated rumour tweet threads')

  verified_labels_and_texts = list(
      filter(lambda d : d['label'] != 2, labels_and_texts))

  print('Returning', len(verified_labels_and_texts),
        'verified rumour tweet threads')
  
  return verified_labels_and_texts
topics = [
  f.path
  for f in os.scandir('/tmp/PHEME/all-rnr-annotated-threads/')
  if f.is_dir()
]
topics<jupyter_output><empty_output><jupyter_text>Load tokenizers, model architectures and pre-trained weights<jupyter_code>configuration = BertConfig()

"""
Uncomment any tokenizer-model pair for experimentation with other models.
"""

TOKENIZERS = {
    # 'bert_base' : BertTokenizer.from_pretrained('bert-base-uncased'),
    # 'bert_large' : BertTokenizer.from_pretrained('bert-large-uncased'),
    # 'xlnet_base' : XLNetTokenizer.from_pretrained('xlnet-base-cased'),
    # 'xlnet_large' : XLNetTokenizer.from_pretrained('xlnet-large-cased'),
    # 'roberta_base' : RobertaTokenizer.from_pretrained('roberta-base'),
    # 'roberta_large' : RobertaTokenizer.from_pretrained('roberta-large'),
    'albert_base' : AlbertTokenizer.from_pretrained('albert-base-v2'),
    # 'albert_large' : AlbertTokenizer.from_pretrained('albert-large-v2'),
    # 'albert_xlarge' : AlbertTokenizer.from_pretrained('albert-xlarge-v2'),
    # 'albert_xxlarge' : AlbertTokenizer.from_pretrained('albert-xxlarge-v2'),
    # 't5_small' : T5Tokenizer.from_pretrained('t5-small'),
    # 't5_base' : T5Tokenizer.from_pretrained('t5-base'),
    # 't5_large' : T5Tokenizer.from_pretrained('t5-large'),
    # 'electra_small' : ElectraTokenizer.from_pretrained('google/electra-small-discriminator'),
    # 'electra_large' : ElectraTokenizer.from_pretrained('google/electra-large-discriminator'),
}

PRETRAINED_MODELS = {
    # 'bert_base' : TFBertModel.from_pretrained('bert-base-uncased'),
    # 'bert_large' : TFBertModel.from_pretrained('bert-large-uncased'),
    # 'xlnet_base' : TFXLNetModel.from_pretrained('xlnet-base-cased'),
    # 'xlnet_large' : TFXLNetModel.from_pretrained('xlnet-large-cased'),
    # 'roberta_base' : TFRobertaModel.from_pretrained('roberta-base'),
    # 'roberta_large' : TFRobertaModel.from_pretrained('roberta-large'),
    'albert_base' : TFAlbertModel.from_pretrained('albert-base-v2'),
    # 'albert_large' : TFAlbertModel.from_pretrained('albert-large-v2'),
    # 'albert_xlarge' : TFAlbertModel.from_pretrained('albert-xlarge-v2'),
    # 'albert_xxlarge' : TFAlbertModel.from_pretrained('albert-xxlarge-v2'),
    # 't5_small' : TFT5Model.from_pretrained('t5-small'),
    # 't5_base' : TFT5Model.from_pretrained('t5-base'),
    # 't5_large' : TFT5Model.from_pretrained('t5-large'),
    # 'electra_small' : TFElectraForPreTraining.from_pretrained('google/electra-small-discriminator'),
    # 'electra_large' : TFElectraForPreTraining.from_pretrained('google/electra-large-discriminator')
}<jupyter_output><empty_output><jupyter_text>Tokenization and model inference functions<jupyter_code># Given that tweets are short, use a small value for reduced inference time
TOKENIZER_MAX_SEQ_LENGTH = 64 
 

def bert_embedding_model_inference(model, tokenizer, text):
  """ Tokenizes and computes BERT `model` inference for input `text`. """
  input_ids = tf.constant(
      tokenizer.encode(text, add_special_tokens=True))[None, :]  # Batch size 1
  outputs = model(input_ids)
  # The last hidden-state is the first element of the output tuple 
  last_hidden_states = outputs[0]
  cls_embedding = last_hidden_states[0][0]
  return cls_embedding


def albert_embedding_model_inference(model, tokenizer, text):
  """ Tokenizes and computes ALBERT `model` inference for input `text`. """
  encoded_text = tf.constant(tokenizer.encode(text))[None, :]
  outputs = model(encoded_text)
  # The last hidden-state is the first element of the output tuple 
  last_hidden_states = outputs[0]
  cls_embedding = last_hidden_states[0][0]
  return cls_embedding


def albert_embedding_batch_model_inference(model, tokenizer, text_batch):
  """ Tokenizes and computes ALBERT `model` inference for input `text_batch`.
  """
  encoded_text = [
      tf.constant(tokenizer.encode(
          t, max_length=TOKENIZER_MAX_SEQ_LENGTH, pad_to_max_length=True))
      for t in text_batch
  ]
  encoded_batch = tf.stack(encoded_text)
  outputs = model(encoded_text)
  cls_embeddings = []
  for last_hidden_state in outputs[0]:
    cls_embeddings.append(last_hidden_state[0])
  return cls_embeddings


def t5_embedding_model_inference(model, tokenizer, text):
  """ Tokenizes and computes T5 `model` inference for input `text`. """
  inputs = tokenizer.encode(text, return_tensors="tf")  # Batch size 1
  outputs = model(inputs, decoder_input_ids=inputs)
  # The last hidden-state is the first element of the output tuple 
  last_hidden_states = outputs[0]
  cls_embedding = last_hidden_states[0][0]
  return cls_embedding

'''
More model inference functions can be added, following documentation on
https://huggingface.co/transformers/
'''

INFERENCE_MODELS = {
  'bert' : bert_embedding_model_inference,
  'albert' : albert_embedding_model_inference,
  'albert_batch' : albert_embedding_batch_model_inference,
  't5' : t5_embedding_model_inference
}<jupyter_output><empty_output><jupyter_text>Testing tokenization and model inference with an example<jupyter_code>PRETRAINED_MODEL = 'albert_base'
INFERENCE = 'albert'

test_inference = INFERENCE_MODELS[INFERENCE](
    PRETRAINED_MODELS[PRETRAINED_MODEL],
    TOKENIZERS[PRETRAINED_MODEL],
    "Why is the sky blue"
)

print('Example embedding', test_inference)

embedding_dim = test_inference.shape[0]<jupyter_output><empty_output><jupyter_text>Hyperparameters for the classifier to be trained<jupyter_code>class HParams(object):
  """Hyperparameters used for training."""
  def __init__(self):
    ### dataset parameters
    self.num_classes = 2
    self.embedding_dim = embedding_dim
    ### neural graph learning parameters
    self.distance_type = nsl.configs.DistanceType.L2
    self.graph_regularization_multiplier = 0.1
    self.num_neighbors = 5
    ### model architecture
    self.num_fc_units = [64, 64]
    ### training parameters
    self.train_epochs = 50
    self.batch_size = 128
    self.encoder_inference_batch_size = 32
    self.dropout_rate = 0.2
    ### eval parameters
    self.eval_steps = None  # All instances in the test set are evaluated.

HPARAMS = HParams()<jupyter_output><empty_output><jupyter_text>Process dataset in batch or single inference mode<jupyter_code>def add_batch_embeddings(labels_and_texts):
  """Splits `labels_and_texts` into batches and performs tokenization and
  converts tweet text into its corresponding embedding.

  Args:
    labels_and_texts: A List of dictionaries such that each entry E contains:
      E['label']: (integer) the rumour veracity annotation.
      E['source_text']: (string) the source tweet text.
      E['reactions']: (List of strings) the texts from the tweet replies.

  Each entry E from labels_and_texts is updated, adding the following key,
  value pairs:
      E['source_embedding']: (Tensor of floats) embeddings for E['source_text']
      E['reaction_embedding']: (List of float Tensors) embeddings for
                               E['reactions'], up to HPARAMS.num_neighbors and
                               in the corresponding E['reactions'] order.
  """
  inputs = []
  print('Accumulating inputs')
  for e in labels_and_texts:
    inputs.append(e['source_text'])
    for r in e['reactions'][:HPARAMS.num_neighbors]:
      # Alternative ways to select neighbors within a tweet thread can be used.
      inputs.append(r)

  def generate_batches(inputs, batch_size):
    """Splits `inputs` list into chunks of (up to) `batch_size` length."""
    for i in range(0, len(inputs), batch_size):
      yield inputs[i: i + batch_size]

  inferences = []
  for i, batch in enumerate(generate_batches(
      inputs, HPARAMS.encoder_inference_batch_size)):
    print('Running model inference for batch', i)
    batch_inference = INFERENCE_MODELS[INFERENCE](
        PRETRAINED_MODELS[PRETRAINED_MODEL],
        TOKENIZERS[PRETRAINED_MODEL],
        batch)
    for inference in batch_inference:
      inferences.append(inference)

  i = 0
  for e in labels_and_texts:
    e['source_embedding'] = inferences[i]
    i += 1
    e['reaction_embedding'] = []
    for r in e['reactions'][:HPARAMS.num_neighbors]:
      e['reaction_embedding'].append(inferences[i])
      i += 1
def add_embeddings(labels_and_texts):
  """Performs tokenization and model inference for each element of
  labels_and_texts, updating it with computed embeddings.

  Args:
    labels_and_texts: A List of dictionaries such that each entry E contains:
      E['label']: (integer) the rumour veracity annotation.
      E['source_text']: (string) the source tweet text.
      E['reactions']: (List of strings) the texts from the tweet replies.

  Each entry E from labels_and_texts is updated, adding the following key,
  value pairs:
      E['source_embedding']: (Tensor of floats) embeddings for E['source_text']
      E['reaction_embedding']: (List of float Tensors) embeddings for
                               E['reactions'], up to HPARAMS.num_neighbors and
                               in the corresponding E['reactions'] order.
  """
  for index, label_and_texts in enumerate(labels_and_texts):
    if index % 100 == 0:
      print('Computing embeddings for item', index)
    label_and_texts['source_embedding'] = INFERENCE_MODELS[INFERENCE](
        PRETRAINED_MODELS[PRETRAINED_MODEL],
        TOKENIZERS[PRETRAINED_MODEL],
        label_and_texts['source_text'])
    label_and_texts['reaction_embedding'] = []
    for r in label_and_texts['reactions'][:HPARAMS.num_neighbors]:
      # Alternative ways to select neighbors within a tweet thread can be used.
      label_and_texts['reaction_embedding'].append(INFERENCE_MODELS[INFERENCE](
        PRETRAINED_MODELS[PRETRAINED_MODEL],
        TOKENIZERS[PRETRAINED_MODEL],
        r))<jupyter_output><empty_output><jupyter_text>Load and inspect dataset<jupyter_code>labels_and_texts = load_labels_and_texts(topics)
for e in random.sample(labels_and_texts, 3):
  pprint.pprint(e)<jupyter_output><empty_output><jupyter_text>Compute and store textual embeddings<jupyter_code>if 'batch' in INFERENCE:
  add_batch_embeddings(labels_and_texts)
else:
  add_embeddings(labels_and_texts)<jupyter_output><empty_output><jupyter_text>Create train and test datasetsUsing a 80:20 train:test split<jupyter_code># Alternative ways to split include a split by time or by news event.
random.shuffle(labels_and_texts)

train_size = int(0.8 * len(labels_and_texts))

TRAIN_DATA = labels_and_texts[:train_size]
TEST_DATA = labels_and_texts[train_size:]

# Constants used to identify neighbor features in the input.
NBR_FEATURE_PREFIX = 'NL_nbr_'
NBR_WEIGHT_SUFFIX = '_weight'
def create_np_tensors_from_datum(datum, propagate_label=True):
  """Creates a node and neighbor numpy tensors given input datum.

  Args:
    datum: A dictionary with node and neighbor features and annotation label.
    propagate_label: Boolean indicating if we labels should be propagated to
    neighbors.

  Returns:
    A tuple T (node_tensor, neighbor_tensors) such that:
      T[0]: a dictionary containing node embeddings and label
      T[1]: a List of dictionaries, each containing embeddings for a
      neighbor and, if propagate_label is true, the corresponding label.

  """
  label_tensor =  datum['label']
  
  def get_float32_tensor(d):
    np_array = np.array(d, dtype='f')
    return np_array
    
  node_tensor = {
    'embedding' : get_float32_tensor(datum['source_embedding']),
    'label' : label_tensor
  }

  neighbor_tensors = []
  for re in datum['reaction_embedding']:
    tensor = {
      'embedding' : get_float32_tensor(re),
    }
    if propagate_label:
      tensor['label'] = label_tensor
    neighbor_tensors.append(dict(tensor))

  return node_tensor, neighbor_tensors
TRAIN_TENSORS = [
  create_np_tensors_from_datum(d) for d in TRAIN_DATA
]
TEST_TENSORS = [
  create_np_tensors_from_datum(d) for d in TEST_DATA
]
TRAIN_TENSORS[0][0], TEST_TENSORS[0][0]<jupyter_output><empty_output><jupyter_text>Create train and test tf.data.TFRecordDataset<jupyter_code>def make_dataset(tf_features, training=False):
  """Creates a `tf.data.TFRecordDataset`.

  Args:
    tf_features: List of (node_tensor, neighbor_tensors) tuples.
    training: Boolean indicating if we are in training mode.

  Returns:
    An instance of `tf.data.TFRecordDataset` containing the `tf.train.Example`
    objects.
  """

  def get_tf_examples_with_nsl_signals(node_feature, neighbor_features):
    """Merges `neighbor_features` and `node_feature`.

    Args:
      node_feature: A dictionary of `tf.train.Feature`.
      neighbor_features: A list of `tf.train.Feature` dictionaries.

    Returns:
      A pair whose first value is a dictionary containing relevant features
      and whose second value contains the ground truth label.
    """
    feature_dict = dict(node_feature)
    # We also extract corresponding neighbor features in a similar manner to
    # the features above during training.
    if training:
      for i in range(HPARAMS.num_neighbors):
        nbr_feature_key = '{}{}_{}'.format(NBR_FEATURE_PREFIX, i, 'embedding')
        nbr_weight_key = '{}{}{}'.format(NBR_FEATURE_PREFIX, i,
                                         NBR_WEIGHT_SUFFIX)
        if i < len(neighbor_features):
          nf = neighbor_features[i]
          feature_dict[nbr_feature_key] = nf['embedding']
          feature_dict[nbr_weight_key] = 1.0
        else:
          feature_dict[nbr_feature_key] = np.zeros(
              HPARAMS.embedding_dim, dtype='f')
          feature_dict[nbr_weight_key] = 0.0

    label = feature_dict.pop('label')
    
    return feature_dict, label

  print('Adding NSL features for entries')
  tensors_with_nsl = {}
  labels = []
  for (node, neighbors) in tf_features:
    feature_dict, label = get_tf_examples_with_nsl_signals(node, neighbors)
    for k,v in feature_dict.items():
      if k not in tensors_with_nsl:
        tensors_with_nsl[k] = []
      tensors_with_nsl[k].append(v)
    labels.append(label)
  
  print('Creating tf.data.Dataset from tensors')
  dataset = tf.data.Dataset.from_tensor_slices((tensors_with_nsl, labels))
  dataset = dataset.batch(HPARAMS.batch_size)
  return dataset
train_dataset = make_dataset(TRAIN_TENSORS, training=True)
test_dataset = make_dataset(TEST_TENSORS)<jupyter_output><empty_output><jupyter_text>Inspecting train dataset<jupyter_code>for feature_batch, label_batch in train_dataset.take(1):
  print('Feature list:', list(feature_batch.keys()))
  print('Batch of inputs:', feature_batch['embedding'])
  nbr_feature_key = '{}{}_{}'.format(NBR_FEATURE_PREFIX, 0, 'embedding')
  nbr_weight_key = '{}{}{}'.format(NBR_FEATURE_PREFIX, 0, NBR_WEIGHT_SUFFIX)
  print('Batch of neighbor inputs:', feature_batch[nbr_feature_key])
  print('Batch of neighbor weights:',
        tf.reshape(feature_batch[nbr_weight_key], [-1]))
  print('Batch of labels:', label_batch)<jupyter_output><empty_output><jupyter_text>Inspecting test dataset<jupyter_code>for feature_batch, label_batch in test_dataset.take(1):
  print('Feature list:', list(feature_batch.keys()))
  print('Batch of inputs:', feature_batch['embedding'])
  print('Batch of labels:', label_batch)<jupyter_output><empty_output><jupyter_text>Multi-layer perceptron classification model  Sequential MLP model<jupyter_code>def make_mlp_sequential_model(hparams):
  """Creates a sequential multi-layer perceptron model."""
  model = tf.keras.Sequential()
  model.add(
      tf.keras.layers.InputLayer(
          input_shape=(hparams.embedding_dim,), name='embedding'))
  for num_units in hparams.num_fc_units:
    model.add(tf.keras.layers.Dense(num_units, activation='relu'))
    # For sequential models, by default, Keras ensures that the 'dropout' layer
    # is invoked only during training.
    model.add(tf.keras.layers.Dropout(hparams.dropout_rate))
  model.add(tf.keras.layers.Dense(hparams.num_classes, activation='softmax'))
  return model<jupyter_output><empty_output><jupyter_text>Functional MLP model<jupyter_code>def make_mlp_functional_model(hparams):
  """Creates a functional API-based multi-layer perceptron model."""
  inputs = tf.keras.Input(
      shape=(hparams.embedding_dim,), dtype='int64', name='embedding')

  cur_layer = inputs

  for num_units in hparams.num_fc_units:
    cur_layer = tf.keras.layers.Dense(num_units, activation='relu')(cur_layer)
    # For functional models, by default, Keras ensures that the 'dropout' layer
    # is invoked only during training.
    cur_layer = tf.keras.layers.Dropout(hparams.dropout_rate)(cur_layer)

  outputs = tf.keras.layers.Dense(
      hparams.num_classes, activation='softmax')(
          cur_layer)

  model = tf.keras.Model(inputs, outputs=outputs)
  return model<jupyter_output><empty_output><jupyter_text>tf.Keras.Model subclass MLP<jupyter_code>def make_mlp_subclass_model(hparams):
  """Creates a multi-layer perceptron subclass model in Keras."""

  class MLP(tf.keras.Model):
    """Subclass model defining a multi-layer perceptron."""

    def __init__(self):
      super(MLP, self).__init__()
      self.dense_layers = [
          tf.keras.layers.Dense(num_units, activation='relu')
          for num_units in hparams.num_fc_units
      ]
      self.dropout_layer = tf.keras.layers.Dropout(hparams.dropout_rate)
      self.output_layer = tf.keras.layers.Dense(
          hparams.num_classes, activation='softmax')

    def call(self, inputs, training=False):
      cur_layer = inputs['embedding']
      for dense_layer in self.dense_layers:
        cur_layer = dense_layer(cur_layer)
        cur_layer = self.dropout_layer(cur_layer, training=training)

      outputs = self.output_layer(cur_layer)

      return outputs

  return MLP()<jupyter_output><empty_output><jupyter_text>Base MLP model<jupyter_code># Create a base MLP model using the functional API.
# Alternatively, you can also create a sequential or subclass base model using
# the make_mlp_sequential_model() or make_mlp_subclass_model() functions
# respectively, defined above. Note that if a subclass model is used, its
# summary cannot be generated until it is built.
base_model_tag, base_model = 'FUNCTIONAL', make_mlp_functional_model(HPARAMS)
base_model.summary()<jupyter_output><empty_output><jupyter_text>Compile and train the base MLP model<jupyter_code>base_model.compile(
    optimizer='adam',
    loss='sparse_categorical_crossentropy',
    metrics=['accuracy'])
base_model.fit(train_dataset, epochs=HPARAMS.train_epochs, verbose=1)<jupyter_output><empty_output><jupyter_text>Evaluate base model on test dataset<jupyter_code># Helper function to print evaluation metrics.
def print_metrics(model_desc, eval_metrics):
  """Prints evaluation metrics.

  Args:
    model_desc: A description of the model.
    eval_metrics: A dictionary mapping metric names to corresponding values. It
      must contain the loss and accuracy metrics.
  """
  print('\n')
  print('Eval accuracy for ', model_desc, ': ', eval_metrics['accuracy'])
  print('Eval loss for ', model_desc, ': ', eval_metrics['loss'])
  if 'graph_loss' in eval_metrics:
    print('Eval graph loss for ', model_desc, ': ', eval_metrics['graph_loss'])
eval_results = dict(
    zip(base_model.metrics_names,
        base_model.evaluate(test_dataset, steps=HPARAMS.eval_steps)))
print_metrics('Base MLP model', eval_results)<jupyter_output><empty_output><jupyter_text>Create, compile and train MLP model with graph regularization<jupyter_code># Build a new base MLP model.
base_reg_model_tag, base_reg_model = 'FUNCTIONAL', make_mlp_functional_model(
    HPARAMS)

# Wrap the base MLP model with graph regularization.
graph_reg_config = nsl.configs.make_graph_reg_config(
    max_neighbors=HPARAMS.num_neighbors,
    multiplier=HPARAMS.graph_regularization_multiplier,
    distance_type=HPARAMS.distance_type,
    sum_over_axis=-1)
graph_reg_model = nsl.keras.GraphRegularization(base_reg_model,
                                                graph_reg_config)
graph_reg_model.compile(
    optimizer='adam',
    loss='sparse_categorical_crossentropy',
    metrics=['accuracy'])
graph_reg_model.fit(train_dataset, epochs=HPARAMS.train_epochs, verbose=1)<jupyter_output><empty_output><jupyter_text>Evaluate MLP model with graph regularization on test dataset<jupyter_code>eval_results = dict(
    zip(graph_reg_model.metrics_names,
        graph_reg_model.evaluate(test_dataset, steps=HPARAMS.eval_steps)))
print_metrics('MLP + graph regularization', eval_results)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10057471264367816, 'fraction_numerical': 0.006066411238825032, 'mean_word_length': 4.337814807801553, 'pattern_counts': {'":': 0, '<': 101, '<?xml version=': 0, '>': 100, 'https://': 6, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '24', 'dir': 'jupyter-structured-clean-dedup', 'id': '101560', 'license': 'Apache-2.0', 'n_tokens_mistral': 9155, 'n_tokens_neox': 8835, 'n_words': 2158, 'path': 'workshops/kdd_2020/graph_regularization_pheme_natural_graph.ipynb', 'repo_name': 'srihari-humbarwadi/neural-structured-learning', 'size': '28212'}