<jupyter_start><jupyter_text>Chapter 6: Conv Nets Sequence processing with convnetsThe previous notebook was getting unwieldy - here we look at 1D convnets for sequence processing. These are competitive with RNNs, and tend to be much quicker<jupyter_code>import keras
import numpy as np
import matplotlib.pyplot as plt
import zipfile
import pandas as pd
max_features=10000
max_length = 500

# Overwrite np.load to allow keras to work with the new Numpy defaults
old = np.load
np.load = lambda *a,**k: old(*a,**k,allow_pickle=True)
(X_train, y_train), (X_test, y_test) = keras.datasets.imdb.load_data(num_words=max_features)
np.load = old
del(old)


# Turns the list of integers into a 2D integer tensor
X_train = keras.preprocessing.sequence.pad_sequences(X_train, maxlen=max_length)
X_test = keras.preprocessing.sequence.pad_sequences(X_test, maxlen=max_length)

model = keras.models.Sequential()
model.add(keras.layers.Embedding(max_features, 128, input_length=max_length))
model.add(keras.layers.Conv1D(32, 7, activation="relu"))  # In 1D, we can afford to use larger convs
model.add(keras.layers.MaxPooling1D(5))
model.add(keras.layers.Conv1D(32, 7, activation="relu"))
model.add(keras.layers.GlobalMaxPooling1D())
model.add(keras.layers.Dense(1))

print(model.summary())

model.compile(optimizer=keras.optimizers.RMSprop(lr=1e-4), 
              loss="binary_crossentropy",
              metrics=["acc"]
             )

history = model.fit(X_train, y_train,
                    epochs=10,
                    batch_size=128,
                    validation_split=0.2
                   )
def plotLossAndAccuracy(history):
  history_dict = history.history
  acc = history_dict["acc"]
  val_acc = history_dict["val_acc"]
  loss = history_dict["loss"]
  val_loss = history_dict["val_loss"]
  epochs = range(1, len(loss) +1)

  plt.plot(epochs, acc, "bo", label="Training Accuracy")
  plt.plot(epochs, val_acc, "b", label="Validation Accuracy")
  plt.title("Training and validation accuracy")
  plt.xlabel("Epochs")
  plt.ylabel("Accuracy")
  plt.legend()
  plt.show()

  plt.plot(epochs, loss, "bo", label="Training Loss")
  plt.plot(epochs, val_loss, "b", label="Validation Loss")
  plt.title("Training and validation loss")
  plt.xlabel("Epochs")
  plt.ylabel("Loss")
  plt.legend()
  plt.show()
plotLossAndAccuracy(history)<jupyter_output><empty_output><jupyter_text>So the 1D convnet does about as well as the LSTM, but is quite a bit quicker. Note however that beyond the convolution window, a 1D convnet isn't sensitive to the order of the timestamps. We can see this if we try to run  a 1D convnet on the temperature-forecasting problem:<jupyter_code># First, download the raw  data  (14 different atmospheric quantities + temperature over 7 years).
!wget --no-check-certificate \
    https://s3.amazonaws.com/keras-datasets/jena_climate_2009_2016.csv.zip \
    -O /tmp/climatedata.zip
zip_ref = zipfile.ZipFile("/tmp/climatedata.zip", 'r')
zip_ref.extractall('/tmp')
zip_ref.close()

fname = "/tmp/jena_climate_2009_2016.csv"
df = pd.read_csv(fname, index_col=0, parse_dates=False)
float_data = df.to_numpy()

# Normalize
mean = float_data[:200000].mean(axis=0)
float_data -= mean
std = float_data[:200000].std(axis=0)
float_data /= std


# Generator yielding timeseries samples and their targets

def generator(data, lookback, delay, min_index, max_index, shuffle=False, batch_size=128, step=6):
  """
  Parameters
  ----------
  data : 2d numpy array
      The array of floating-point timeseries data, normalized.
  lookback : int
      How many timesteps back the input data should go.
  delay: int
      How many timesteps in the future the target should be.
  min_index: int
      The lower bound of timesteps to draw from (helps keep validation/test data separate) 
  max_index: int
      The upper bound of timesteps to draw from (helps keep validation/test data separate)
  shuffle: bool
      Whether to shuffle the samples or draw them in chronological order
  batch_size: int
      The number of samples per batch.
  step: int
      The period, in timesteps, at which you sample data.
  """
  if max_index is None:
    max_index = len(data) - delay - 1
  i = min_index + lookback
  while True:
    if shuffle:
      rows = np.random.randint(min_index+lookback, max_index, size=batch_size)
    else:
      if i+batch_size >= max_index:
        i = min_index + lookback
      rows = np.arange(i, min(i+batch_size, max_index))
      i += len(rows)
      
    samples = np.zeros((len(rows), lookback // step, data.shape[-1]))
    targets = np.zeros((len(rows),))
    for j, row in enumerate(rows):
      indices = range(rows[j] - lookback, rows[j], step)
      samples[j] = data[indices]
      targets[j] = data[rows[j] + delay][1]  # temperature is the second column
    yield samples, targets
    
    
# Create a training, validation, and test generator looking at different segments of the data
lookback = 1440
step = 6
delay = 144
batch_size = 128

train_gen = generator(float_data,
                      lookback=lookback,
                      delay=delay,
                      min_index=0,
                      max_index=200000,
                      shuffle=True,
                      step=step,
                      batch_size=batch_size
                     )
val_gen = generator(float_data,
                    lookback=lookback,
                    delay=delay,
                    min_index=200001,
                    max_index=300000,
                    step=step,
                    batch_size=batch_size
                   )
test_gen = generator(float_data,
                     lookback=lookback,
                     delay=delay,
                     min_index=300001,
                     max_index=None,
                     step=step,
                     batch_size=batch_size
                    )

# How many steps to draw from val_gen in order to see the entire validation set
val_steps = (300000 - 200001 - lookback) // batch_size
# As above, for the test set
test_steps = (len(float_data) - 300001 - lookback) // batch_size


model = keras.models.Sequential()
model.add(keras.layers.Conv1D(32, 5, activation="relu", input_shape=(None, float_data.shape[-1])))
model.add(keras.layers.MaxPooling1D(3))
model.add(keras.layers.Conv1D(32, 5, activation="relu"))
model.add(keras.layers.MaxPooling1D(3))
model.add(keras.layers.Conv1D(32, 5, activation="relu"))
model.add(keras.layers.GlobalMaxPooling1D())
model.add(keras.layers.Dense(1))



model.compile(optimizer=keras.optimizers.RMSprop(), 
              loss="mae"
             )

history = model.fit_generator(train_gen,
                              steps_per_epoch=500,
                              epochs=20,
                              validation_data=val_gen,
                              validation_steps=val_steps
                             )
def plotLoss(history):
  history_dict = history.history
  loss = history_dict["loss"]
  val_loss = history_dict["val_loss"]
  epochs = range(1, len(loss) +1)

  plt.plot(epochs, loss, "bo", label="Training Loss")
  plt.plot(epochs, val_loss, "b", label="Validation Loss")
  plt.title("Training and validation loss")
  plt.xlabel("Epochs")
  plt.ylabel("Loss")
  plt.legend()
  plt.show()
plotLoss(history)<jupyter_output><empty_output><jupyter_text>The 1D convnet is rubbish on the temperature-prediction task, you can't beat the common-sense baseline (or even come close). Combining CNNs and RNNs to process long sequencesWe can try to combine the speed of a convnet with the order-sensitivity of an RNN  by using a 1D convnet as a preprocessing step, which downsamples the sequence.<jupyter_code>lookback = 720
step = 3
delay = 144
batch_size = 128

train_gen = generator(float_data,
                      lookback=lookback,
                      delay=delay,
                      min_index=0,
                      max_index=200000,
                      shuffle=True,
                      step=step,
                      batch_size=batch_size
                     )
val_gen = generator(float_data,
                    lookback=lookback,
                    delay=delay,
                    min_index=200001,
                    max_index=300000,
                    step=step,
                    batch_size=batch_size
                   )

test_gen = generator(float_data,
                     lookback=lookback,
                     delay=delay,
                     min_index=300001,
                     max_index=None,
                     step=step,
                     batch_size=batch_size
                    )

# How many steps to draw from val_gen in order to see the entire validation set
val_steps = (300000 - 200001 - lookback) // batch_size
# As above, for the test set
test_steps = (len(float_data) - 300001 - lookback) // batch_size


model = keras.models.Sequential()
model.add(keras.layers.Conv1D(32, 5, activation="relu", input_shape=(None, float_data.shape[-1])))
model.add(keras.layers.MaxPooling1D(3))
model.add(keras.layers.Conv1D(32, 5, activation="relu"))
model.add(keras.layers.GRU(32, dropout=0.1, recurrent_dropout=0.5))
model.add(keras.layers.Dense(1))



model.compile(optimizer=keras.optimizers.RMSprop(), 
              loss="mae"
             )

history = model.fit_generator(train_gen,
                              steps_per_epoch=500,
                              epochs=20,
                              validation_data=val_gen,
                              validation_steps=val_steps
                             )
plotLoss(history)<jupyter_output><empty_output><jupyter_text>This setup is about as good as the regularised `GRU` by itself, but is much quicker.<jupyter_code><jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10022661722290895, 'fraction_numerical': 0.024927894519983517, 'mean_word_length': 2.5512070226773957, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 18, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '479524', 'license': 'MIT', 'n_tokens_mistral': 3223, 'n_tokens_neox': 2968, 'n_words': 734, 'path': 'Chapter6_Convnets.ipynb', 'repo_name': 'wllgrnt/keras-examples', 'size': '9708'}