<jupyter_start><jupyter_text>NLP with Deep Learning for EveryoneNN with Keras        <NAME>        www.data4sci.com            @bgoncalves, @data4sci<jupyter_code>import warnings
warnings.filterwarnings('ignore')

from collections import Counter
from pprint import pprint

import pandas as pd
import numpy as np

import matplotlib
import matplotlib.pyplot as plt 

import keras
import tensorflow as tf
from keras.layers import Activation, Dense

from tqdm import tqdm

import watermark

%load_ext watermark
%matplotlib inline<jupyter_output><empty_output><jupyter_text>We start by print out the versions of the libraries we're using for future reference<jupyter_code>%watermark -n -v -m -g -iv<jupyter_output>Python implementation: CPython
Python version       : 3.8.5
IPython version      : 7.19.0

Compiler    : Clang 10.0.0 
OS          : Darwin
Release     : 20.5.0
Machine     : x86_64
Processor   : i386
CPU cores   : 16
Architecture: 64bit

Git hash: 39ce5cf3bd8432047933c28fa699f7e39cb38900

numpy     : 1.19.2
json      : 2.0.9
keras     : 2.4.3
pandas    : 1.1.3
matplotlib: 3.3.2
watermark : 2.1.0
tensorflow: 2.5.0<jupyter_text>Load default figure style<jupyter_code>plt.style.use('./d4sci.mplstyle')
colors = plt.rcParams['axes.prop_cycle'].by_key()['color']<jupyter_output><empty_output><jupyter_text>Activation functions We start by setting up the range of the input values we'll use for vizualization purposes<jupyter_code>z = np.linspace(-6, 6, 100)
def gradient(values, z):
    delta = z[1]-z[0]
    
    values = np.diff(values)

    return values/delta<jupyter_output><empty_output><jupyter_text>Linear<jupyter_code>linear = keras.layers.Activation('linear')
values = linear(z).numpy()
grad = gradient(values, z)
plt.plot(z, values, '-')
plt.plot(z[1:], grad, '-')
plt.xlabel('z')
plt.title('Linear activation function')
plt.legend(['function', 'gradient'])<jupyter_output><empty_output><jupyter_text>Sigmoid<jupyter_code>sigmoid = keras.layers.Activation('sigmoid')
values = sigmoid(z).numpy()
grad = gradient(values, z)
plt.plot(z, values, '-')
plt.plot(z[1:], grad, '-')
plt.legend(['function', 'gradient'])
plt.xlabel('z')
plt.title('Sigmoid activation function')<jupyter_output><empty_output><jupyter_text>Rectified Linear<jupyter_code>relu = keras.layers.Activation('relu')
values = relu(z).numpy()
grad = gradient(values, z)
plt.plot(z, values, '-')
plt.plot(z[1:], grad, '-')
plt.legend(['function', 'gradient'])
plt.xlabel('z')
plt.title('Rectified Linear activation function')<jupyter_output><empty_output><jupyter_text>Hyperbolic Tangent<jupyter_code>tanh = keras.layers.Activation('tanh')
values = tanh(z).numpy()
grad = gradient(values, z)
plt.plot(z, values, '-')
plt.plot(z[1:], grad, '-')
plt.legend(['function', 'gradient'])
plt.xlabel('z')
plt.title('Hyperbolic Tangent activation function')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1392095138160196, 'fraction_numerical': 0.03112976565232599, 'mean_word_length': 5.39821029082774, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '124699', 'license': 'MIT', 'n_tokens_mistral': 1134, 'n_tokens_neox': 1043, 'n_words': 219, 'path': '2. Neural Networks with Keras.ipynb', 'repo_name': 'netangel/AdvancedNLP', 'size': '2868'}