<jupyter_start><jupyter_text>Load data<jupyter_code>data_root = Path.home() / "data" / "tmp"
reuters_dir = data_root / "reuters21578"
reuters_corpus_path = reuters_dir / "corpus.pkl"
reuters = pickle.load(open(str(reuters_corpus_path), "rb"))
top_ten_ids, top_ten_names = reuters.top_n(n=10)

cache_dir = reuters_dir / "cache"<jupyter_output><empty_output><jupyter_text>Build dataframe<jupyter_code>df, top_ten_ids, train_labels, test_labels = reuters.build_dataframe()
df.head()<jupyter_output><empty_output><jupyter_text>Build feature extraction pipeline<jupyter_code>from sklearn.base import BaseEstimator, TransformerMixin

from sklearn.pipeline import Pipeline
from sklearn.pipeline import FeatureUnion

from sklearn.preprocessing import Imputer
from sklearn.preprocessing import StandardScaler

from sklearn.decomposition import TruncatedSVD
from sklearn.metrics import classification_report
from sklearn.feature_extraction import DictVectorizer
from sklearn.feature_extraction.text import TfidfVectorizer

from sklearn.svm import LinearSVC<jupyter_output><empty_output><jupyter_text>Transformer<jupyter_code>from tensorflow.python.keras.preprocessing import text
from tensorflow.python.keras.preprocessing import sequence
from ds_tutorial.transformers import TextFromColumns, TextStats, TextFromColumns2
class SequentialTextFromColumns(BaseEstimator, TransformerMixin):
    """Extract the text from a list of columns in a single pass.

    Takes a pandas dataframe and produces a series of sequence data
    from joined columns defined in `text_cols`.
    """
    text_cols = ["title", "body"]
    
    def __init__(self, text_cols=["title", "body"], max_sequence_length=1323, top_k=None):
        self.text_cols = text_cols
        self.max_sequence_length = max_sequence_length
        self.tokenizer = text.Tokenizer(num_words=top_k)
        
    def text_from_df(self, df):
        return TextFromColumns2().transform(df)
        
    def fit(self, df_train, y_train):
        train_text = self.text_from_df(df_train)
        self.tokenizer.fit_on_texts(train_text)
        return self

    def transform(self, df):
        text = TextFromColumns2().transform(df)        
        X = self.tokenizer.texts_to_sequences(text)
        
        max_length = len(max(X, key=len))
        if max_length > self.max_sequence_length:
            max_length = self.max_sequence_length

        # Fix sequence length to max value. Sequences shorter than the length are
        # padded in the beginning and sequences longer are truncated
        # at the beginning.
        X = sequence.pad_sequences(X, maxlen=max_length)
        return X<jupyter_output><empty_output><jupyter_text>Model<jupyter_code>from tensorflow.python.keras import models
from tensorflow.python.keras import initializers
from tensorflow.python.keras import regularizers

from tensorflow.python.keras.layers import Dense
from tensorflow.python.keras.layers import Dropout
from tensorflow.python.keras.layers import Embedding
from tensorflow.python.keras.layers import SeparableConv1D
from tensorflow.python.keras.layers import MaxPooling1D
from tensorflow.python.keras.layers import GlobalAveragePooling1D

def sepcnn_model(blocks,
                 filters,
                 kernel_size,
                 embedding_dim,
                 dropout_rate,
                 pool_size,
                 input_shape,
                 num_classes,
                 num_features,
                 use_pretrained_embedding=False,
                 is_embedding_trainable=False,
                 embedding_matrix=None):
    """Creates an instance of a separable CNN model.

    # Arguments
        blocks: int, number of pairs of sepCNN and pooling blocks in the model.
        filters: int, output dimension of the layers.
        kernel_size: int, length of the convolution window.
        embedding_dim: int, dimension of the embedding vectors.
        dropout_rate: float, percentage of input to drop at Dropout layers.
        pool_size: int, factor by which to downscale input at MaxPooling layer.
        input_shape: tuple, shape of input to the model.
        num_classes: int, number of output classes.
        num_features: int, number of words (embedding input dimension).
        use_pretrained_embedding: bool, true if pre-trained embedding is on.
        is_embedding_trainable: bool, true if embedding layer is trainable.
        embedding_matrix: dict, dictionary with embedding coefficients.

    # Returns
        A sepCNN model instance.
    """
    # op_units, op_activation = _get_last_layer_units_and_activation(num_classes)
    op_activation = "softmax"
    op_units = num_classes
    model = models.Sequential()

    # Add embedding layer. If pre-trained embedding is used add weights to the
    # embeddings layer and set trainable to input is_embedding_trainable flag.
    if use_pretrained_embedding:
        model.add(Embedding(input_dim=num_features,
                            output_dim=embedding_dim,
                            input_length=input_shape[0],
                            weights=[embedding_matrix],
                            trainable=is_embedding_trainable))
    else:
        model.add(Embedding(input_dim=num_features,
                            output_dim=embedding_dim,
                            input_length=input_shape[0]))

    for _ in range(blocks-1):
        model.add(Dropout(rate=dropout_rate))
        model.add(SeparableConv1D(filters=filters,
                                  kernel_size=kernel_size,
                                  activation='relu',
                                  bias_initializer='random_uniform',
                                  depthwise_initializer='random_uniform',
                                  padding='same'))
        model.add(SeparableConv1D(filters=filters,
                                  kernel_size=kernel_size,
                                  activation='relu',
                                  bias_initializer='random_uniform',
                                  depthwise_initializer='random_uniform',
                                  padding='same'))
        model.add(MaxPooling1D(pool_size=pool_size))

    model.add(SeparableConv1D(filters=filters * 2,
                              kernel_size=kernel_size,
                              activation='relu',
                              bias_initializer='random_uniform',
                              depthwise_initializer='random_uniform',
                              padding='same'))
    model.add(SeparableConv1D(filters=filters * 2,
                              kernel_size=kernel_size,
                              activation='relu',
                              bias_initializer='random_uniform',
                              depthwise_initializer='random_uniform',
                              padding='same'))
    model.add(GlobalAveragePooling1D())
    model.add(Dropout(rate=dropout_rate))
    model.add(Dense(op_units, activation=op_activation))
    return model
df_train = df.query("modapte == 'train'")
df_test = df.query("modapte == 'test'")
y_train = df_train.label.values
y_test = df_test.label.values
top_k = 2000000
stfc = SequentialTextFromColumns(top_k=top_k)
stfc.fit(df_train, y_train)
num_features = min(len(stfc.tokenizer.word_index) + 1, top_k)<jupyter_output><empty_output><jupyter_text>Sequential model<jupyter_code>from tensorflow.python.keras.optimizers import Adam

model = sepcnn_model(
    blocks=2,
    filters=64,
    kernel_size=3,
    embedding_dim=200,
    dropout_rate=0.2,
    pool_size=3,
    input_shape=(stfc.max_sequence_length,),
    num_classes=75,
    num_features=num_features
)

optimizer = Adam(lr=1e-3)
model.compile(optimizer=optimizer, loss="sparse_categorical_crossentropy", metrics=['acc'])
pipeline = Pipeline(steps=[
    ("combined_text_sequential", Pipeline([
        ("sequences", SequentialTextFromColumns(top_k=20000)),
    ])),
    # ("clf", model)
])
X_train = pipeline.fit_transform(df_train, y_train)
X_test = pipeline.transform(df_test)
import tensorflow as tf
callbacks = [tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=10)]
history = model.fit(
    X_train,
    y_train,
    #epochs=500,
    epochs=70,
    validation_data=(X_test, y_test),
    #callbacks=callbacks,
    batch_size=128
)
import json

models_dir = data_root / "models" 
models_dir.mkdir(exist_ok=True)
model.save(str(models_dir / "reuters_sequential.h5"))

history_path = models_dir / "reuters_sequential.json"
with open(str(history_path), "w") as f:
    json.dump(history.history, f)
history = json.load(open(str(history_path)))
y_pred = model.predict_classes(X_test)
print(classification_report(y_test, y_pred, target_names=top_ten_names, labels=top_ten_ids, digits=3))
y_pred = model.predict_classes(X_test)
print(classification_report(y_test, y_pred, target_names=top_ten_names, labels=top_ten_ids, digits=3))
y_pred = model.predict_classes(X_test)
print(classification_report(y_test, y_pred, target_names=top_ten_names, labels=top_ten_ids, digits=3))
y_pred = model.predict_classes(X_test)
print(classification_report(y_test, y_pred, target_names=top_ten_names, labels=top_ten_ids, digits=3))<jupyter_output>precision    recall  f1-score   support

       earn      0.971     0.977     0.974      1087
        acq      0.911     0.962     0.936       710
   money-fx      0.606     0.731     0.662       145
      grain      0.319     0.357     0.337        42
      crude      0.799     0.799     0.799       164
      trade      0.676     0.688     0.682       109
   interest      0.686     0.692     0.689       117
       ship      0.351     0.479     0.405        71
      wheat      0.744     0.582     0.653        55
       corn      0.543     0.556     0.549        45

avg / total      0.856     0.882     0.868      2545<jupyter_text>Plot training vs test accuracy<jupyter_code>import matplotlib.pyplot as plt

acc = history['acc']
val_acc = history['val_acc']
loss = history['loss']
val_loss = history['val_loss']

epochs = range(len(acc))

plt.figure(figsize=(15, 20))

plt.subplot(211)

plt.plot(epochs, acc, 'bo', label='Training acc')
plt.plot(epochs, val_acc, 'b', label='Validation acc')
plt.title('Training and validation accuracy')
plt.legend()

plt.subplot(212)

plt.plot(epochs, loss, 'bo', label='Training loss')
plt.plot(epochs, val_loss, 'b', label='Validation loss')
plt.title('Training and validation loss')
plt.legend()

plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.08695652173913043, 'fraction_numerical': 0.02326468344774981, 'mean_word_length': 2.8100254268071194, 'pattern_counts': {'":': 0, '<': 28, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '283342', 'license': 'MIT', 'n_tokens_mistral': 3351, 'n_tokens_neox': 3173, 'n_words': 655, 'path': 'notebooks/text_classification/sequential_model.ipynb', 'repo_name': 'ephes/data_science_tutorial', 'size': '10488'}