<jupyter_start><jupyter_text>Load already pickled data:<jupyter_code>with open('data/df_retail.bin', 'rb') as f_in:
    df = pickle.load(f_in)
df.columns = df.columns.str.lower()
df = df[~df.invoiceno.astype('str').str.startswith('C')].reset_index(drop=True)
df.customerid = df.customerid.fillna(-1).astype('int32')<jupyter_output><empty_output><jupyter_text>Special pre-processor for sequences:<jupyter_code>class LabelEncoder:
    def fit(self, seq):
        self.vocab = sorted(set(seq))
        self.idx = {c: i + 1 for i, c in enumerate(self.vocab)}

    def vocab_size(self):
        return len(self.vocab) + 1

    def transfrom(self, seq):
        n = len(seq)
        result = np.zeros(n, dtype='int32')

        for i in range(n):
            result[i] = self.idx.get(seq[i], 0)

        return result

    def fit_transform(self, seq):
        self.fit(seq)
        return self.transfrom(seq)
item_enc = LabelEncoder()
df.stockcode = item_enc.fit_transform(df.stockcode.astype('str'))
df.stockcode = df.stockcode.astype('int32')<jupyter_output><empty_output><jupyter_text>Train-test split:<jupyter_code>df_train = df[df.invoicedate < '2011-10-09'].reset_index(drop=True)
df_val = df[(df.invoicedate >= '2011-10-09') & (df.invoicedate <= '2011-11-09') ].reset_index(drop=True)
df_test = df[df.invoicedate >= '2011-11-09'].reset_index(drop=True)
df_train.shape, df_val.shape, df_test.shape
user_enc = LabelEncoder()
user_enc.fit(df_train[df_train.customerid != -1].customerid)

df_train.customerid = user_enc.transfrom(df_train.customerid)
df_val.customerid = user_enc.transfrom(df_val.customerid)
uid_train = df_train.drop_duplicates(subset='invoiceno').customerid.values
uid_val = df_val.drop_duplicates(subset='invoiceno').customerid.values
def group_indptr(df):
    indptr, = np.where(df.invoiceno != df.invoiceno.shift())
    indptr = np.append(indptr, len(df)).astype('int32')
    return indptr

indptr_train = group_indptr(df_train)
indptr_val = group_indptr(df_val)
from collections import Counter
top_train = Counter(df_train.stockcode)<jupyter_output><empty_output><jupyter_text>Simple baseline<jupyter_code>def baseline(uid, indptr, items, top, k=5):
    n_groups = len(uid)
    n_items = len(items)

    pred_all = np.zeros((n_items, k), dtype=np.int32)

    for g in range(n_groups):
        t = top.copy()

        start = indptr[g]
        end = indptr[g+1]
        
        for i in range(start, end):
            pred = [k for (k, c) in t.most_common(5)]
            pred_all[i] = pred

            actual = items[i]
            if actual in t:
                del t[actual]

    return pred_all
iid_val = df_val.stockcode.values
pred_baseline = baseline(uid_val, indptr_val, iid_val, top_train, k=5)
@njit
def accuracy_k(y_true, y_pred):
    n, k = y_pred.shape

    acc = 0
    for i in range(n):
        for j in range(k):
            if y_pred[i, j] == y_true[i]:
                acc = acc + 1
                break

    return acc / n
accuracy_k(iid_val, pred_baseline)<jupyter_output><empty_output><jupyter_text>RNN naive modelData preparation<jupyter_code>def pack_items(users, items_indptr, items_vals):
    n = len(items_indptr) - 1

    result = []
    for i in range(n):
        start = items_indptr[i]
        end = items_indptr[i+1]
        result.append(items_vals[start:end])

    return result
train_items = pack_items(indptr_train, indptr_train, df_train.stockcode.values)

df_train_wrap = pd.DataFrame()
df_train_wrap['customerid'] = uid_train
df_train_wrap['items'] = train_items
df_train_wrap.head()
def pad_seq(data, num_steps):
    data = np.pad(data, pad_width=(1, 0), mode='constant')

    n = len(data)

    if n <= num_steps:
        pad_right = num_steps - n + 1
        data = np.pad(data, pad_width=(0, pad_right), mode='constant')

    return data

def prepare_train_data(data, num_steps):
    data = pad_seq(data, num_steps)

    X = []
    Y = []

    for i in range(num_steps, len(data)):
        start = i - num_steps
        X.append(data[start:i])
        Y.append(data[start+1:i+1])

    return X, Y<jupyter_output><empty_output><jupyter_text>Now ready to do some tensorflow<jupyter_code>import tensorflow as tf
rnn = tf.contrib.rnn
class Config:
    num_steps = 5

    num_items = item_enc.vocab_size()
    num_users = user_enc.vocab_size()

    init_scale = 0.1
    learning_rate = 1.0
    max_grad_norm = 5
    num_layers = 2
    hidden_size = 200
    embedding_size = 200
    batch_size = 20    

config = Config()
train_items = df_train_wrap['items']

X_train = []
Y_train = []

for i in range(len(train_items)):
    X, Y = prepare_train_data(train_items[i], config.num_steps)
    X_train.extend(X)
    Y_train.extend(Y)

X_train = np.array(X_train, dtype='int32')
Y_train = np.array(Y_train, dtype='int32')<jupyter_output><empty_output><jupyter_text>Model graph:<jupyter_code>def lstm_cell(hidden_size, is_training):
    return rnn.BasicLSTMCell(hidden_size, forget_bias=0.0, 
                             state_is_tuple=True, reuse=not is_training)

def rnn_model(inputs, hidden_size, num_layers, batch_size, num_steps, is_training):
    cells = [lstm_cell(hidden_size, is_training) for _ in range(num_layers)]
    cell = rnn.MultiRNNCell(cells, state_is_tuple=True)

    initial_state = cell.zero_state(batch_size, tf.float32)
    inputs = tf.unstack(inputs, num=num_steps, axis=1)
    outputs, final_state = rnn.static_rnn(cell, inputs, initial_state=initial_state)
    output = tf.reshape(tf.concat(outputs, 1), [-1, hidden_size])

    return output, initial_state, final_state


def model(config, is_training):
    batch_size = config.batch_size
    num_steps = config.num_steps
    embedding_size = config.embedding_size
    hidden_size = config.hidden_size
    num_items = config.num_items

    place_x = tf.placeholder(shape=[batch_size, num_steps], dtype=tf.int32)
    place_y = tf.placeholder(shape=[batch_size, num_steps], dtype=tf.int32)

    embedding = tf.get_variable("items", [num_items, embedding_size], dtype=tf.float32)
    inputs = tf.nn.embedding_lookup(embedding, place_x)

    output, initial_state, final_state = \
        rnn_model(inputs, hidden_size, config.num_layers, batch_size, num_steps, is_training)

    W = tf.get_variable("W", [hidden_size, num_items], dtype=tf.float32)
    b = tf.get_variable("b", [num_items], dtype=tf.float32)
    logits = tf.nn.xw_plus_b(output, W, b)
    logits = tf.reshape(logits, [batch_size, num_steps, num_items])

    loss = tf.losses.sparse_softmax_cross_entropy(place_y, logits)
    total_loss = tf.reduce_mean(loss)

    tvars = tf.trainable_variables()
    gradient = tf.gradients(total_loss, tvars)
    clipped, _ = tf.clip_by_global_norm(gradient, config.max_grad_norm)
    optimizer = tf.train.GradientDescentOptimizer(config.learning_rate)

    global_step = tf.train.get_or_create_global_step()
    train_op = optimizer.apply_gradients(zip(clipped, tvars), global_step=global_step)

    out = {}
    out['place_x'] = place_x
    out['place_y'] = place_y
    
    out['logits'] = logits
    out['initial_state'] = initial_state
    out['final_state'] = final_state

    out['total_loss'] = total_loss
    out['train_op'] = train_op

    return out<jupyter_output><empty_output><jupyter_text>Initialiation and training<jupyter_code>config = Config()
config_val = Config()
config_val.batch_size = 1
config_val.num_steps = 1

graph = tf.Graph()
graph.seed = 1

with graph.as_default():
    initializer = tf.random_uniform_initializer(-config.init_scale, config.init_scale)

    with tf.name_scope("Train"):
        with tf.variable_scope("Model", reuse=None, initializer=initializer):
            train_model = model(config, is_training=True)

    with tf.name_scope("Valid"):
        with tf.variable_scope("Model", reuse=True, initializer=initializer):
            val_model = model(config_val, is_training=False)

    init = tf.global_variables_initializer()
def prepare_batches(seq, step):
    n = len(seq)
    res = []
    for i in range(0, n, step):
        res.append(seq[i:i+step])
    return res
def run_epoch(session, model, X, Y, batch_size):
    fetches = {
        "total_loss": model['total_loss'],
        "final_state": model['final_state'],
        "eval_op": model['train_op']
    }

    num_steps = X.shape[1]
    all_idx = np.arange(X.shape[0])
    np.random.shuffle(all_idx)
    batches = prepare_batches(all_idx, batch_size)

    initial_state = session.run(model['initial_state'])
    current_state = initial_state

    progress = tqdm(total=len(batches))
    for idx in batches:
        if len(idx) < batch_size:
            continue

        feed_dict = {}
        for i, (c, h) in enumerate(model['initial_state']):
            feed_dict[c] = current_state[i].c
            feed_dict[h] = current_state[i].h

        feed_dict[model['place_x']] = X[idx]
        feed_dict[model['place_y']] = Y[idx]

        vals = session.run(fetches, feed_dict)
        loss = vals["total_loss"]
        current_state = vals["final_state"]

        progress.update(1)
        progress.set_description('%.3f' % loss)
    progress.close()
session = tf.Session(config=None, graph=graph) 
session.run(init)

np.random.seed(0)

run_epoch(session, train_model, X_train, Y_train, batch_size=config.batch_size)
def generate_prediction(uid, indptr, items, model, k):
    n_groups = len(uid)
    n_items = len(items)

    pred_all = np.zeros((n_items, k), dtype=np.int32)
    initial_state = session.run(model['initial_state'])

    fetches = {
        "logits": model['logits'],
        "final_state": model['final_state'],
    }

    for g in tqdm(range(n_groups)):    
        start = indptr[g]
        end = indptr[g+1]

        current_state = initial_state

        feed_dict = {}

        for i, (c, h) in enumerate(model['initial_state']):
            feed_dict[c] = current_state[i].c
            feed_dict[h] = current_state[i].h

        prev = np.array([[0]], dtype=np.int32)

        for i in range(start, end):
            feed_dict[model['place_x']] = prev

            actual = items[i]
            prev[0, 0] = actual

            values = session.run(fetches, feed_dict)
            current_state = values["final_state"]

            logits = values['logits'].reshape(-1)
            pred = np.argpartition(-logits, k)[:k]
            pred_all[i] = pred

    return pred_all
pred_lstm = generate_prediction(uid_val, indptr_val, iid_val, val_model, k=5)
accuracy_k(iid_val, pred_lstm)<jupyter_output><empty_output><jupyter_text>Let's add the user features<jupyter_code>X_train = []
U_train = []
Y_train = []


for t in df_train_wrap.itertuples():
    X, Y = prepare_train_data(t.items, config.num_steps)
    U_train.extend([t.customerid] * len(X))
    X_train.extend(X)
    Y_train.extend(Y)

X_train = np.array(X_train, dtype='int32')
Y_train = np.array(Y_train, dtype='int32')
U_train = np.array(U_train, dtype='int32')
def user_model(config, is_training):
    batch_size = config.batch_size
    num_steps = config.num_steps
    embedding_size = config.embedding_size
    hidden_size = config.hidden_size
    num_items = config.num_items
    num_users = config.num_users

    place_x = tf.placeholder(shape=[batch_size, num_steps], dtype=tf.int32)
    place_u = tf.placeholder(shape=[batch_size, 1], dtype=tf.int32)
    place_y = tf.placeholder(shape=[batch_size, num_steps], dtype=tf.int32)

    item_embedding = tf.get_variable("items", [num_items, embedding_size], dtype=tf.float32)
    item_inputs = tf.nn.embedding_lookup(item_embedding, place_x)
    
    user_embedding = tf.get_variable("users", [num_items, embedding_size], dtype=tf.float32)
    u_repeat = tf.tile(place_u, [1, num_steps])
    user_inputs = tf.nn.embedding_lookup(user_embedding, u_repeat)

    inputs = tf.concat([user_inputs, item_inputs], axis=2)
    
    output, initial_state, final_state = \
        rnn_model(inputs, hidden_size, config.num_layers, batch_size, num_steps, is_training)

    W = tf.get_variable("W", [hidden_size, num_items], dtype=tf.float32)
    b = tf.get_variable("b", [num_items], dtype=tf.float32)

    logits = tf.nn.xw_plus_b(output, W, b)
    logits = tf.reshape(logits, [batch_size, num_steps, num_items])

    loss = tf.losses.sparse_softmax_cross_entropy(place_y, logits)
    total_loss = tf.reduce_mean(loss)

    tvars = tf.trainable_variables()
    gradient = tf.gradients(total_loss, tvars)
    clipped, _ = tf.clip_by_global_norm(gradient, config.max_grad_norm)
    optimizer = tf.train.GradientDescentOptimizer(config.learning_rate)

    global_step = tf.train.get_or_create_global_step()
    train_op = optimizer.apply_gradients(zip(clipped, tvars), global_step=global_step)

    out = {}
    out['place_x'] = place_x
    out['place_u'] = place_u
    out['place_y'] = place_y
    

    out['logits'] = logits
    out['initial_state'] = initial_state
    out['final_state'] = final_state

    out['total_loss'] = total_loss
    out['train_op'] = train_op

    return out
graph = tf.Graph()
graph.seed = 1

with graph.as_default():
    initializer = tf.random_uniform_initializer(-config.init_scale, config.init_scale)

    with tf.name_scope("Train"):
        with tf.variable_scope("Model", reuse=None, initializer=initializer):
            train_model = user_model(config, is_training=True)

    with tf.name_scope("Valid"):
        with tf.variable_scope("Model", reuse=True, initializer=initializer):
            val_model = user_model(config_val, is_training=False)

    init = tf.global_variables_initializer()

session = tf.Session(config=None, graph=graph) 
session.run(init)<jupyter_output><empty_output><jupyter_text>Trainining:<jupyter_code>def user_model_epoch(session, model, X, U, Y, batch_size):
    fetches = {
        "total_loss": model['total_loss'],
        "final_state": model['final_state'],
        "eval_op": model['train_op']
    }

    num_steps = X.shape[1]
    all_idx = np.arange(X.shape[0])
    np.random.shuffle(all_idx)
    batches = prepare_batches(all_idx, batch_size)

    initial_state = session.run(model['initial_state'])
    current_state = initial_state

    progress = tqdm(total=len(batches))
    for idx in batches:
        if len(idx) < batch_size:
            continue

        feed_dict = {}
        for i, (c, h) in enumerate(model['initial_state']):
            feed_dict[c] = current_state[i].c
            feed_dict[h] = current_state[i].h

        feed_dict[model['place_x']] = X[idx]
        feed_dict[model['place_y']] = Y[idx]
        feed_dict[model['place_u']] = U[idx].reshape(-1, 1)

        vals = session.run(fetches, feed_dict)
        loss = vals["total_loss"]
        current_state = vals["final_state"]

        progress.update(1)
        progress.set_description('%.3f' % loss)
    progress.close()
session = tf.Session(config=None, graph=graph) 
session.run(init)

np.random.seed(0)

user_model_epoch(session, train_model, X_train, U_train, Y_train, batch_size=config.batch_size)
def generate_prediction_user_model(uid, indptr, items, model, k):
    n_groups = len(uid)
    n_items = len(items)

    pred_all = np.zeros((n_items, k), dtype=np.int32)
    initial_state = session.run(model['initial_state'])

    fetches = {
        "logits": model['logits'],
        "final_state": model['final_state'],
    }

    for g in tqdm(range(n_groups)):    
        start = indptr[g]
        end = indptr[g+1]
        u = uid[g]

        current_state = initial_state

        feed_dict = {}
        feed_dict[model['place_u']] = np.array([[u]], dtype=np.int32)

        for i, (c, h) in enumerate(model['initial_state']):
            feed_dict[c] = current_state[i].c
            feed_dict[h] = current_state[i].h

        prev = np.array([[0]], dtype=np.int32)

        for i in range(start, end):
            feed_dict[model['place_x']] = prev

            actual = items[i]
            prev[0, 0] = actual

            values = session.run(fetches, feed_dict)
            current_state = values["final_state"]

            logits = values['logits'].reshape(-1)
            pred = np.argpartition(-logits, k)[:k]
            pred_all[i] = pred

    return pred_all
pred_lstm = generate_prediction_user_model(uid_val, indptr_val, iid_val, val_model, k=5)
accuracy_k(iid_val, pred_lstm)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1291076923076923, 'fraction_numerical': 0.009784615384615385, 'mean_word_length': 3.5393854748603353, 'pattern_counts': {'":': 10, '<': 46, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '36745', 'license': 'MIT', 'n_tokens_mistral': 5961, 'n_tokens_neox': 5760, 'n_words': 1012, 'path': 'Chapter09/02-rnn-model.ipynb', 'repo_name': 'Diponly/Tensorflow_Projects', 'size': '16250'}