<jupyter_start><jupyter_text>Label encode TensorFlow<jupyter_code>import os
import sys
import zipfile
import pandas as pd
import tensorflow as tf
from sklearn import preprocessing
from sklearn.model_selection import train_test_split

sys.path.insert(0,os.path.join(os.getcwd(), os.pardir,  'src', 'data'))
import stack_data

sys.path.insert(0,os.path.join(os.getcwd(), os.pardir,  'src', 'features'))
import encoded

SHOW_DISPLAY = True
 # Fetch the data
raw_data = stack_data.get_data()

print(raw_data.shape)
if SHOW_DISPLAY:
    display(raw_data.head())
# Label encode categorical data 
raw_data = encoded.get_encoded(raw_data, stack_data.LABEL_NAME)
print(raw_data.shape)
if SHOW_DISPLAY:
    display(raw_data.head())
# Split to train and test data.
# TODO: Consider cross validation
# https://towardsdatascience.com/train-test-split-and-cross-validation-in-python-80b61beca4b6
train, test = train_test_split(raw_data, train_size = 0.8, test_size = 0.2)
if SHOW_DISPLAY:
    display(train.head())
    display(test.head())
# 1. Assign the DataFrame's labels (the right-most column) to train_label.
# 2. Delete (pop) the labels from the DataFrame.
# 3. Assign the remainder of the DataFrame to train_features
X_train, Y_train = train, train.pop(stack_data.LABEL_NAME)
X_test, Y_test = test, test.pop(stack_data.LABEL_NAME)
# Create feature columns for all features.
my_feature_columns = []
for key in X_train.keys():
    my_feature_columns.append(tf.feature_column.numeric_column(key=key))
    
if SHOW_DISPLAY:
    display(my_feature_columns)
# Instantiate DNNClassifier
classifier = tf.estimator.DNNClassifier(
        feature_columns=my_feature_columns,
        hidden_units=[10, 10],
        n_classes=14)
# Train model
BATCH_SIZE = 100
TRAIN_STEPS = 1000

def train_input_fn(features, labels, batch_size):
    """An input function for training"""
    # Convert the inputs to a Dataset.
    dataset = tf.data.Dataset.from_tensor_slices((dict(features), labels))

    # Shuffle, repeat, and batch the examples.
    dataset = dataset.shuffle(1000).repeat().batch(batch_size)

    # Return the dataset.
    return dataset

classifier.train(
    input_fn=lambda:train_input_fn(X_train, Y_train, BATCH_SIZE),
    steps=TRAIN_STEPS)
 # Evaluate the model.
def eval_input_fn(features, labels, batch_size):
    """An input function for evaluation or prediction"""
    features=dict(features)
    if labels is None:
        # No labels, use only features.
        inputs = features
    else:
        inputs = (features, labels)

    # Convert the inputs to a Dataset.
    dataset = tf.data.Dataset.from_tensor_slices(inputs)

    # Batch the examples
    assert batch_size is not None, "batch_size must not be None"
    dataset = dataset.batch(batch_size)

    # Return the dataset.
    return dataset

    
eval_result = classifier.evaluate(
    input_fn=lambda:eval_input_fn(X_test, Y_test,
                                            BATCH_SIZE))

print('\nTest set accuracy: {accuracy:0.3f}\n'.format(**eval_result))<jupyter_output>INFO:tensorflow:Starting evaluation at 2018-02-20-18:33:15
INFO:tensorflow:Restoring parameters from C:\Users\vagrant\AppData\Local\Temp\tmpwyhgxdec\model.ckpt-1000
INFO:tensorflow:Finished evaluation at 2018-02-20-18:33:15
INFO:tensorflow:Saving dict for global step 1000: accuracy = 0.64567167, average_loss = 1.3496914, global_step = 1000, loss = 132.9843

Test set accuracy: 0.646
{'fraction_non_alphanumeric': 0.10125841381328651, 'fraction_numerical': 0.029850746268656716, 'mean_word_length': 4.307453416149069, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '557015', 'license': 'Apache-2.0', 'n_tokens_mistral': 1206, 'n_tokens_neox': 1121, 'n_words': 276, 'path': 'notebooks/label-tensor-flow.ipynb', 'repo_name': 'PioneerCode/pioneer-stack-over-you-ml', 'size': '3417'}