<commit_before>import os
import shutil
import sys

import pandas
import tensorflow
from tensorflow.contrib import learn, layers, metrics, tensorboard

COLUMNS = [
    "age", "workclass", "fnlwgt", "education", "education_num", "marital_status", "occupation", "relationship", "race",
    "gender", "capital_gain", "capital_loss", "hours_per_week", "native_country", "income_bracket",
]
CATEGORICAL_COLUMNS = [
    "occupation", "native_country",
]
LABEL_COLUMN = "label"
LOG_DIR = os.path.join(os.path.dirname(__file__), '../log/model_r')
if os.path.exists(LOG_DIR):
    shutil.rmtree(LOG_DIR)
METADATA_PATH = os.path.join(LOG_DIR, 'metadata.tsv')
TRAINING_SET = os.path.join(os.path.dirname(__file__), "../resources/adult.training.csv")
TESTING_SET = os.path.join(os.path.dirname(__file__), "../resources/adult.testing.csv")


def input_fn(df):
    categorical_cols = {
        attribute: tensorflow.SparseTensor(
            indices=[[i, 0] for i in range(df[attribute].size)],
            values=df[attribute].values,
            dense_shape=[df[attribute].size, 1]
        )
        for attribute in CATEGORICAL_COLUMNS
    }

    feature_cols = dict(categorical_cols)
    # Converts the label column into a constant Tensor.
    label = tensorflow.constant(df[LABEL_COLUMN].values)
    return feature_cols, label


def train_and_eval(train_steps, ):
    tensorflow.logging.set_verbosity(tensorflow.logging.INFO)
    df_train = pandas.read_csv(TRAINING_SET, names=COLUMNS, skipinitialspace=True, engine="python", )
    df_test = pandas.read_csv(TESTING_SET, names=COLUMNS, skipinitialspace=True, skiprows=1, engine="python", )

    # remove NaN elements
    df_train = df_train.dropna(how='any', axis=0)
    df_test = df_test.dropna(how='any', axis=0)

    df_train[LABEL_COLUMN] = (df_train["income_bracket"].apply(lambda x: ">50K" in x)).astype(int)
    df_test[LABEL_COLUMN] = (df_test["income_bracket"].apply(lambda x: ">50K" in x)).astype(int)

    sparse_columns = [
        layers.sparse_column_with_hash_bucket(
            attribute, hash_bucket_size=len(set(df_train[attribute]))
        ) for attribute in CATEGORICAL_COLUMNS
    ]
    embedding_columns = [
        layers.embedding_column(column, dimension=8) for column in sparse_columns
    ]
    m = learn.DNNClassifier(
        hidden_units=[10, 50, ],
        feature_columns=embedding_columns,
        model_dir=LOG_DIR,
        config=learn.RunConfig(save_checkpoints_secs=1, ),
    )
    validation_metrics = {
        "accuracy": learn.MetricSpec(metric_fn=metrics.streaming_accuracy, prediction_key="classes"),
        "precision": learn.MetricSpec(metric_fn=metrics.streaming_precision, prediction_key="classes"),
        "recall": learn.MetricSpec(metric_fn=metrics.streaming_recall, prediction_key="classes"),
    }
    configuration = tensorboard.plugins.projector.ProjectorConfig()
    embedding = configuration.embeddings.add()
    embedding.tensor_name = 'dnn/input_from_feature_columns/native_country_embedding/weights:0'
    embedding.metadata_path = METADATA_PATH
    summary_writer = tensorflow.summary.FileWriter(LOG_DIR)
    tensorboard.plugins.projector.visualize_embeddings(summary_writer, configuration)
    monitors = [
        # learn.monitors.CaptureVariable(),
        # learn.monitors.PrintTensor(),
        learn.monitors.ValidationMonitor(
            input_fn=lambda: input_fn(df_test), every_n_steps=1000, metrics=validation_metrics, early_stopping_rounds=1,
        ),
    ]
    m.fit(
        input_fn=lambda: input_fn(df_train),
        steps=train_steps,
        monitors=monitors,
    )
    results = m.evaluate(input_fn=lambda: input_fn(df_test), steps=1)
    for key in sorted(results):
        print("%s: %s" % (key, results[key]))
    with open(METADATA_PATH, 'w+') as metadata:
        for country in df_train['native_country'].unique():
            print(country, file=metadata)


def main(_):
    train_and_eval(200, )


if __name__ == "__main__":
    tensorflow.app.run(main=main, argv=[sys.argv[0]])<commit_msg>store metadata for both occupation and native country in log dir<commit_after>import os
import shutil
import sys

import pandas
import tensorflow
from tensorflow.contrib import learn, layers, metrics, tensorboard

COLUMNS = [
    "age", "workclass", "fnlwgt", "education", "education_num", "marital_status", "occupation", "relationship", "race",
    "gender", "capital_gain", "capital_loss", "hours_per_week", "native_country", "income_bracket",
]
CATEGORICAL_COLUMNS = [
    "occupation", "native_country",
]
LABEL_COLUMN = "label"
LOG_DIR = os.path.join(os.path.dirname(__file__), '../log/model_r')
if os.path.exists(LOG_DIR):
    shutil.rmtree(LOG_DIR)

NATIVE_COUNTRY_EMBEDDING = os.path.join(LOG_DIR, 'native_country_embedding.tsv')
OCCUPATION_EMBEDDING = os.path.join(LOG_DIR, 'occupation_embedding.tsv')

TENSORS = [{
    'tensor_name': tensor_name, 'metadata_path': os.path.join(LOG_DIR, tensor_name + '_metadata.tsv')} for
    tensor_name in ['native_country', 'occupation']
]

TRAINING_SET = os.path.join(os.path.dirname(__file__), "../resources/adult.training.csv")
TESTING_SET = os.path.join(os.path.dirname(__file__), "../resources/adult.testing.csv")


def input_fn(df):
    categorical_cols = {
        attribute: tensorflow.SparseTensor(
            indices=[[i, 0] for i in range(df[attribute].size)],
            values=df[attribute].values,
            dense_shape=[df[attribute].size, 1]
        )
        for attribute in CATEGORICAL_COLUMNS
    }

    feature_cols = dict(categorical_cols)
    # Converts the label column into a constant Tensor.
    label = tensorflow.constant(df[LABEL_COLUMN].values)
    return feature_cols, label


def train_and_eval(train_steps, ):
    tensorflow.logging.set_verbosity(tensorflow.logging.INFO)
    df_train = pandas.read_csv(TRAINING_SET, names=COLUMNS, skipinitialspace=True, engine="python", )
    df_test = pandas.read_csv(TESTING_SET, names=COLUMNS, skipinitialspace=True, skiprows=1, engine="python", )

    # remove NaN elements
    df_train = df_train.dropna(how='any', axis=0)
    df_test = df_test.dropna(how='any', axis=0)

    df_train[LABEL_COLUMN] = (df_train["income_bracket"].apply(lambda x: ">50K" in x)).astype(int)
    df_test[LABEL_COLUMN] = (df_test["income_bracket"].apply(lambda x: ">50K" in x)).astype(int)

    sparse_columns = [
        layers.sparse_column_with_hash_bucket(
            attribute, hash_bucket_size=len(set(df_train[attribute]))
        ) for attribute in CATEGORICAL_COLUMNS
    ]
    embedding_columns = [
        layers.embedding_column(column, dimension=8) for column in sparse_columns
    ]
    m = learn.DNNClassifier(
        hidden_units=[10, 50, ],
        feature_columns=embedding_columns,
        model_dir=LOG_DIR,
        config=learn.RunConfig(save_checkpoints_secs=1, ),
    )
    validation_metrics = {
        "accuracy": learn.MetricSpec(metric_fn=metrics.streaming_accuracy, prediction_key="classes"),
        "precision": learn.MetricSpec(metric_fn=metrics.streaming_precision, prediction_key="classes"),
        "recall": learn.MetricSpec(metric_fn=metrics.streaming_recall, prediction_key="classes"),
    }
    monitors = [
        # learn.monitors.CaptureVariable(),
        # learn.monitors.PrintTensor(),
        learn.monitors.ValidationMonitor(
            input_fn=lambda: input_fn(df_test), every_n_steps=1000, metrics=validation_metrics, early_stopping_rounds=1,
        ),
    ]
    m.fit(
        input_fn=lambda: input_fn(df_train),
        steps=train_steps,
        monitors=monitors,
    )
    results = m.evaluate(input_fn=lambda: input_fn(df_test), steps=1)
    for key in sorted(results):
        print("%s: %s" % (key, results[key]))

    configuration = tensorboard.plugins.projector.ProjectorConfig()
    for tensor in TENSORS:
        embedding = configuration.embeddings.add()
        embedding.tensor_name = 'dnn/input_from_feature_columns/' + tensor['tensor_name'] + '_embedding/weights:0'
        embedding.metadata_path = tensor['metadata_path']
        with open(tensor['metadata_path'], 'w+') as metadata:
            for item in df_train[tensor['tensor_name']].unique():
                print(item, file=metadata)
    tensorboard.plugins.projector.visualize_embeddings(tensorflow.summary.FileWriter(LOG_DIR), configuration)


def main(_):
    train_and_eval(200, )


if __name__ == "__main__":
    tensorflow.app.run(main=main, argv=[sys.argv[0]])
{'fraction_non_alphanumeric': 0.12663185378590078, 'fraction_numerical': 0.006171374317588417, 'mean_word_length': 4.119684082624544, 'pattern_counts': {'":': 8, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4465750', 'n_tokens_mistral': 2823, 'n_tokens_neox': 2750, 'n_words': 448}