<commit_before>import tensorflow as tf, numpy as np, math

DISTANCES = ['euc', 'man', 'cos']
DISTANCE_MAPPINGS = {'euc': 'euclidean', 'man': 1}

# Shape Constructions
def shape(X): return X.get_shape().as_list() if type(X) == tf.Tensor else X.shape
def get_dim(X): return shape(X)[1]
def num_samples(X): return shape(X)[0]

def make_compatible(X, Y):
    num_x_samples, num_y_samples = num_samples(X), num_y_samples(Y)
    pad = lambda T, diff: return tf.pad(T, [[0, diff], [0, 0]], mode='Symmetric')
    return tf.case({
            tf.greater(num_x_samples, num_y_samples): lambda: (X, pad(Y, num_x_samples - num_y_samples)),
            tf.greater(num_y_samples, num_x_samples): lambda: (pad(X, num_y_samples - num_x_samples), Y),
        },
        default=lambda: (X, Y),
    )

# Distances
def _cosine_distance(X, Y, pad=1e-7):
    return 1 - tf.reduce_mean((pad + tf.reduce_sum(X*Y, axis=1))/(pad + tf.norm(X, axis=1)*tf.norm(Y, axis=1)))

def dist(X, Y, ord='euc'):
    assert ord in DISTANCES

    if ord == 'cos': return _cosine_distance(X, Y)
    else: return tf.reduce_mean(tf.norm(X - Y, axis=1, ord=DISTANCE_MAPPINGS[ord]))

# Neural Network Constructions
def linear(
      X, out_dim, scope,
      weights_initializer=tf.truncated_normal_initializer(mean=0.0, stddev=0.2),
      bias_initializer=tf.constant_initializer(0.0),
):
    source_dim = get_dim(X)

    with tf.variable_scope(scope):<commit_msg>Merging in changes made out-of-repo.<commit_after>import tensorflow as tf, numpy as np, math

DISTANCES = ['euc', 'man', 'cos']
DISTANCE_MAPPINGS = {'euc': 'euclidean', 'man': 1}

# Shape Constructions
def shape(X): return X.get_shape().as_list() if type(X) == tf.Tensor else X.shape
def get_dim(X): return shape(X)[1]
def num_samples(X): return shape(X)[0]

def make_compatible(X, Y):
    num_x_samples, num_y_samples = num_samples(X), num_y_samples(Y)
    pad = lambda T, diff: return tf.pad(T, [[0, diff], [0, 0]], mode='Symmetric')
    return tf.case({
            tf.greater(num_x_samples, num_y_samples): lambda: (X, pad(Y, num_x_samples - num_y_samples)),
            tf.greater(num_y_samples, num_x_samples): lambda: (pad(X, num_y_samples - num_x_samples), Y),
        },
        default=lambda: (X, Y),
    )

# Distances
def _cosine_distance(X, Y, pad=1e-7):
    return 1 - tf.reduce_mean((pad + tf.reduce_sum(X*Y, axis=1))/(pad + tf.norm(X, axis=1)*tf.norm(Y, axis=1)))

def dist(X, Y, ord='euc'):
    assert ord in DISTANCES

    if ord == 'cos': return _cosine_distance(X, Y)
    else: return tf.reduce_mean(tf.norm(X - Y, axis=1, ord=DISTANCE_MAPPINGS[ord]))

# Centering data:
def center(sample_df, axis=0):
    sample_mean = np.mean(sample_df, axis=axis)
    return (
        lambda X: X - sample_mean,
        lambda Y: Y + sample_mean,
    )

# Neural Network Constructions
def leaky_relu(X, alpha=0.2): return tf.maximum(alpha*X, X)
def linear(
      X, out_dim, scope,
      weights_initializer=tf.truncated_normal_initializer(mean=0.0, stddev=0.2),
      bias_initializer=tf.constant_initializer(0.0),
):
    source_dim = get_dim(X)

    with tf.variable_scope(scope):
{'fraction_non_alphanumeric': 0.15604890604890606, 'fraction_numerical': 0.013191763191763193, 'mean_word_length': 4.080065359477124, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '213931', 'n_tokens_mistral': 1214, 'n_tokens_neox': 1175, 'n_words': 268}