<commit_before>import tensorflow as tf
from baseline.tf.tfy import embed, pool_chars


class TensorFlowEmbeddings(object):

    def __init__(self):
        pass

    def get_dsz(self):
        pass

    def get_vsz(self):
        pass

    def encode(self):
        pass

    def save_md(self):
        pass

    @classmethod
    def create_placeholder(cls, name):
        pass


class TensorFlowTokenEmbeddings(TensorFlowEmbeddings):

    @classmethod
    def create_placeholder(cls, name):
        return tf.placeholder(tf.int32, [None, None], name=name)

    def __init__(self, model, name, **kwargs):
        super(TensorFlowTokenEmbeddings, self).__init__()
        self.model = model
        self.finetune = kwargs.get('finetune', True)
        self.scope = kwargs.get('scope', 'LUT')
        self.name = name
        self.x = kwargs.get(self.name, self.create_placeholder(name))

    def get_vsz(self):
        return self.model.get_vsz()

    def get_dsz(self):
        return self.model.get_dsz()

    def encode(self):
        return embed(self.x,
                     len(self.model.vocab),
                     self.get_dsz(),
                     tf.constant_initializer(self.model.weights, dtype=tf.float32, verify_shape=True),
                     self.finetune,
                     self.scope)

    def save_md(self, model_file):
        #self.model.save(model_file, ignore_weights=True)
        pass

    @staticmethod
    def load():
        pass


class TensorFlowCharConvEmbeddings(TensorFlowEmbeddings):
<commit_msg>make sure to use the feature name to disambig<commit_after>import tensorflow as tf
from baseline.tf.tfy import embed, pool_chars


class TensorFlowEmbeddings(object):

    def __init__(self):
        pass

    def get_dsz(self):
        pass

    def get_vsz(self):
        pass

    def encode(self):
        pass

    def save_md(self):
        pass

    def save(self):
        pass

    @classmethod
    def create_placeholder(cls, name):
        pass


class TensorFlowTokenEmbeddings(TensorFlowEmbeddings):

    @classmethod
    def create_placeholder(cls, name):
        return tf.placeholder(tf.int32, [None, None], name=name)

    def __init__(self, model, name, **kwargs):
        super(TensorFlowTokenEmbeddings, self).__init__()
        self.model = model
        self.finetune = kwargs.get('finetune', True)
        self.name = name
        self.scope = kwargs.get('scope', '{}/LUT'.format(self.name))
        self.x = kwargs.get(self.name, self.create_placeholder(name))

    def get_vsz(self):
        return self.model.get_vsz()

    def get_dsz(self):
        return self.model.get_dsz()

    def encode(self):
        return embed(self.x,
                     len(self.model.vocab),
                     self.get_dsz(),
                     tf.constant_initializer(self.model.weights, dtype=tf.float32, verify_shape=True),
                     self.finetune,
                     self.scope)

    def save_md(self, model_file):
        #self.model.save(model_file, ignore_weights=True)
        pass

    @staticmethod
    def load():
        pass


class TensorFlowCharConvEmbeddings(TensorFlowEmbeddings):

{'fraction_non_alphanumeric': 0.10116977552956054, 'fraction_numerical': 0.002529244388239014, 'mean_word_length': 2.2753623188405796, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '906315', 'n_tokens_mistral': 995, 'n_tokens_neox': 940, 'n_words': 197}