<commit_before>#encoding=utf-8

# Copyright 2017 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Vocabulary related functions.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

"""
total vocab = special_words + actual vocab(special words first)  
"""

import codecs
import six
import csv
import collections
import tensorflow as tf
from tensorflow import gfile
from seq2seq.features import SpecialWords

#["PAD","UNK", "SEQUENCE_START", "SEQUENCE_END", "PARA_START", "PARA_END"]
SpecialVocab = collections.namedtuple("SpecialVocab", SpecialWords._total_words)

class VocabInfo(
    collections.namedtuple("VocabInfo",
                           ["path", "vocab_size", "special_vocab"])):
  """Convenience structure for vocabulary information.
  """

  @property
  def total_size(self):
    """Returns size the the base vocabulary plus the size of extra vocabulary"""
    return self.vocab_size + len(self.special_vocab)


def get_vocab_info(vocab_path, special_words=SpecialWords):
  """Creates a `VocabInfo` instance that contains the vocabulary size and
    the special vocabulary for the given file.

  Args:
    vocab_path: Path to a vocabulary file with one word per line.

  Returns:
    A VocabInfo tuple.
  """
  with gfile.GFile(vocab_path) as file:
    vocab_size = sum(1 for _ in file)
  special_vocab = get_special_vocab(0, special_words)
  return VocabInfo(vocab_path, vocab_size, special_vocab)


def get_special_vocab(first_index, special_words):
  """Returns the `SpecialVocab` instance for a given vocabulary size.
  """
  SpecialVocab = collections.namedtuple("SpecialVocab", special_words)
  return SpecialVocab(*range(first_index, first_index + len(SpecialVocab._fields)))

def create_tensor_vocab(vocab_instance):

  assert isinstance(vocab_instance, Vocab)
  word_to_ids = vocab_instance._word_to_id
  word_to_count = vocab_instance._word_to_count

  vocab, ids = list(word_to_ids.keys()), list(word_to_ids.values())
  vocab_size = len(vocab)

  counts = [word_to_count[v] for v in word_to_count]
  count_tensor = tf.constant(counts, dtype=tf.int64)
  vocab_tensor = tf.constant(vocab)
  vocab_idx_tensor = tf.constant(ids, dtype=tf.int64)

  # Create ID -> word mapping
  id_to_vocab_init = tf.contrib.lookup.KeyValueTensorInitializer(
      vocab_idx_tensor, vocab_tensor, tf.int64, tf.string)
  id_to_vocab_table = tf.contrib.lookup.HashTable(id_to_vocab_init, "UNK")

  # Create word -> id mapping
  vocab_to_id_init = tf.contrib.lookup.KeyValueTensorInitializer(
      vocab_tensor, vocab_idx_tensor, tf.string, tf.int64)
  vocab_to_id_table = tf.contrib.lookup.HashTable(vocab_to_id_init,
                                                  vocab_instance.special_vocab.UNK)

  # Create word -> count mapping
  word_to_count_init = tf.contrib.lookup.KeyValueTensorInitializer(
      vocab_tensor, count_tensor, tf.string, tf.int64)
  word_to_count_table = tf.contrib.lookup.HashTable(word_to_count_init, -1)

  return vocab_to_id_table, id_to_vocab_table, word_to_count_table, vocab_size

def create_vocabulary_lookup_table(filename, specaial_words = SpecialWords, default_value=None):

  #old version
  """Creates a lookup table for a vocabulary file.

  Args:
    filename: Path to a vocabulary file containg one word per line.
      Each word is mapped to its line number.
    default_value: UNK tokens will be mapped to this id.<commit_msg>generate vocab<commit_after>#encoding=utf-8

# Copyright 2017 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Vocabulary related functions.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

"""
total vocab = special_words + actual vocab(special words first)  
"""

import codecs
import six
import csv
import collections
import tensorflow as tf
from tensorflow import gfile
from seq2seq.features import SpecialWords

#["PAD","UNK", "SEQUENCE_START", "SEQUENCE_END", "PARA_START", "PARA_END"]
SpecialVocab = collections.namedtuple("SpecialVocab", SpecialWords._total_words)

class VocabInfo(
    collections.namedtuple("VocabInfo",
                           ["path", "vocab_size", "special_vocab"])):
  """Convenience structure for vocabulary information.
  """

  @property
  def total_size(self):
    """Returns size the the base vocabulary plus the size of extra vocabulary"""
    return self.vocab_size + len(self.special_vocab)


def get_vocab_info(vocab_path, special_words=SpecialWords._total_words):
  """Creates a `VocabInfo` instance that contains the vocabulary size and
    the special vocabulary for the given file.

  Args:
    vocab_path: Path to a vocabulary file with one word per line.

  Returns:
    A VocabInfo tuple.
  """
  with gfile.GFile(vocab_path) as file:
    vocab_size = sum(1 for _ in file)
  special_vocab = get_special_vocab(0, special_words)
  return VocabInfo(vocab_path, vocab_size, special_vocab)


def get_special_vocab(first_index, special_words):
  """Returns the `SpecialVocab` instance for a given vocabulary size.
  """
  SpecialVocab = collections.namedtuple("SpecialVocab", special_words)
  return SpecialVocab(*range(first_index, first_index + len(SpecialVocab._fields)))

def create_tensor_vocab(vocab_instance):

  assert isinstance(vocab_instance, Vocab)
  word_to_ids = vocab_instance._word_to_id
  word_to_count = vocab_instance._word_to_count

  vocab, ids = list(word_to_ids.keys()), list(word_to_ids.values())
  vocab_size = len(vocab)

  counts = [word_to_count[v] for v in word_to_count]
  count_tensor = tf.constant(counts, dtype=tf.int64)
  vocab_tensor = tf.constant(vocab)
  vocab_idx_tensor = tf.constant(ids, dtype=tf.int64)

  # Create ID -> word mapping
  id_to_vocab_init = tf.contrib.lookup.KeyValueTensorInitializer(
      vocab_idx_tensor, vocab_tensor, tf.int64, tf.string)
  id_to_vocab_table = tf.contrib.lookup.HashTable(id_to_vocab_init, "UNK")

  # Create word -> id mapping
  vocab_to_id_init = tf.contrib.lookup.KeyValueTensorInitializer(
      vocab_tensor, vocab_idx_tensor, tf.string, tf.int64)
  vocab_to_id_table = tf.contrib.lookup.HashTable(vocab_to_id_init,
                                                  vocab_instance.special_vocab.UNK)

  # Create word -> count mapping
  word_to_count_init = tf.contrib.lookup.KeyValueTensorInitializer(
      vocab_tensor, count_tensor, tf.string, tf.int64)
  word_to_count_table = tf.contrib.lookup.HashTable(word_to_count_init, -1)

  return vocab_to_id_table, id_to_vocab_table, word_to_count_table, vocab_size

def create_vocabulary_lookup_table(filename, specaial_words = SpecialWords._total_words, default_value=None):

  #old version
  """Creates a lookup table for a vocabulary file.

  Args:
    filename: Path to a vocabulary file containg one word per line.
      Each word is mapped to its line number.
    default_value: UNK tokens will be mapped to this id.
{'fraction_non_alphanumeric': 0.08433278626436055, 'fraction_numerical': 0.005807347557126625, 'mean_word_length': 4.5553997194950915, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7519033', 'n_tokens_mistral': 2590, 'n_tokens_neox': 2543, 'n_words': 700}