<commit_before># coding=utf-8
# Copyright 2017 The Tensor2Tensor Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Data reader module."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import math
import os

# Dependency imports

import six
from six.moves import zip  # pylint: disable=redefined-builtin

from tensor2tensor.data_generators import problem_hparams
from tensor2tensor.data_generators.problem import preprocess_examples_common
from tensor2tensor.utils import registry

import tensorflow as tf


def examples_reader(data_sources,
                    data_fields_to_features,
                    training,
                    capacity=32,
                    data_items_to_decoders=None,
                    data_items_to_decode=None):
  """Reads Examples from data_sources and decodes to Tensors.

  The dictionary data_fields_to_features for an image dataset can be:

  data_fields_to_features = {
    'image/encoded': tf.FixedLenFeature((), tf.string, default_value=''),
    'image/format': tf.FixedLenFeature((), tf.string, default_value='raw'),
    'image/class/label': tf.FixedLenFeature(
        [1], tf.int64, default_value=tf.zeros([1], dtype=tf.int64)),
  }

  and for a simple algorithmic dataset with variable-length data it is:

  data_fields_to_features = {
    'inputs': tf.VarLenFeature(tf.int64),
    'targets': tf.VarLenFeature(tf.int64),
  }

  The data_items_to_decoders dictionary argument can be left as None if there
  is no decoding to be performed. But, e.g. for images, it should be set so that
  the images are decoded from the features, e.g., for MNIST:

  data_items_to_decoders = {
    'image': tfexample_decoder.Image(
      image_key = 'image/encoded',
      format_key = 'image/format',
      shape=[28, 28],
      channels=1),
    'label': tfexample_decoder.Tensor('image/class/label'),
  }

  These arguments are compatible with the use of tf.contrib.slim.data module,
  see there for more documentation.

  Args:
    data_sources: a list or tuple of sources from which the data will be read,
      for example [/path/to/train@128, /path/to/train2*, /tmp/.../train3*]
    data_fields_to_features: a dictionary from data fields in the data sources
      to features, such as tf.VarLenFeature(tf.int64), see above for examples.
    training: a Boolean, whether to read for training or evaluation.
    capacity: integer, buffer capacity; set to 2 * max_batch_size or more.
    data_items_to_decoders: a dictionary mapping data items (that will be
      in the returned result) to decoders that will decode them using features
      defined in data_fields_to_features; see above for examples. By default
      (if this is None), we grab the tensor from every feature.
    data_items_to_decode: a subset of data items that will be decoded;
      by default (if this is None), we decode all items.

  Returns:
    A dictionary mapping each data_field to a corresponding 1D int64 tensor
    read from the created Dataset.
  """

  def decode_record(record):
    """Serialized Example to dict of <feature name, Tensor>."""
    example_serialized = record
    item_decoders = data_items_to_decoders
    if item_decoders is None:
      item_decoders = {
          field: tf.contrib.slim.tfexample_decoder.Tensor(field)
          for field in data_fields_to_features
      }

    decoder = tf.contrib.slim.tfexample_decoder.TFExampleDecoder(
        data_fields_to_features, item_decoders)

    decode_items = data_items_to_decode
    if decode_items is None:
      decode_items = list(item_decoders)

    decoded = decoder.decode(example_serialized, items=decode_items)
    return dict(zip(decode_items, decoded))

  with tf.name_scope("examples_in"):
    # Read serialized examples using slim parallel_reader.
    data_files = tf.contrib.slim.parallel_reader.get_data_files(data_sources)
    num_readers = min(4 if training else 1, len(data_files))
    _, example_serialized = tf.contrib.slim.parallel_reader.parallel_read(
        data_sources,
        tf.TFRecordReader,
        num_epochs=None if training else 1,
        shuffle=training,
        capacity=2 * capacity,
        min_after_dequeue=capacity,
        num_readers=num_readers)
    return decode_record(example_serialized)


def preprocessing(examples, data_file_pattern):
  """Preprocessing of examples."""
  # This function is for obsolete problems only, as we're porting them
  # all to the Problem class and its preprocess_examples method. Don't add.
  if "image" in data_file_pattern:
    def resize(img, size):
      return tf.to_int64(tf.image.resize_images(img, [size, size]))

    if "img2img" in data_file_pattern:
      inputs = examples["inputs"]
      examples["inputs"] = resize(inputs, 16)
      examples["targets"] = resize(inputs, 64)
    elif "image_celeba" in data_file_pattern:
      inputs = examples["inputs"]
      examples["inputs"] = resize(inputs, 8)
      examples["targets"] = resize(inputs, 32)
  elif "audio" in data_file_pattern:
    # Reshape audio to proper shape
    sample_count = tf.to_int32(examples.pop("audio/sample_count"))
    sample_width = tf.to_int32(examples.pop("audio/sample_width"))
    channel_count = 1
    examples["inputs"] = tf.reshape(examples["inputs"],
                                    [sample_count, sample_width, channel_count])
    if "wsj" in data_file_pattern:
      examples["inputs"] = tf.bitcast(examples["inputs"], tf.int32)
  elif "a2q_20161229" in data_file_pattern:
    # we forgot the EOS when we preprocessed this data.
    examples["targets"] = tf.concat([examples["targets"], [1]], 0)
  return examples


def cast_int64_to_int32(features):<commit_msg>Crop CelebA images, change resizing method<commit_after># coding=utf-8
# Copyright 2017 The Tensor2Tensor Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Data reader module."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import math
import os

# Dependency imports

import six
from six.moves import zip  # pylint: disable=redefined-builtin

from tensor2tensor.data_generators import problem_hparams
from tensor2tensor.data_generators.problem import preprocess_examples_common
from tensor2tensor.utils import registry

import tensorflow as tf


def examples_reader(data_sources,
                    data_fields_to_features,
                    training,
                    capacity=32,
                    data_items_to_decoders=None,
                    data_items_to_decode=None):
  """Reads Examples from data_sources and decodes to Tensors.

  The dictionary data_fields_to_features for an image dataset can be:

  data_fields_to_features = {
    'image/encoded': tf.FixedLenFeature((), tf.string, default_value=''),
    'image/format': tf.FixedLenFeature((), tf.string, default_value='raw'),
    'image/class/label': tf.FixedLenFeature(
        [1], tf.int64, default_value=tf.zeros([1], dtype=tf.int64)),
  }

  and for a simple algorithmic dataset with variable-length data it is:

  data_fields_to_features = {
    'inputs': tf.VarLenFeature(tf.int64),
    'targets': tf.VarLenFeature(tf.int64),
  }

  The data_items_to_decoders dictionary argument can be left as None if there
  is no decoding to be performed. But, e.g. for images, it should be set so that
  the images are decoded from the features, e.g., for MNIST:

  data_items_to_decoders = {
    'image': tfexample_decoder.Image(
      image_key = 'image/encoded',
      format_key = 'image/format',
      shape=[28, 28],
      channels=1),
    'label': tfexample_decoder.Tensor('image/class/label'),
  }

  These arguments are compatible with the use of tf.contrib.slim.data module,
  see there for more documentation.

  Args:
    data_sources: a list or tuple of sources from which the data will be read,
      for example [/path/to/train@128, /path/to/train2*, /tmp/.../train3*]
    data_fields_to_features: a dictionary from data fields in the data sources
      to features, such as tf.VarLenFeature(tf.int64), see above for examples.
    training: a Boolean, whether to read for training or evaluation.
    capacity: integer, buffer capacity; set to 2 * max_batch_size or more.
    data_items_to_decoders: a dictionary mapping data items (that will be
      in the returned result) to decoders that will decode them using features
      defined in data_fields_to_features; see above for examples. By default
      (if this is None), we grab the tensor from every feature.
    data_items_to_decode: a subset of data items that will be decoded;
      by default (if this is None), we decode all items.

  Returns:
    A dictionary mapping each data_field to a corresponding 1D int64 tensor
    read from the created Dataset.
  """

  def decode_record(record):
    """Serialized Example to dict of <feature name, Tensor>."""
    example_serialized = record
    item_decoders = data_items_to_decoders
    if item_decoders is None:
      item_decoders = {
          field: tf.contrib.slim.tfexample_decoder.Tensor(field)
          for field in data_fields_to_features
      }

    decoder = tf.contrib.slim.tfexample_decoder.TFExampleDecoder(
        data_fields_to_features, item_decoders)

    decode_items = data_items_to_decode
    if decode_items is None:
      decode_items = list(item_decoders)

    decoded = decoder.decode(example_serialized, items=decode_items)
    return dict(zip(decode_items, decoded))

  with tf.name_scope("examples_in"):
    # Read serialized examples using slim parallel_reader.
    data_files = tf.contrib.slim.parallel_reader.get_data_files(data_sources)
    num_readers = min(4 if training else 1, len(data_files))
    _, example_serialized = tf.contrib.slim.parallel_reader.parallel_read(
        data_sources,
        tf.TFRecordReader,
        num_epochs=None if training else 1,
        shuffle=training,
        capacity=2 * capacity,
        min_after_dequeue=capacity,
        num_readers=num_readers)
    return decode_record(example_serialized)


def preprocessing(examples, data_file_pattern):
  """Preprocessing of examples."""
  # This function is for obsolete problems only, as we're porting them
  # all to the Problem class and its preprocess_examples method. Don't add.
  if "image" in data_file_pattern:
    def resize(img, size):
      return tf.to_int64(tf.image.resize_images(
          img, [size, size], tf.image.ResizeMethod.AREA))

    if "img2img" in data_file_pattern:
      inputs = examples["inputs"]
      examples["inputs"] = resize(inputs, 16)
      examples["targets"] = resize(inputs, 64)
    elif "image_celeba" in data_file_pattern:
      inputs = examples["inputs"]
      # Remove boundaries in CelebA images. Remove 40 pixels each side
      # vertically and 20 pixels each side horizontally.
      inputs = tf.image.crop_to_bounding_box(inputs, 40, 20, 218-80, 178-40)
      examples["inputs"] = resize(inputs, 8)
      examples["targets"] = resize(inputs, 32)
  elif "audio" in data_file_pattern:
    # Reshape audio to proper shape
    sample_count = tf.to_int32(examples.pop("audio/sample_count"))
    sample_width = tf.to_int32(examples.pop("audio/sample_width"))
    channel_count = 1
    examples["inputs"] = tf.reshape(examples["inputs"],
                                    [sample_count, sample_width, channel_count])
    if "wsj" in data_file_pattern:
      examples["inputs"] = tf.bitcast(examples["inputs"], tf.int32)
  elif "a2q_20161229" in data_file_pattern:
    # we forgot the EOS when we preprocessed this data.
    examples["targets"] = tf.concat([examples["targets"], [1]], 0)
  return examples


def cast_int64_to_int32(features):
{'fraction_non_alphanumeric': 0.08506273179199873, 'fraction_numerical': 0.013572161287777164, 'mean_word_length': 3.6003629764065335, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4873057', 'n_tokens_mistral': 3875, 'n_tokens_neox': 3672, 'n_words': 1175}