# Path: datasets/Dataset.py
# class FileListDataset(AbstractDataset):
#   def __init__(self, config, dataset_name, subset, default_path, num_classes):
#     super().__init__(config, subset, num_classes)
#     self.inputfile_lists = None
#     self.fraction = config.float("data_fraction", 1.0)
#     self.data_dir = config.string(dataset_name + "_data_dir", default_path)
#     self._num_parallel_calls = config.int("num_parallel_calls", 32)
#     self._prefetch_buffer_size = config.int("prefetch_buffer_size", 20)
# 
#   def _load_inputfile_lists(self):
#     if self.inputfile_lists is not None:
#       return
#     self.inputfile_lists = self.read_inputfile_lists()
#     assert len(self.inputfile_lists) > 0
#     for l in self.inputfile_lists:
#       assert len(l) > 0
#     # make sure all lists have the same length
#     assert all([len(l) == len(self.inputfile_lists[0]) for l in self.inputfile_lists])
#     if self.fraction < 1.0:
#       n = int(self.fraction * len(self.inputfile_lists[0]))
#       self.inputfile_lists = tuple([l[:n] for l in self.inputfile_lists])
# 
#   def n_examples_per_epoch(self):
#     self._load_inputfile_lists()
#     n_examples = super().n_examples_per_epoch()
#     if n_examples is None:
#       return len(self.inputfile_lists[0])
#     else:
#       return n_examples
# 
#   def create_input_tensors_dict(self, batch_size):
#     self._load_inputfile_lists()
#     if self.subset == "train":
#       # shuffle lists together, for this zip, shuffle, and unzip
#       zipped = list(zip(*self.inputfile_lists))
#       shuffle(zipped)
#       inputfile_lists_shuffled = tuple([x[idx] for x in zipped] for idx in range(len(self.inputfile_lists)))
#     else:
#       inputfile_lists_shuffled = self.inputfile_lists
#     tfdata = tf.data.Dataset.from_tensor_slices(inputfile_lists_shuffled)
#     if self.subset == "train":
#       tfdata = tfdata.shuffle(buffer_size=self.shuffle_buffer_size)
# 
#     def _load_example(*input_filenames):
#       example = self.load_example(input_filenames)
#       # this has different sizes and therefore cannot be batched
#       if batch_size > 1:
#         if DataKeys.SEGMENTATION_LABELS_ORIGINAL_SIZE in example:
#           del example[DataKeys.SEGMENTATION_LABELS_ORIGINAL_SIZE]
#         if DataKeys.RAW_IMAGES in example:
#           del example[DataKeys.RAW_IMAGES]
#       return example
# 
#     def _filter_example(tensors):
#       if DataKeys.SKIP_EXAMPLE in tensors:
#         return tf.logical_not(tensors[DataKeys.SKIP_EXAMPLE])
#       else:
#         return tf.constant(True)
# 
#     tfdata = tfdata.map(_load_example, num_parallel_calls=self._num_parallel_calls)
#     tfdata = tfdata.filter(_filter_example)
#     tfdata = tfdata.repeat()
#     tfdata = self._batch(tfdata, batch_size)
#     tfdata = tfdata.prefetch(buffer_size=self._prefetch_buffer_size)
#     # TODO: maybe we can improve the performance like this
#     #tf.contrib.data.prefetch_to_device("/gpu:0", self._prefetch_buffer_size)
#     res = tfdata.make_one_shot_iterator().get_next()
# 
#     if self.use_summaries:
#       self.create_summaries(res)
#     return res
# 
#   def _batch(self, tfdata, batch_size):
#     if batch_size > 1:
#       tfdata = tfdata.batch(batch_size, drop_remainder=True)
#     elif batch_size == 1:
#       # like this we are able to retain the batch size in the shape information
#       tfdata = tfdata.map(lambda x: {k: tf.expand_dims(v, axis=0) for k, v in x.items()})
#     else:
#       assert False, ("invalid batch size", batch_size)
#     return tfdata
# 
#   # Override to add extraction keys that will be used by trainer.
#   def get_extraction_keys(self):
#     return []
# 
#   @abstractmethod
#   def read_inputfile_lists(self):
#     raise NotImplementedError
# 
# Path: core/Log.py
# class Stream:
# class Log(object):
#   def __init__(self, log, lvl):
#   def write(self, msg):
#   def flush(self):
#   def initialize(self, logs=[], verbosity=[], formatter=[]):
#   def write(self, msg):


from random import shuffle
from datasets.Dataset import FileListDataset
from core.Log import log
import tensorflow as tf



NUM_CLASSES = 2


class MapillaryLikeInstanceDataset(FileListDataset):
  def __init__(self, config, subset, name, default_path, data_list_path, id_divisor, cat_ids_to_use):
    super().__init__(config, name, subset, default_path, NUM_CLASSES)
    self.validation_set_size = self.config.int("validation_set_size", -1)
    # note: in case of mapillary the min sizes are always based on the sizes in quarter resolution!
    self.min_size = config.int("min_size", 0)
    self._cat_ids_to_use = cat_ids_to_use
    self._data_list_path = data_list_path
    self._id_divisor = id_divisor
    self.name = name

  def read_inputfile_lists(self):
    data_list = "training.txt" if self.subset == "train" else "validation.txt"

# Path: datasets/Dataset.py
# class FileListDataset(AbstractDataset):
#   def __init__(self, config, dataset_name, subset, default_path, num_classes):
#     super().__init__(config, subset, num_classes)
#     self.inputfile_lists = None
#     self.fraction = config.float("data_fraction", 1.0)
#     self.data_dir = config.string(dataset_name + "_data_dir", default_path)
#     self._num_parallel_calls = config.int("num_parallel_calls", 32)
#     self._prefetch_buffer_size = config.int("prefetch_buffer_size", 20)
# 
#   def _load_inputfile_lists(self):
#     if self.inputfile_lists is not None:
#       return
#     self.inputfile_lists = self.read_inputfile_lists()
#     assert len(self.inputfile_lists) > 0
#     for l in self.inputfile_lists:
#       assert len(l) > 0
#     # make sure all lists have the same length
#     assert all([len(l) == len(self.inputfile_lists[0]) for l in self.inputfile_lists])
#     if self.fraction < 1.0:
#       n = int(self.fraction * len(self.inputfile_lists[0]))
#       self.inputfile_lists = tuple([l[:n] for l in self.inputfile_lists])
# 
#   def n_examples_per_epoch(self):
#     self._load_inputfile_lists()
#     n_examples = super().n_examples_per_epoch()
#     if n_examples is None:
#       return len(self.inputfile_lists[0])
#     else:
#       return n_examples
# 
#   def create_input_tensors_dict(self, batch_size):
#     self._load_inputfile_lists()
#     if self.subset == "train":
#       # shuffle lists together, for this zip, shuffle, and unzip
#       zipped = list(zip(*self.inputfile_lists))
#       shuffle(zipped)
#       inputfile_lists_shuffled = tuple([x[idx] for x in zipped] for idx in range(len(self.inputfile_lists)))
#     else:
#       inputfile_lists_shuffled = self.inputfile_lists
#     tfdata = tf.data.Dataset.from_tensor_slices(inputfile_lists_shuffled)
#     if self.subset == "train":
#       tfdata = tfdata.shuffle(buffer_size=self.shuffle_buffer_size)
# 
#     def _load_example(*input_filenames):
#       example = self.load_example(input_filenames)
#       # this has different sizes and therefore cannot be batched
#       if batch_size > 1:
#         if DataKeys.SEGMENTATION_LABELS_ORIGINAL_SIZE in example:
#           del example[DataKeys.SEGMENTATION_LABELS_ORIGINAL_SIZE]
#         if DataKeys.RAW_IMAGES in example:
#           del example[DataKeys.RAW_IMAGES]
#       return example
# 
#     def _filter_example(tensors):
#       if DataKeys.SKIP_EXAMPLE in tensors:
#         return tf.logical_not(tensors[DataKeys.SKIP_EXAMPLE])
#       else:
#         return tf.constant(True)
# 
#     tfdata = tfdata.map(_load_example, num_parallel_calls=self._num_parallel_calls)
#     tfdata = tfdata.filter(_filter_example)
#     tfdata = tfdata.repeat()
#     tfdata = self._batch(tfdata, batch_size)
#     tfdata = tfdata.prefetch(buffer_size=self._prefetch_buffer_size)
#     # TODO: maybe we can improve the performance like this
#     #tf.contrib.data.prefetch_to_device("/gpu:0", self._prefetch_buffer_size)
#     res = tfdata.make_one_shot_iterator().get_next()
# 
#     if self.use_summaries:
#       self.create_summaries(res)
#     return res
# 
#   def _batch(self, tfdata, batch_size):
#     if batch_size > 1:
#       tfdata = tfdata.batch(batch_size, drop_remainder=True)
#     elif batch_size == 1:
#       # like this we are able to retain the batch size in the shape information
#       tfdata = tfdata.map(lambda x: {k: tf.expand_dims(v, axis=0) for k, v in x.items()})
#     else:
#       assert False, ("invalid batch size", batch_size)
#     return tfdata
# 
#   # Override to add extraction keys that will be used by trainer.
#   def get_extraction_keys(self):
#     return []
# 
#   @abstractmethod
#   def read_inputfile_lists(self):
#     raise NotImplementedError
# 
# Path: core/Log.py
# class Stream:
# class Log(object):
#   def __init__(self, log, lvl):
#   def write(self, msg):
#   def flush(self):
#   def initialize(self, logs=[], verbosity=[], formatter=[]):
#   def write(self, msg):


# Path: datasets/Mapillary/MapillaryLike_instance.py
from random import shuffle
from datasets.Dataset import FileListDataset
from core.Log import log
import tensorflow as tf




NUM_CLASSES = 2


class MapillaryLikeInstanceDataset(FileListDataset):
  def __init__(self, config, subset, name, default_path, data_list_path, id_divisor, cat_ids_to_use):
    super().__init__(config, name, subset, default_path, NUM_CLASSES)
    self.validation_set_size = self.config.int("validation_set_size", -1)
    # note: in case of mapillary the min sizes are always based on the sizes in quarter resolution!
    self.min_size = config.int("min_size", 0)
    self._cat_ids_to_use = cat_ids_to_use
    self._data_list_path = data_list_path
    self._id_divisor = id_divisor
    self.name = name

  def read_inputfile_lists(self):
    data_list = "training.txt" if self.subset == "train" else "validation.txt"

    print("{} ({}): using data_dir:".format(self.name, self.subset), self.data_dir, file=log.v5)