# Path: libml/data.py
# _DATA_CACHE = None
# DATA_DIR = os.environ['ML_DATA']
# FLAGS = flags.FLAGS
# DATASETS = {}
# def record_parse(serialized_example):
# def default_parse(dataset: tf.data.Dataset, parse_fn=record_parse) -> tf.data.Dataset:
# def dataset(filenames: list) -> tf.data.Dataset:
# def memoize(dataset: tf.data.Dataset) -> tf.data.Dataset:
#     def tf_get(index):
#         def get(index):
# def augment_mirror(x):
# def augment_shift(x, w):
# def augment_noise(x, std):
# def compute_mean_std(data: tf.data.Dataset):
#         def iterator():
#     def __init__(self, name, train_labeled, train_unlabeled, test, valid, eval_labeled, eval_unlabeled,
#                  height=32, width=32, colors=3, nclass=10, mean=0, std=1, p_labeled=None, p_unlabeled=None):
#     def creator(cls, name, seed, label, valid, augment, parse_fn=default_parse, do_memoize=True, colors=3,
#                 nclass=10, height=32, width=32, name_suffix=''):
#         def create():
# class DataSet:
# 
# Path: libml/utils.py
# _GPUS = None
# FLAGS = flags.FLAGS
#         _GPUS = tuple([x.name for x in local_device_protos if x.device_type == 'GPU'])
# def get_config():
# def setup_tf():
# def smart_shape(x):
# def ilog2(x):
# def find_latest_checkpoint(dir, glob_term='model.ckpt-*.meta'):
# def get_latest_global_step(dir):
# def get_latest_global_step_in_subdir(dir):
# def getter_ema(ema, getter, name, *args, **kwargs):
# def model_vars(scope=None):
# def gpu(x):
# def get_available_gpus():
# def average_gradients(tower_grads):
# def para_list(fn, *args):
# def para_mean(fn, *args):
# def para_cat(fn, *args):


import os
from absl import flags
from libml import data, utils
# Copyright 2019 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.



FLAGS = flags.FLAGS


class DataSetFS(data.DataSet):
    @classmethod
    def creator(cls, name, train_files, test_files, valid, augment, parse_fn=data.default_parse, do_memoize=True,
                colors=3, nclass=10, height=32, width=32):
        train_files = [os.path.join(data.DATA_DIR, x) for x in train_files]
        test_files = [os.path.join(data.DATA_DIR, x) for x in test_files]
        fn = data.memoize if do_memoize else lambda x: x.repeat().shuffle(FLAGS.shuffle)

        def create():
            para = max(1, len(utils.get_available_gpus())) * FLAGS.para_augment

# Path: libml/data.py
# _DATA_CACHE = None
# DATA_DIR = os.environ['ML_DATA']
# FLAGS = flags.FLAGS
# DATASETS = {}
# def record_parse(serialized_example):
# def default_parse(dataset: tf.data.Dataset, parse_fn=record_parse) -> tf.data.Dataset:
# def dataset(filenames: list) -> tf.data.Dataset:
# def memoize(dataset: tf.data.Dataset) -> tf.data.Dataset:
#     def tf_get(index):
#         def get(index):
# def augment_mirror(x):
# def augment_shift(x, w):
# def augment_noise(x, std):
# def compute_mean_std(data: tf.data.Dataset):
#         def iterator():
#     def __init__(self, name, train_labeled, train_unlabeled, test, valid, eval_labeled, eval_unlabeled,
#                  height=32, width=32, colors=3, nclass=10, mean=0, std=1, p_labeled=None, p_unlabeled=None):
#     def creator(cls, name, seed, label, valid, augment, parse_fn=default_parse, do_memoize=True, colors=3,
#                 nclass=10, height=32, width=32, name_suffix=''):
#         def create():
# class DataSet:
# 
# Path: libml/utils.py
# _GPUS = None
# FLAGS = flags.FLAGS
#         _GPUS = tuple([x.name for x in local_device_protos if x.device_type == 'GPU'])
# def get_config():
# def setup_tf():
# def smart_shape(x):
# def ilog2(x):
# def find_latest_checkpoint(dir, glob_term='model.ckpt-*.meta'):
# def get_latest_global_step(dir):
# def get_latest_global_step_in_subdir(dir):
# def getter_ema(ema, getter, name, *args, **kwargs):
# def model_vars(scope=None):
# def gpu(x):
# def get_available_gpus():
# def average_gradients(tower_grads):
# def para_list(fn, *args):
# def para_mean(fn, *args):
# def para_cat(fn, *args):


# Path: fully_supervised/lib/data.py
import os
from absl import flags
from libml import data, utils

# Copyright 2019 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.



FLAGS = flags.FLAGS


class DataSetFS(data.DataSet):
    @classmethod
    def creator(cls, name, train_files, test_files, valid, augment, parse_fn=data.default_parse, do_memoize=True,
                colors=3, nclass=10, height=32, width=32):
        train_files = [os.path.join(data.DATA_DIR, x) for x in train_files]
        test_files = [os.path.join(data.DATA_DIR, x) for x in test_files]
        fn = data.memoize if do_memoize else lambda x: x.repeat().shuffle(FLAGS.shuffle)

        def create():
            para = max(1, len(utils.get_available_gpus())) * FLAGS.para_augment

            train_labeled = parse_fn(data.dataset(train_files).skip(valid))