<commit_before>from __future__ import absolute_import
from __future__ import print_function
from __future__ import division

import tensorflow as tf
import numpy as np

import os
import sys
import time

import sagan_model as sagan

sys.path.append('../')
import tfutil as t
import image_utils as iu

from config import get_config
from datasets import DataIterator
from datasets import CelebADataSet as DataSet


cfg, _ = get_config()


train_step = {
    'epochs': 11,
    'batch_size': 64,
    'global_step': 10001,
    'logging_interval': 500,
}


def main():
    start_time = time.time()  # Clocking start

    height, width, channel = 128, 128, 3

    # loading CelebA DataSet # from 'raw images' or 'h5'
    use_h5 = True
    if not use_h5:
        ds = DataSet(height=height,
                     width=height,
                     channel=channel,
                     # ds_image_path="D:\\DataSet/CelebA/CelebA-%d.h5" % height,
                     ds_label_path=os.path.join(cfg.celeba, "Anno/list_attr_celeba.txt"),
                     ds_image_path=os.path.join(cfg.celeba, "Img/img_align_celeba/"),
                     ds_type="CelebA",
                     use_save=True,
                     save_file_name=os.path.join(cfg.celeba, "CelebA-%d.h5" % height),
                     save_type="to_h5",
                     use_img_scale=False,
                     )
    else:
        ds = DataSet(height=height,
                     width=height,
                     channel=channel,
                     ds_image_path=os.path.join(cfg.celeba, "CelebA-%d.h5" % height),
                     ds_label_path=os.path.join(cfg.celeba, "Anno/list_attr_celeba.txt"),
                     # ds_image_path=os.path.join(cfg.celeba, "Img/img_align_celeba/"),
                     ds_type="CelebA",
                     use_save=False,
                     # save_file_name=os.path.join(cfg.celeba, "CelebA-%d.h5" % height),
                     # save_type="to_h5",
                     use_img_scale=False,
                     )

    # saving sample images
    test_images = np.reshape(iu.transform(ds.images[:16], inv_type='127'), (16, height, width, channel))
    iu.save_images(test_images,
                   size=[4, 4],
                   image_path=os.path.join(cfg.output, "sample.png"),
                   inv_type='127')

    ds_iter = DataIterator(x=ds.images,
                           y=None,
                           batch_size=train_step['batch_size'],
                           label_off=True)

    del ds

    # GPU configure
    config = tf.ConfigProto()
    config.gpu_options.allow_growth = True

    with tf.Session(config=config) as s:
        # SAGAN Model
        model = sagan.SAGAN(s,
                            height=height, width=width, channel=channel,
                            batch_size=train_step['batch_size'],
                            use_gp=False,
                            use_hinge_loss=True
                            )

        # Initializing
        s.run(tf.global_variables_initializer())

        print("[*] Reading checkpoints...")

        saved_global_step = 0
        ckpt = tf.train.get_checkpoint_state(cfg.model_path)
        if ckpt and ckpt.model_checkpoint_path:
            # Restores from checkpoint
            model.saver.restore(s, ckpt.model_checkpoint_path)

            saved_global_step = int(ckpt.model_checkpoint_path.split('/')[-1].split('-')[-1])
            print("[+] global step : %d" % saved_global_step, " successfully loaded")
        else:
            print('[-] No checkpoint file found')

        global_step = saved_global_step
        start_epoch = global_step // (ds.num_images // model.batch_size)           # recover n_epoch
        ds_iter.pointer = saved_global_step % (ds.num_images // model.batch_size)  # recover n_iter
        for epoch in range(start_epoch, train_step['epochs']):
            for batch_x in ds_iter.iterate():
                batch_x = iu.transform(batch_x, inv_type='127')
                batch_x = np.reshape(batch_x, (model.batch_size, model.height, model.width, model.channel))
                batch_z = np.random.uniform(-1., 1., [model.batch_size, model.z_dim]).astype(np.float32)

                # Update D network
                _, d_loss = s.run([model.d_op, model.d_loss],
                                  feed_dict={
                                      model.x: batch_x,
                                      model.z: batch_z,
                                  })<commit_msg>fix: assign vars before deleting obj<commit_after>from __future__ import absolute_import
from __future__ import print_function
from __future__ import division

import tensorflow as tf
import numpy as np

import os
import sys
import time

import sagan_model as sagan

sys.path.append('../')
import tfutil as t
import image_utils as iu

from config import get_config
from datasets import DataIterator
from datasets import CelebADataSet as DataSet


cfg, _ = get_config()


train_step = {
    'epochs': 11,
    'batch_size': 64,
    'global_step': 10001,
    'logging_interval': 500,
}


def main():
    start_time = time.time()  # Clocking start

    height, width, channel = 128, 128, 3

    # loading CelebA DataSet # from 'raw images' or 'h5'
    use_h5 = True
    if not use_h5:
        ds = DataSet(height=height,
                     width=height,
                     channel=channel,
                     # ds_image_path="D:\\DataSet/CelebA/CelebA-%d.h5" % height,
                     ds_label_path=os.path.join(cfg.celeba, "Anno/list_attr_celeba.txt"),
                     ds_image_path=os.path.join(cfg.celeba, "Img/img_align_celeba/"),
                     ds_type="CelebA",
                     use_save=True,
                     save_file_name=os.path.join(cfg.celeba, "CelebA-%d.h5" % height),
                     save_type="to_h5",
                     use_img_scale=False,
                     )
    else:
        ds = DataSet(height=height,
                     width=height,
                     channel=channel,
                     ds_image_path=os.path.join(cfg.celeba, "CelebA-%d.h5" % height),
                     ds_label_path=os.path.join(cfg.celeba, "Anno/list_attr_celeba.txt"),
                     # ds_image_path=os.path.join(cfg.celeba, "Img/img_align_celeba/"),
                     ds_type="CelebA",
                     use_save=False,
                     # save_file_name=os.path.join(cfg.celeba, "CelebA-%d.h5" % height),
                     # save_type="to_h5",
                     use_img_scale=False,
                     )

    num_images = ds.num_images

    # saving sample images
    test_images = np.reshape(iu.transform(ds.images[:16], inv_type='127'), (16, height, width, channel))
    iu.save_images(test_images,
                   size=[4, 4],
                   image_path=os.path.join(cfg.output, "sample.png"),
                   inv_type='127')

    ds_iter = DataIterator(x=ds.images,
                           y=None,
                           batch_size=train_step['batch_size'],
                           label_off=True)

    del ds

    # GPU configure
    config = tf.ConfigProto()
    config.gpu_options.allow_growth = True

    with tf.Session(config=config) as s:
        # SAGAN Model
        model = sagan.SAGAN(s,
                            height=height, width=width, channel=channel,
                            batch_size=train_step['batch_size'],
                            use_gp=False,
                            use_hinge_loss=True
                            )

        # Initializing
        s.run(tf.global_variables_initializer())

        print("[*] Reading checkpoints...")

        saved_global_step = 0
        ckpt = tf.train.get_checkpoint_state(cfg.model_path)
        if ckpt and ckpt.model_checkpoint_path:
            # Restores from checkpoint
            model.saver.restore(s, ckpt.model_checkpoint_path)

            saved_global_step = int(ckpt.model_checkpoint_path.split('/')[-1].split('-')[-1])
            print("[+] global step : %d" % saved_global_step, " successfully loaded")
        else:
            print('[-] No checkpoint file found')

        global_step = saved_global_step
        start_epoch = global_step // (num_images // model.batch_size)           # recover n_epoch
        ds_iter.pointer = saved_global_step % (num_images // model.batch_size)  # recover n_iter
        for epoch in range(start_epoch, train_step['epochs']):
            for batch_x in ds_iter.iterate():
                batch_x = iu.transform(batch_x, inv_type='127')
                batch_x = np.reshape(batch_x, (model.batch_size, model.height, model.width, model.channel))
                batch_z = np.random.uniform(-1., 1., [model.batch_size, model.z_dim]).astype(np.float32)

                # Update D network
                _, d_loss = s.run([model.d_op, model.d_loss],
                                  feed_dict={
                                      model.x: batch_x,
                                      model.z: batch_z,
                                  })
{'fraction_non_alphanumeric': 0.10386606454455337, 'fraction_numerical': 0.011014428901861438, 'mean_word_length': 1.684008276677505, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1160959', 'n_tokens_mistral': 2866, 'n_tokens_neox': 2683, 'n_words': 505}