# Path: autoencoder/core/ae.py
# class AutoEncoder(object):
#     def __init__(self, input_size, dim, comp_topk=None, ctype=None, save_model='best_model'):
#     def build(self):
#     def fit(self, train_X, val_X, nb_epoch=50, batch_size=100, contractive=None):
# def save_ae_model(model, model_file):
# def load_ae_model(model_file):
# 
# Path: autoencoder/preprocessing/preprocessing.py
# def load_corpus(corpus_path):
#     corpus = load_json(corpus_path)
# 
#     return corpus
# 
# def doc2vec(doc, dim):
#     vec = np.zeros(dim)
#     for idx, val in doc.items():
#         vec[int(idx)] = val
# 
#     return vec
# 
# def vocab_weights(vocab_dict, word_freq, max_=100., ratio=.75):
#     weights = np.zeros((len(vocab_dict), 1))
# 
#     for word, idx in vocab_dict.items():
#         weights[idx] = word_freq[str(idx)]
#     weights = np.clip(weights / max_, 0., 1.)
# 
#     return np.power(weights, ratio)
# 
# Path: autoencoder/utils/op_utils.py
# def vecnorm(vec, norm, epsilon=1e-3):
#     """
#     Scale a vector to unit length. The only exception is the zero vector, which
#     is returned back unchanged.
#     """
#     if norm not in ('prob', 'max1', 'logmax1'):
#         raise ValueError("'%s' is not a supported norm. Currently supported norms include 'prob',\
#              'max1' and 'logmax1'." % norm)
# 
#     if isinstance(vec, np.ndarray):
#         vec = np.asarray(vec, dtype=float)
#         if norm == 'prob':
#             veclen = np.sum(np.abs(vec)) + epsilon * len(vec) # smoothing
#         elif norm == 'max1':
#             veclen = np.max(vec) + epsilon
#         elif norm == 'logmax1':
#             vec = np.log10(1. + vec)
#             veclen = np.max(vec) + epsilon
#         if veclen > 0.0:
#             return (vec + epsilon) / veclen
#         else:
#             return vec
#     else:
#         raise ValueError('vec should be ndarray, found: %s' % type(vec))
# 
# def add_gaussian_noise(X, corruption_ratio, range_=[0, 1]):
#     X_noisy = X + corruption_ratio * np.random.normal(loc=0.0, scale=1.0, size=X.shape)
#     X_noisy = np.clip(X_noisy, range_[0], range_[1])
# 
#     return X_noisy
# 
# def add_masking_noise(X, fraction):
#     assert fraction >= 0 and fraction <= 1
#     X_noisy = np.copy(X)
#     nrow, ncol = X.shape
#     n = int(ncol * fraction)
#     for i in range(nrow):
#         idx_noisy  = np.random.choice(ncol, n, replace=False)
#         X_noisy[i, idx_noisy] = 0
# 
#     return X_noisy
# 
# def add_salt_pepper_noise(X, fraction):
#     assert fraction >= 0 and fraction <= 1
#     X_noisy = np.copy(X)
#     nrow, ncol = X.shape
#     n = int(ncol * fraction)
#     for i in range(nrow):
#         idx_noisy  = np.random.choice(ncol, n, replace=False)
#         X_noisy[i, idx_noisy] = np.random.binomial(1, .5, n)
# 
#     return X_noisy
# 
# Path: autoencoder/utils/io_utils.py
# def dump_json(data, file):
#     try:
#         with open(file, 'w') as datafile:
#             json.dump(data, datafile)
#     except Exception as e:
#         raise e


import timeit
import argparse
import numpy as np
from os import path
from autoencoder.core.ae import AutoEncoder, load_model, save_model
from autoencoder.preprocessing.preprocessing import load_corpus, doc2vec, vocab_weights
from autoencoder.utils.op_utils import vecnorm, add_gaussian_noise, add_masking_noise, add_salt_pepper_noise
from autoencoder.utils.io_utils import dump_json
'''
Created on Nov, 2016

@author: hugo

'''
from __future__ import absolute_import



def train(args):
    corpus = load_corpus(args.input)
    n_vocab, docs = len(corpus['vocab']), corpus['docs']
    corpus.clear() # save memory

    doc_keys = docs.keys()
    X_docs = []
    for k in doc_keys:
        X_docs.append(vecnorm(doc2vec(docs[k], n_vocab), 'logmax1', 0))
        del docs[k]
    X_docs = np.r_[X_docs]

    if args.noise == 'gs':
        X_docs_noisy = add_gaussian_noise(X_docs, 0.1)
    elif args.noise == 'sp':

# Path: autoencoder/core/ae.py
# class AutoEncoder(object):
#     def __init__(self, input_size, dim, comp_topk=None, ctype=None, save_model='best_model'):
#     def build(self):
#     def fit(self, train_X, val_X, nb_epoch=50, batch_size=100, contractive=None):
# def save_ae_model(model, model_file):
# def load_ae_model(model_file):
# 
# Path: autoencoder/preprocessing/preprocessing.py
# def load_corpus(corpus_path):
#     corpus = load_json(corpus_path)
# 
#     return corpus
# 
# def doc2vec(doc, dim):
#     vec = np.zeros(dim)
#     for idx, val in doc.items():
#         vec[int(idx)] = val
# 
#     return vec
# 
# def vocab_weights(vocab_dict, word_freq, max_=100., ratio=.75):
#     weights = np.zeros((len(vocab_dict), 1))
# 
#     for word, idx in vocab_dict.items():
#         weights[idx] = word_freq[str(idx)]
#     weights = np.clip(weights / max_, 0., 1.)
# 
#     return np.power(weights, ratio)
# 
# Path: autoencoder/utils/op_utils.py
# def vecnorm(vec, norm, epsilon=1e-3):
#     """
#     Scale a vector to unit length. The only exception is the zero vector, which
#     is returned back unchanged.
#     """
#     if norm not in ('prob', 'max1', 'logmax1'):
#         raise ValueError("'%s' is not a supported norm. Currently supported norms include 'prob',\
#              'max1' and 'logmax1'." % norm)
# 
#     if isinstance(vec, np.ndarray):
#         vec = np.asarray(vec, dtype=float)
#         if norm == 'prob':
#             veclen = np.sum(np.abs(vec)) + epsilon * len(vec) # smoothing
#         elif norm == 'max1':
#             veclen = np.max(vec) + epsilon
#         elif norm == 'logmax1':
#             vec = np.log10(1. + vec)
#             veclen = np.max(vec) + epsilon
#         if veclen > 0.0:
#             return (vec + epsilon) / veclen
#         else:
#             return vec
#     else:
#         raise ValueError('vec should be ndarray, found: %s' % type(vec))
# 
# def add_gaussian_noise(X, corruption_ratio, range_=[0, 1]):
#     X_noisy = X + corruption_ratio * np.random.normal(loc=0.0, scale=1.0, size=X.shape)
#     X_noisy = np.clip(X_noisy, range_[0], range_[1])
# 
#     return X_noisy
# 
# def add_masking_noise(X, fraction):
#     assert fraction >= 0 and fraction <= 1
#     X_noisy = np.copy(X)
#     nrow, ncol = X.shape
#     n = int(ncol * fraction)
#     for i in range(nrow):
#         idx_noisy  = np.random.choice(ncol, n, replace=False)
#         X_noisy[i, idx_noisy] = 0
# 
#     return X_noisy
# 
# def add_salt_pepper_noise(X, fraction):
#     assert fraction >= 0 and fraction <= 1
#     X_noisy = np.copy(X)
#     nrow, ncol = X.shape
#     n = int(ncol * fraction)
#     for i in range(nrow):
#         idx_noisy  = np.random.choice(ncol, n, replace=False)
#         X_noisy[i, idx_noisy] = np.random.binomial(1, .5, n)
# 
#     return X_noisy
# 
# Path: autoencoder/utils/io_utils.py
# def dump_json(data, file):
#     try:
#         with open(file, 'w') as datafile:
#             json.dump(data, datafile)
#     except Exception as e:
#         raise e


# Path: legacy_train.py
import timeit
import argparse
import numpy as np
from os import path
from autoencoder.core.ae import AutoEncoder, load_model, save_model
from autoencoder.preprocessing.preprocessing import load_corpus, doc2vec, vocab_weights
from autoencoder.utils.op_utils import vecnorm, add_gaussian_noise, add_masking_noise, add_salt_pepper_noise
from autoencoder.utils.io_utils import dump_json

'''
Created on Nov, 2016

@author: hugo

'''
from __future__ import absolute_import



def train(args):
    corpus = load_corpus(args.input)
    n_vocab, docs = len(corpus['vocab']), corpus['docs']
    corpus.clear() # save memory

    doc_keys = docs.keys()
    X_docs = []
    for k in doc_keys:
        X_docs.append(vecnorm(doc2vec(docs[k], n_vocab), 'logmax1', 0))
        del docs[k]
    X_docs = np.r_[X_docs]

    if args.noise == 'gs':
        X_docs_noisy = add_gaussian_noise(X_docs, 0.1)
    elif args.noise == 'sp':

        X_docs_noisy = add_salt_pepper_noise(X_docs, 0.1)