# Path: cptm/utils/experiment.py
# def load_config(fName):
#     with open(fName) as f:
#         config = json.load(f)
# 
#     logger.debug('configuration of experiment: ')
#     params = ['{}: {}'.format(p, v) for p, v in config.iteritems()]
#     for p in params:
#         logger.debug(p)
# 
#     params = {}
#     params['inputData'] = config.get('inputData')
#     params['outDir'] = config.get('outDir', '/{}')
#     params['testSplit'] = config.get('testSplit', 20)
#     params['minFreq'] = config.get('minFreq')
#     params['removeTopTF'] = config.get('removeTopTF')
#     params['removeTopDF'] = config.get('removeTopDF')
#     params['nIter'] = config.get('nIter', 200)
#     params['beta'] = config.get('beta', 0.02)
#     params['beta_o'] = config.get('beta_o', 0.02)
#     params['expNumTopics'] = config.get('expNumTopics', range(20, 201, 20))
#     params['nTopics'] = config.get('nTopics')
#     params['nProcesses'] = config.get('nProcesses', None)
#     params['topicLines'] = config.get('topicLines', [0])
#     params['opinionLines'] = config.get('opinionLines', [1])
#     params['sampleEstimateStart'] = config.get('sampleEstimateStart')
#     params['sampleEstimateEnd'] = config.get('sampleEstimateEnd')
# 
#     return params
# 
# def get_corpus(params):
#     out_dir = params.get('outDir')
#     files = glob(params.get('inputData'))
# 
#     if not os.path.isfile(out_dir.format('corpus.json')):
#         corpus = CPTCorpus(files,
#                            testSplit=params.get('testSplit'),
#                            topicLines=params.get('topicLines'),
#                            opinionLines=params.get('opinionLines'))
#         minFreq = params.get('minFreq')
#         removeTopTF = params.get('removeTopTF')
#         removeTopDF = params.get('removeTopDF')
#         if (not minFreq is None) or (not removeTopTF is None) or \
#            (not removeTopDF is None):
#             corpus.filter_dictionaries(minFreq=minFreq,
#                                        removeTopTF=removeTopTF,
#                                        removeTopDF=removeTopDF)
#         corpus.save_dictionaries(directory=out_dir.format(''))
#         corpus.save(out_dir.format('corpus.json'))
#     else:
#         corpus = CPTCorpus.load(file_name=out_dir.format('corpus.json'),
#                                 topicLines=params.get('topicLines'),
#                                 opinionLines=params.get('opinionLines'),
#                                 topicDict=out_dir.format('topicDict.dict'),
#                                 opinionDict=out_dir.format('opinionDict.dict'))
#     return corpus
# 
# def get_sampler(params, corpus, nTopics=None, initialize=True):
#     if nTopics is None:
#         nTopics = params.get('nTopics')
#     out_dir = params.get('outDir')
#     nIter = params.get('nIter')
#     alpha = 50.0/nTopics
#     beta = params.get('beta')
#     beta_o = params.get('beta_o')
#     logger.info('creating Gibbs sampler (nTopics: {}, nIter: {}, alpha: {}, '
#                 'beta: {}, beta_o: {})'.format(nTopics, nIter, alpha, beta,
#                                                beta_o))
#     sampler = GibbsSampler(corpus, nTopics=nTopics, nIter=nIter,
#                            alpha=alpha, beta=beta, beta_o=beta_o,
#                            out_dir=out_dir.format(nTopics),
#                            initialize=initialize)
#     return sampler


import logging
import argparse
from os import remove
from cptm.utils.experiment import load_config, get_corpus, get_sampler
"""Remove saved parameter samples for certain iterations

Before, the Gibbs sampler saved estimates for all iterations. However, because
this took to much disk space, now the sampler only saves every tenth estimate.
This script removes samples for results generated with an old version of the
sampler.

Usage: python experiment_prune_samples.py /path/to/experiment.json
"""



logger = logging.getLogger(__name__)
logging.basicConfig(format='%(levelname)s : %(message)s', level=logging.INFO)

logging.getLogger('gensim').setLevel(logging.ERROR)
logging.getLogger('CPTCorpus').setLevel(logging.ERROR)
logging.getLogger('CPT_Gibbs').setLevel(logging.ERROR)

parser = argparse.ArgumentParser()
parser.add_argument('json', help='json file containing experiment '
                    'configuration.')
args = parser.parse_args()


# Path: cptm/utils/experiment.py
# def load_config(fName):
#     with open(fName) as f:
#         config = json.load(f)
# 
#     logger.debug('configuration of experiment: ')
#     params = ['{}: {}'.format(p, v) for p, v in config.iteritems()]
#     for p in params:
#         logger.debug(p)
# 
#     params = {}
#     params['inputData'] = config.get('inputData')
#     params['outDir'] = config.get('outDir', '/{}')
#     params['testSplit'] = config.get('testSplit', 20)
#     params['minFreq'] = config.get('minFreq')
#     params['removeTopTF'] = config.get('removeTopTF')
#     params['removeTopDF'] = config.get('removeTopDF')
#     params['nIter'] = config.get('nIter', 200)
#     params['beta'] = config.get('beta', 0.02)
#     params['beta_o'] = config.get('beta_o', 0.02)
#     params['expNumTopics'] = config.get('expNumTopics', range(20, 201, 20))
#     params['nTopics'] = config.get('nTopics')
#     params['nProcesses'] = config.get('nProcesses', None)
#     params['topicLines'] = config.get('topicLines', [0])
#     params['opinionLines'] = config.get('opinionLines', [1])
#     params['sampleEstimateStart'] = config.get('sampleEstimateStart')
#     params['sampleEstimateEnd'] = config.get('sampleEstimateEnd')
# 
#     return params
# 
# def get_corpus(params):
#     out_dir = params.get('outDir')
#     files = glob(params.get('inputData'))
# 
#     if not os.path.isfile(out_dir.format('corpus.json')):
#         corpus = CPTCorpus(files,
#                            testSplit=params.get('testSplit'),
#                            topicLines=params.get('topicLines'),
#                            opinionLines=params.get('opinionLines'))
#         minFreq = params.get('minFreq')
#         removeTopTF = params.get('removeTopTF')
#         removeTopDF = params.get('removeTopDF')
#         if (not minFreq is None) or (not removeTopTF is None) or \
#            (not removeTopDF is None):
#             corpus.filter_dictionaries(minFreq=minFreq,
#                                        removeTopTF=removeTopTF,
#                                        removeTopDF=removeTopDF)
#         corpus.save_dictionaries(directory=out_dir.format(''))
#         corpus.save(out_dir.format('corpus.json'))
#     else:
#         corpus = CPTCorpus.load(file_name=out_dir.format('corpus.json'),
#                                 topicLines=params.get('topicLines'),
#                                 opinionLines=params.get('opinionLines'),
#                                 topicDict=out_dir.format('topicDict.dict'),
#                                 opinionDict=out_dir.format('opinionDict.dict'))
#     return corpus
# 
# def get_sampler(params, corpus, nTopics=None, initialize=True):
#     if nTopics is None:
#         nTopics = params.get('nTopics')
#     out_dir = params.get('outDir')
#     nIter = params.get('nIter')
#     alpha = 50.0/nTopics
#     beta = params.get('beta')
#     beta_o = params.get('beta_o')
#     logger.info('creating Gibbs sampler (nTopics: {}, nIter: {}, alpha: {}, '
#                 'beta: {}, beta_o: {})'.format(nTopics, nIter, alpha, beta,
#                                                beta_o))
#     sampler = GibbsSampler(corpus, nTopics=nTopics, nIter=nIter,
#                            alpha=alpha, beta=beta, beta_o=beta_o,
#                            out_dir=out_dir.format(nTopics),
#                            initialize=initialize)
#     return sampler


# Path: cptm/experiment_prune_samples.py
import logging
import argparse
from os import remove
from cptm.utils.experiment import load_config, get_corpus, get_sampler

"""Remove saved parameter samples for certain iterations

Before, the Gibbs sampler saved estimates for all iterations. However, because
this took to much disk space, now the sampler only saves every tenth estimate.
This script removes samples for results generated with an old version of the
sampler.

Usage: python experiment_prune_samples.py /path/to/experiment.json
"""



logger = logging.getLogger(__name__)
logging.basicConfig(format='%(levelname)s : %(message)s', level=logging.INFO)

logging.getLogger('gensim').setLevel(logging.ERROR)
logging.getLogger('CPTCorpus').setLevel(logging.ERROR)
logging.getLogger('CPT_Gibbs').setLevel(logging.ERROR)

parser = argparse.ArgumentParser()
parser.add_argument('json', help='json file containing experiment '
                    'configuration.')
args = parser.parse_args()


config = load_config(args.json)