# Path: cptm/utils/experiment.py
# def get_sampler(params, corpus, nTopics=None, initialize=True):
#     if nTopics is None:
#         nTopics = params.get('nTopics')
#     out_dir = params.get('outDir')
#     nIter = params.get('nIter')
#     alpha = 50.0/nTopics
#     beta = params.get('beta')
#     beta_o = params.get('beta_o')
#     logger.info('creating Gibbs sampler (nTopics: {}, nIter: {}, alpha: {}, '
#                 'beta: {}, beta_o: {})'.format(nTopics, nIter, alpha, beta,
#                                                beta_o))
#     sampler = GibbsSampler(corpus, nTopics=nTopics, nIter=nIter,
#                            alpha=alpha, beta=beta, beta_o=beta_o,
#                            out_dir=out_dir.format(nTopics),
#                            initialize=initialize)
#     return sampler
# 
# def load_config(fName):
#     with open(fName) as f:
#         config = json.load(f)
# 
#     logger.debug('configuration of experiment: ')
#     params = ['{}: {}'.format(p, v) for p, v in config.iteritems()]
#     for p in params:
#         logger.debug(p)
# 
#     params = {}
#     params['inputData'] = config.get('inputData')
#     params['outDir'] = config.get('outDir', '/{}')
#     params['testSplit'] = config.get('testSplit', 20)
#     params['minFreq'] = config.get('minFreq')
#     params['removeTopTF'] = config.get('removeTopTF')
#     params['removeTopDF'] = config.get('removeTopDF')
#     params['nIter'] = config.get('nIter', 200)
#     params['beta'] = config.get('beta', 0.02)
#     params['beta_o'] = config.get('beta_o', 0.02)
#     params['expNumTopics'] = config.get('expNumTopics', range(20, 201, 20))
#     params['nTopics'] = config.get('nTopics')
#     params['nProcesses'] = config.get('nProcesses', None)
#     params['topicLines'] = config.get('topicLines', [0])
#     params['opinionLines'] = config.get('opinionLines', [1])
#     params['sampleEstimateStart'] = config.get('sampleEstimateStart')
#     params['sampleEstimateEnd'] = config.get('sampleEstimateEnd')
# 
#     return params
# 
# def get_corpus(params):
#     out_dir = params.get('outDir')
#     files = glob(params.get('inputData'))
# 
#     if not os.path.isfile(out_dir.format('corpus.json')):
#         corpus = CPTCorpus(files,
#                            testSplit=params.get('testSplit'),
#                            topicLines=params.get('topicLines'),
#                            opinionLines=params.get('opinionLines'))
#         minFreq = params.get('minFreq')
#         removeTopTF = params.get('removeTopTF')
#         removeTopDF = params.get('removeTopDF')
#         if (not minFreq is None) or (not removeTopTF is None) or \
#            (not removeTopDF is None):
#             corpus.filter_dictionaries(minFreq=minFreq,
#                                        removeTopTF=removeTopTF,
#                                        removeTopDF=removeTopDF)
#         corpus.save_dictionaries(directory=out_dir.format(''))
#         corpus.save(out_dir.format('corpus.json'))
#     else:
#         corpus = CPTCorpus.load(file_name=out_dir.format('corpus.json'),
#                                 topicLines=params.get('topicLines'),
#                                 opinionLines=params.get('opinionLines'),
#                                 topicDict=out_dir.format('topicDict.dict'),
#                                 opinionDict=out_dir.format('opinionDict.dict'))
#     return corpus
# 
# def thetaFileName(params):
#     nTopics = params.get('nTopics')
#     return os.path.join(params.get('outDir').format(''),
#                         'theta_{}.csv'.format(nTopics))


import logging
import argparse
import pandas as pd
import os
import sys
from CPTCorpus import CPTCorpus
from cptm.utils.experiment import get_sampler, load_config, get_corpus, \
    thetaFileName
perspectives = [p.name for p in c_perspectives.perspectives]
logger.info('Perspectives found: {}'.format('; '.join(perspectives)))

input_dirs = [args.data_dir for p in perspectives]

corpus = CPTCorpus(input=input_dirs, topicDict=topicDict,
                   opinionDict=opinionDict, testSplit=100, file_dict=None,
                   topicLines=params.get('topicLines'),
                   opinionLines=params.get('opinionLines'))

# Update perspective names (default name is directory name, which is currently
# the same for all perspectives)
for p, name in zip(corpus.perspectives, perspectives):
    p.name = name

logger.info(str(corpus))

nTopics = params.get('nTopics')

# get estimates of phi_topic and phi_opinions
s = get_sampler(params, corpus, nTopics=nTopics, initialize=False)
s.estimate_parameters(start=params.get('sampleEstimateStart'),
                      end=params.get('sampleEstimateEnd'))
phi_topic = s.topics
phi_opinion = s.opinions

params['outDir'] = args.out_dir

corpus.save(os.path.join(params.get('outDir'), 'corpus.json'))


# Path: cptm/utils/experiment.py
# def get_sampler(params, corpus, nTopics=None, initialize=True):
#     if nTopics is None:
#         nTopics = params.get('nTopics')
#     out_dir = params.get('outDir')
#     nIter = params.get('nIter')
#     alpha = 50.0/nTopics
#     beta = params.get('beta')
#     beta_o = params.get('beta_o')
#     logger.info('creating Gibbs sampler (nTopics: {}, nIter: {}, alpha: {}, '
#                 'beta: {}, beta_o: {})'.format(nTopics, nIter, alpha, beta,
#                                                beta_o))
#     sampler = GibbsSampler(corpus, nTopics=nTopics, nIter=nIter,
#                            alpha=alpha, beta=beta, beta_o=beta_o,
#                            out_dir=out_dir.format(nTopics),
#                            initialize=initialize)
#     return sampler
# 
# def load_config(fName):
#     with open(fName) as f:
#         config = json.load(f)
# 
#     logger.debug('configuration of experiment: ')
#     params = ['{}: {}'.format(p, v) for p, v in config.iteritems()]
#     for p in params:
#         logger.debug(p)
# 
#     params = {}
#     params['inputData'] = config.get('inputData')
#     params['outDir'] = config.get('outDir', '/{}')
#     params['testSplit'] = config.get('testSplit', 20)
#     params['minFreq'] = config.get('minFreq')
#     params['removeTopTF'] = config.get('removeTopTF')
#     params['removeTopDF'] = config.get('removeTopDF')
#     params['nIter'] = config.get('nIter', 200)
#     params['beta'] = config.get('beta', 0.02)
#     params['beta_o'] = config.get('beta_o', 0.02)
#     params['expNumTopics'] = config.get('expNumTopics', range(20, 201, 20))
#     params['nTopics'] = config.get('nTopics')
#     params['nProcesses'] = config.get('nProcesses', None)
#     params['topicLines'] = config.get('topicLines', [0])
#     params['opinionLines'] = config.get('opinionLines', [1])
#     params['sampleEstimateStart'] = config.get('sampleEstimateStart')
#     params['sampleEstimateEnd'] = config.get('sampleEstimateEnd')
# 
#     return params
# 
# def get_corpus(params):
#     out_dir = params.get('outDir')
#     files = glob(params.get('inputData'))
# 
#     if not os.path.isfile(out_dir.format('corpus.json')):
#         corpus = CPTCorpus(files,
#                            testSplit=params.get('testSplit'),
#                            topicLines=params.get('topicLines'),
#                            opinionLines=params.get('opinionLines'))
#         minFreq = params.get('minFreq')
#         removeTopTF = params.get('removeTopTF')
#         removeTopDF = params.get('removeTopDF')
#         if (not minFreq is None) or (not removeTopTF is None) or \
#            (not removeTopDF is None):
#             corpus.filter_dictionaries(minFreq=minFreq,
#                                        removeTopTF=removeTopTF,
#                                        removeTopDF=removeTopDF)
#         corpus.save_dictionaries(directory=out_dir.format(''))
#         corpus.save(out_dir.format('corpus.json'))
#     else:
#         corpus = CPTCorpus.load(file_name=out_dir.format('corpus.json'),
#                                 topicLines=params.get('topicLines'),
#                                 opinionLines=params.get('opinionLines'),
#                                 topicDict=out_dir.format('topicDict.dict'),
#                                 opinionDict=out_dir.format('opinionDict.dict'))
#     return corpus
# 
# def thetaFileName(params):
#     nTopics = params.get('nTopics')
#     return os.path.join(params.get('outDir').format(''),
#                         'theta_{}.csv'.format(nTopics))


# Path: cptm/experiment_manifesto.py
import logging
import argparse
import pandas as pd
import os
import sys
from CPTCorpus import CPTCorpus
from cptm.utils.experiment import get_sampler, load_config, get_corpus, \
    thetaFileName

perspectives = [p.name for p in c_perspectives.perspectives]
logger.info('Perspectives found: {}'.format('; '.join(perspectives)))

input_dirs = [args.data_dir for p in perspectives]

corpus = CPTCorpus(input=input_dirs, topicDict=topicDict,
                   opinionDict=opinionDict, testSplit=100, file_dict=None,
                   topicLines=params.get('topicLines'),
                   opinionLines=params.get('opinionLines'))

# Update perspective names (default name is directory name, which is currently
# the same for all perspectives)
for p, name in zip(corpus.perspectives, perspectives):
    p.name = name

logger.info(str(corpus))

nTopics = params.get('nTopics')

# get estimates of phi_topic and phi_opinions
s = get_sampler(params, corpus, nTopics=nTopics, initialize=False)
s.estimate_parameters(start=params.get('sampleEstimateStart'),
                      end=params.get('sampleEstimateEnd'))
phi_topic = s.topics
phi_opinion = s.opinions

params['outDir'] = args.out_dir

corpus.save(os.path.join(params.get('outDir'), 'corpus.json'))


sampler = get_sampler(params, corpus, nTopics=nTopics, initialize=False)