# Path: cptm/utils/experiment.py
# def get_sampler(params, corpus, nTopics=None, initialize=True):
#     if nTopics is None:
#         nTopics = params.get('nTopics')
#     out_dir = params.get('outDir')
#     nIter = params.get('nIter')
#     alpha = 50.0/nTopics
#     beta = params.get('beta')
#     beta_o = params.get('beta_o')
#     logger.info('creating Gibbs sampler (nTopics: {}, nIter: {}, alpha: {}, '
#                 'beta: {}, beta_o: {})'.format(nTopics, nIter, alpha, beta,
#                                                beta_o))
#     sampler = GibbsSampler(corpus, nTopics=nTopics, nIter=nIter,
#                            alpha=alpha, beta=beta, beta_o=beta_o,
#                            out_dir=out_dir.format(nTopics),
#                            initialize=initialize)
#     return sampler
# 
# def load_config(fName):
#     with open(fName) as f:
#         config = json.load(f)
# 
#     logger.debug('configuration of experiment: ')
#     params = ['{}: {}'.format(p, v) for p, v in config.iteritems()]
#     for p in params:
#         logger.debug(p)
# 
#     params = {}
#     params['inputData'] = config.get('inputData')
#     params['outDir'] = config.get('outDir', '/{}')
#     params['testSplit'] = config.get('testSplit', 20)
#     params['minFreq'] = config.get('minFreq')
#     params['removeTopTF'] = config.get('removeTopTF')
#     params['removeTopDF'] = config.get('removeTopDF')
#     params['nIter'] = config.get('nIter', 200)
#     params['beta'] = config.get('beta', 0.02)
#     params['beta_o'] = config.get('beta_o', 0.02)
#     params['expNumTopics'] = config.get('expNumTopics', range(20, 201, 20))
#     params['nTopics'] = config.get('nTopics')
#     params['nProcesses'] = config.get('nProcesses', None)
#     params['topicLines'] = config.get('topicLines', [0])
#     params['opinionLines'] = config.get('opinionLines', [1])
#     params['sampleEstimateStart'] = config.get('sampleEstimateStart')
#     params['sampleEstimateEnd'] = config.get('sampleEstimateEnd')
# 
#     return params
# 
# def get_corpus(params):
#     out_dir = params.get('outDir')
#     files = glob(params.get('inputData'))
# 
#     if not os.path.isfile(out_dir.format('corpus.json')):
#         corpus = CPTCorpus(files,
#                            testSplit=params.get('testSplit'),
#                            topicLines=params.get('topicLines'),
#                            opinionLines=params.get('opinionLines'))
#         minFreq = params.get('minFreq')
#         removeTopTF = params.get('removeTopTF')
#         removeTopDF = params.get('removeTopDF')
#         if (not minFreq is None) or (not removeTopTF is None) or \
#            (not removeTopDF is None):
#             corpus.filter_dictionaries(minFreq=minFreq,
#                                        removeTopTF=removeTopTF,
#                                        removeTopDF=removeTopDF)
#         corpus.save_dictionaries(directory=out_dir.format(''))
#         corpus.save(out_dir.format('corpus.json'))
#     else:
#         corpus = CPTCorpus.load(file_name=out_dir.format('corpus.json'),
#                                 topicLines=params.get('topicLines'),
#                                 opinionLines=params.get('opinionLines'),
#                                 topicDict=out_dir.format('topicDict.dict'),
#                                 opinionDict=out_dir.format('opinionDict.dict'))
#     return corpus
# 
# def thetaFileName(params):
#     nTopics = params.get('nTopics')
#     return os.path.join(params.get('outDir').format(''),
#                         'theta_{}.csv'.format(nTopics))


import logging
import argparse
import pandas as pd
import os
import sys
from CPTCorpus import CPTCorpus
from cptm.utils.experiment import get_sampler, load_config, get_corpus, \
    thetaFileName
documents.

The corpus is not divided in perspectives.

Used to estimate the likelihood of party manifestos given opinions for the
different perspectives (party manifestos come from the manifesto project).

Before this script can be run, a cptm corpus should be created. Use the
manifestoproject2cptm_input.py script to create a corpus that can be used
as input.

Usage: python experiment_manifesto.py <experiment.json> <input dir>
<output dir>
"""


logger = logging.getLogger(__name__)
logging.basicConfig(format='%(levelname)s : %(message)s', level=logging.DEBUG)

logging.getLogger('gensim').setLevel(logging.ERROR)
logging.getLogger('CPTCorpus').setLevel(logging.DEBUG)
logging.getLogger('CPT_Gibbs').setLevel(logging.DEBUG)

parser = argparse.ArgumentParser()
parser.add_argument('json', help='json file containing experiment '
                    'configuration.')
parser.add_argument('data_dir', help='dir containing the input data.')
parser.add_argument('out_dir', help='dir to write results to.')
args = parser.parse_args()


# Path: cptm/utils/experiment.py
# def get_sampler(params, corpus, nTopics=None, initialize=True):
#     if nTopics is None:
#         nTopics = params.get('nTopics')
#     out_dir = params.get('outDir')
#     nIter = params.get('nIter')
#     alpha = 50.0/nTopics
#     beta = params.get('beta')
#     beta_o = params.get('beta_o')
#     logger.info('creating Gibbs sampler (nTopics: {}, nIter: {}, alpha: {}, '
#                 'beta: {}, beta_o: {})'.format(nTopics, nIter, alpha, beta,
#                                                beta_o))
#     sampler = GibbsSampler(corpus, nTopics=nTopics, nIter=nIter,
#                            alpha=alpha, beta=beta, beta_o=beta_o,
#                            out_dir=out_dir.format(nTopics),
#                            initialize=initialize)
#     return sampler
# 
# def load_config(fName):
#     with open(fName) as f:
#         config = json.load(f)
# 
#     logger.debug('configuration of experiment: ')
#     params = ['{}: {}'.format(p, v) for p, v in config.iteritems()]
#     for p in params:
#         logger.debug(p)
# 
#     params = {}
#     params['inputData'] = config.get('inputData')
#     params['outDir'] = config.get('outDir', '/{}')
#     params['testSplit'] = config.get('testSplit', 20)
#     params['minFreq'] = config.get('minFreq')
#     params['removeTopTF'] = config.get('removeTopTF')
#     params['removeTopDF'] = config.get('removeTopDF')
#     params['nIter'] = config.get('nIter', 200)
#     params['beta'] = config.get('beta', 0.02)
#     params['beta_o'] = config.get('beta_o', 0.02)
#     params['expNumTopics'] = config.get('expNumTopics', range(20, 201, 20))
#     params['nTopics'] = config.get('nTopics')
#     params['nProcesses'] = config.get('nProcesses', None)
#     params['topicLines'] = config.get('topicLines', [0])
#     params['opinionLines'] = config.get('opinionLines', [1])
#     params['sampleEstimateStart'] = config.get('sampleEstimateStart')
#     params['sampleEstimateEnd'] = config.get('sampleEstimateEnd')
# 
#     return params
# 
# def get_corpus(params):
#     out_dir = params.get('outDir')
#     files = glob(params.get('inputData'))
# 
#     if not os.path.isfile(out_dir.format('corpus.json')):
#         corpus = CPTCorpus(files,
#                            testSplit=params.get('testSplit'),
#                            topicLines=params.get('topicLines'),
#                            opinionLines=params.get('opinionLines'))
#         minFreq = params.get('minFreq')
#         removeTopTF = params.get('removeTopTF')
#         removeTopDF = params.get('removeTopDF')
#         if (not minFreq is None) or (not removeTopTF is None) or \
#            (not removeTopDF is None):
#             corpus.filter_dictionaries(minFreq=minFreq,
#                                        removeTopTF=removeTopTF,
#                                        removeTopDF=removeTopDF)
#         corpus.save_dictionaries(directory=out_dir.format(''))
#         corpus.save(out_dir.format('corpus.json'))
#     else:
#         corpus = CPTCorpus.load(file_name=out_dir.format('corpus.json'),
#                                 topicLines=params.get('topicLines'),
#                                 opinionLines=params.get('opinionLines'),
#                                 topicDict=out_dir.format('topicDict.dict'),
#                                 opinionDict=out_dir.format('opinionDict.dict'))
#     return corpus
# 
# def thetaFileName(params):
#     nTopics = params.get('nTopics')
#     return os.path.join(params.get('outDir').format(''),
#                         'theta_{}.csv'.format(nTopics))


# Path: cptm/experiment_manifesto.py
import logging
import argparse
import pandas as pd
import os
import sys
from CPTCorpus import CPTCorpus
from cptm.utils.experiment import get_sampler, load_config, get_corpus, \
    thetaFileName

documents.

The corpus is not divided in perspectives.

Used to estimate the likelihood of party manifestos given opinions for the
different perspectives (party manifestos come from the manifesto project).

Before this script can be run, a cptm corpus should be created. Use the
manifestoproject2cptm_input.py script to create a corpus that can be used
as input.

Usage: python experiment_manifesto.py <experiment.json> <input dir>
<output dir>
"""


logger = logging.getLogger(__name__)
logging.basicConfig(format='%(levelname)s : %(message)s', level=logging.DEBUG)

logging.getLogger('gensim').setLevel(logging.ERROR)
logging.getLogger('CPTCorpus').setLevel(logging.DEBUG)
logging.getLogger('CPT_Gibbs').setLevel(logging.DEBUG)

parser = argparse.ArgumentParser()
parser.add_argument('json', help='json file containing experiment '
                    'configuration.')
parser.add_argument('data_dir', help='dir containing the input data.')
parser.add_argument('out_dir', help='dir to write results to.')
args = parser.parse_args()


params = load_config(args.json)