# Path: cptm/utils/experiment.py
# def load_config(fName):
#     with open(fName) as f:
#         config = json.load(f)
# 
#     logger.debug('configuration of experiment: ')
#     params = ['{}: {}'.format(p, v) for p, v in config.iteritems()]
#     for p in params:
#         logger.debug(p)
# 
#     params = {}
#     params['inputData'] = config.get('inputData')
#     params['outDir'] = config.get('outDir', '/{}')
#     params['testSplit'] = config.get('testSplit', 20)
#     params['minFreq'] = config.get('minFreq')
#     params['removeTopTF'] = config.get('removeTopTF')
#     params['removeTopDF'] = config.get('removeTopDF')
#     params['nIter'] = config.get('nIter', 200)
#     params['beta'] = config.get('beta', 0.02)
#     params['beta_o'] = config.get('beta_o', 0.02)
#     params['expNumTopics'] = config.get('expNumTopics', range(20, 201, 20))
#     params['nTopics'] = config.get('nTopics')
#     params['nProcesses'] = config.get('nProcesses', None)
#     params['topicLines'] = config.get('topicLines', [0])
#     params['opinionLines'] = config.get('opinionLines', [1])
#     params['sampleEstimateStart'] = config.get('sampleEstimateStart')
#     params['sampleEstimateEnd'] = config.get('sampleEstimateEnd')
# 
#     return params
# 
# def get_corpus(params):
#     out_dir = params.get('outDir')
#     files = glob(params.get('inputData'))
# 
#     if not os.path.isfile(out_dir.format('corpus.json')):
#         corpus = CPTCorpus(files,
#                            testSplit=params.get('testSplit'),
#                            topicLines=params.get('topicLines'),
#                            opinionLines=params.get('opinionLines'))
#         minFreq = params.get('minFreq')
#         removeTopTF = params.get('removeTopTF')
#         removeTopDF = params.get('removeTopDF')
#         if (not minFreq is None) or (not removeTopTF is None) or \
#            (not removeTopDF is None):
#             corpus.filter_dictionaries(minFreq=minFreq,
#                                        removeTopTF=removeTopTF,
#                                        removeTopDF=removeTopDF)
#         corpus.save_dictionaries(directory=out_dir.format(''))
#         corpus.save(out_dir.format('corpus.json'))
#     else:
#         corpus = CPTCorpus.load(file_name=out_dir.format('corpus.json'),
#                                 topicLines=params.get('topicLines'),
#                                 opinionLines=params.get('opinionLines'),
#                                 topicDict=out_dir.format('topicDict.dict'),
#                                 opinionDict=out_dir.format('opinionDict.dict'))
#     return corpus


import logging
import argparse
import glob
import gzip
import datetime
from lxml import etree
from cptm.utils.experiment import load_config, get_corpus
    print '{},{}'.format(year, len(data_files))

years.sort()
min_year = years[0]
print 'min year', min_year
max_year = years[-1]
print 'max year', max_year

data_files = glob.glob('{}data_folia/*.xml.gz'.format(min_year))
for df in data_files:
    f = gzip.open(df)
    context = etree.iterparse(f, events=('end',), tag=date_tag,
                              huge_tree=True)
    for event, elem in context:
        d = datetime.datetime.strptime(elem.text, "%Y-%m-%d").date()
        if d < min_date:
            min_date = d

data_files = glob.glob('{}data_folia/*.xml.gz'.format(max_year))
for df in data_files:
    f = gzip.open(df)
    context = etree.iterparse(f, events=('end',), tag=date_tag,
                              huge_tree=True)
    for event, elem in context:
        d = datetime.datetime.strptime(elem.text, "%Y-%m-%d").date()
        if d > max_date:
            max_date = d

print 'time period: ', min_date, max_date


# Path: cptm/utils/experiment.py
# def load_config(fName):
#     with open(fName) as f:
#         config = json.load(f)
# 
#     logger.debug('configuration of experiment: ')
#     params = ['{}: {}'.format(p, v) for p, v in config.iteritems()]
#     for p in params:
#         logger.debug(p)
# 
#     params = {}
#     params['inputData'] = config.get('inputData')
#     params['outDir'] = config.get('outDir', '/{}')
#     params['testSplit'] = config.get('testSplit', 20)
#     params['minFreq'] = config.get('minFreq')
#     params['removeTopTF'] = config.get('removeTopTF')
#     params['removeTopDF'] = config.get('removeTopDF')
#     params['nIter'] = config.get('nIter', 200)
#     params['beta'] = config.get('beta', 0.02)
#     params['beta_o'] = config.get('beta_o', 0.02)
#     params['expNumTopics'] = config.get('expNumTopics', range(20, 201, 20))
#     params['nTopics'] = config.get('nTopics')
#     params['nProcesses'] = config.get('nProcesses', None)
#     params['topicLines'] = config.get('topicLines', [0])
#     params['opinionLines'] = config.get('opinionLines', [1])
#     params['sampleEstimateStart'] = config.get('sampleEstimateStart')
#     params['sampleEstimateEnd'] = config.get('sampleEstimateEnd')
# 
#     return params
# 
# def get_corpus(params):
#     out_dir = params.get('outDir')
#     files = glob(params.get('inputData'))
# 
#     if not os.path.isfile(out_dir.format('corpus.json')):
#         corpus = CPTCorpus(files,
#                            testSplit=params.get('testSplit'),
#                            topicLines=params.get('topicLines'),
#                            opinionLines=params.get('opinionLines'))
#         minFreq = params.get('minFreq')
#         removeTopTF = params.get('removeTopTF')
#         removeTopDF = params.get('removeTopDF')
#         if (not minFreq is None) or (not removeTopTF is None) or \
#            (not removeTopDF is None):
#             corpus.filter_dictionaries(minFreq=minFreq,
#                                        removeTopTF=removeTopTF,
#                                        removeTopDF=removeTopDF)
#         corpus.save_dictionaries(directory=out_dir.format(''))
#         corpus.save(out_dir.format('corpus.json'))
#     else:
#         corpus = CPTCorpus.load(file_name=out_dir.format('corpus.json'),
#                                 topicLines=params.get('topicLines'),
#                                 opinionLines=params.get('opinionLines'),
#                                 topicDict=out_dir.format('topicDict.dict'),
#                                 opinionDict=out_dir.format('opinionDict.dict'))
#     return corpus


# Path: cptm/corpusstatistics.py
import logging
import argparse
import glob
import gzip
import datetime
from lxml import etree
from cptm.utils.experiment import load_config, get_corpus

    print '{},{}'.format(year, len(data_files))

years.sort()
min_year = years[0]
print 'min year', min_year
max_year = years[-1]
print 'max year', max_year

data_files = glob.glob('{}data_folia/*.xml.gz'.format(min_year))
for df in data_files:
    f = gzip.open(df)
    context = etree.iterparse(f, events=('end',), tag=date_tag,
                              huge_tree=True)
    for event, elem in context:
        d = datetime.datetime.strptime(elem.text, "%Y-%m-%d").date()
        if d < min_date:
            min_date = d

data_files = glob.glob('{}data_folia/*.xml.gz'.format(max_year))
for df in data_files:
    f = gzip.open(df)
    context = etree.iterparse(f, events=('end',), tag=date_tag,
                              huge_tree=True)
    for event, elem in context:
        d = datetime.datetime.strptime(elem.text, "%Y-%m-%d").date()
        if d > max_date:
            max_date = d

print 'time period: ', min_date, max_date


config = load_config(args.json)