# Path: cptm/utils/inputgeneration.py
# class Perspective():
#     def __init__(self, name, posTopic, posOpinion):
#         """Initialize inputgeneration Perspective.
# 
#         Parameters:
#             name : str
#                 The perspective name. Used as directory name to store the data.
#             posTopic : list of strings
#                 List of strings specifying the pos-tags for topic words.
#             posOpinion : list of strings
#                 List of strings specifying the pos-tags for opinion words.
#         """
#         self.name = name
#         self.wordTypes = posTopic + posOpinion
#         self.posTopic = posTopic
#         self.posOpinion = posOpinion
#         self.words = {}
#         for w in self.wordTypes:
#             self.words[w] = []
# 
#     def __str__(self):
#         len_topic_words, len_opinion_words = self.word_lengths()
#         return 'Perspective: {} - {} topic words; {} opinion words'.format(
#             self.name, len_topic_words, len_opinion_words)
# 
#     def add(self, tag, word):
#         self.words[tag].append(word)
# 
#     def write2file(self, out_dir, file_name):
#         # create dir (if not exists)
#         directory = os.path.join(out_dir, self.name)
#         if not os.path.exists(directory):
#             os.makedirs(directory)
# 
#         # write words to file
#         out_file = os.path.join(directory, file_name)
#         logger.debug('Writing file {} for perspective {}'.format(out_file,
#                      self.name))
#         with codecs.open(out_file, 'wb', 'utf8') as f:
#             for w in self.wordTypes:
#                 f.write(u'{}\n'.format(' '.join(self.words[w])))
# 
#     def word_lengths(self):
#         len_topic_words = sum([len(self.words[w])
#                                for w in self.posTopic])
#         len_opinion_words = sum([len(self.words[w])
#                                 for w in self.posOpinion])
#         return len_topic_words, len_opinion_words
# 
# def remove_trailing_digits(word):
#     """Convert words like d66 to d.
# 
#     In the folia files from politicalmashup, words such as d66 have been
#     extracted as two words (d and 66) and only d ended up in the data input
#     files. The folia files were probably created with an old version of frog,
#     because currenly, words like these are parsed correctly.
# 
#     This function can be used when parsing and lemmatizing new text to match
#     the vocabulary used in the old folia files.
#     """
#     regex = re.compile('^(.+?)(\d+)$', flags=re.UNICODE)
#     m = regex.match(word)
#     if m:
#         return m.group(1)
#     return word
# 
# Path: cptm/utils/dutchdata.py
# def pos_topic_words():
#     return ['N']
# 
# def pos_opinion_words():
#     return ['ADJ', 'BW', 'WW']
# 
# def word_types():
#     return pos_topic_words() + pos_opinion_words()
# 
# Path: cptm/utils/frog.py
# def get_frogclient(port=8020):
#     try:
#         frogclient = FrogClient('localhost', port)
#         return frogclient
#     except:
#         logger.error('Cannot connect to the Frog server. '
#                      'Is it running at port {}?'.format(port))
#         logger.info('Start the Frog server with "docker run -p '
#                     '127.0.0.1:{}:{} -t -i proycon/lamachine frog '
#                     '-S {}"'.format(port, port, port))
#         sys.exit(1)
# 
# def pos_and_lemmas(text, frogclient):
#     # add timeout functionality (so frog won't keep parsing faulty text
#     # forever)
#     signal.signal(signal.SIGALRM, timeout)
#     signal.alarm(300)
# 
#     regex = re.compile(r'\(.*\)')
# 
#     try:
#         for data in frogclient.process(text):
#             word, lemma, morph, ext_pos = data[:4]
#             if ext_pos:  # ext_pos can be None
#                 pos = regex.sub('', ext_pos)
#                 yield pos, lemma
#     except Exception, e:
#         raise e


import pandas as pd
import logging
import argparse
import os
import glob
from cptm.utils.inputgeneration import Perspective, remove_trailing_digits
from cptm.utils.dutchdata import pos_topic_words, pos_opinion_words, word_types
from cptm.utils.frog import get_frogclient, pos_and_lemmas
    args = parser.parse_args()

    dir_in = args.dir_in
    dir_out = args.dir_out

    frogclient = get_frogclient()

    if not os.path.exists(dir_out):
        os.makedirs(dir_out)

    data_files = glob.glob('{}/*.csv'.format(dir_in))

    for i, data_file in enumerate(data_files):
        if i % 5 == 0:
            logger.info('Processing text {} of {}'.format(i + 1,
                        len(data_files)))
        p = Perspective('', pos_topic_words(), pos_opinion_words())
        df = pd.read_csv(data_file, encoding='utf-8')
        text = ' '.join([line for line in df['content']])
        try:
            for pos, lemma in pos_and_lemmas(text, frogclient):
                if pos in word_types():
                    p.add(pos, remove_trailing_digits(lemma))
        except Exception, e:
            logger.warn(str(e))
            del frogclient
            frogclient = get_frogclient()
            logger.info('parsing pseudo sentences instead')
            for text in df['content']:
                for pos, lemma in pos_and_lemmas(text, frogclient):

# Path: cptm/utils/inputgeneration.py
# class Perspective():
#     def __init__(self, name, posTopic, posOpinion):
#         """Initialize inputgeneration Perspective.
# 
#         Parameters:
#             name : str
#                 The perspective name. Used as directory name to store the data.
#             posTopic : list of strings
#                 List of strings specifying the pos-tags for topic words.
#             posOpinion : list of strings
#                 List of strings specifying the pos-tags for opinion words.
#         """
#         self.name = name
#         self.wordTypes = posTopic + posOpinion
#         self.posTopic = posTopic
#         self.posOpinion = posOpinion
#         self.words = {}
#         for w in self.wordTypes:
#             self.words[w] = []
# 
#     def __str__(self):
#         len_topic_words, len_opinion_words = self.word_lengths()
#         return 'Perspective: {} - {} topic words; {} opinion words'.format(
#             self.name, len_topic_words, len_opinion_words)
# 
#     def add(self, tag, word):
#         self.words[tag].append(word)
# 
#     def write2file(self, out_dir, file_name):
#         # create dir (if not exists)
#         directory = os.path.join(out_dir, self.name)
#         if not os.path.exists(directory):
#             os.makedirs(directory)
# 
#         # write words to file
#         out_file = os.path.join(directory, file_name)
#         logger.debug('Writing file {} for perspective {}'.format(out_file,
#                      self.name))
#         with codecs.open(out_file, 'wb', 'utf8') as f:
#             for w in self.wordTypes:
#                 f.write(u'{}\n'.format(' '.join(self.words[w])))
# 
#     def word_lengths(self):
#         len_topic_words = sum([len(self.words[w])
#                                for w in self.posTopic])
#         len_opinion_words = sum([len(self.words[w])
#                                 for w in self.posOpinion])
#         return len_topic_words, len_opinion_words
# 
# def remove_trailing_digits(word):
#     """Convert words like d66 to d.
# 
#     In the folia files from politicalmashup, words such as d66 have been
#     extracted as two words (d and 66) and only d ended up in the data input
#     files. The folia files were probably created with an old version of frog,
#     because currenly, words like these are parsed correctly.
# 
#     This function can be used when parsing and lemmatizing new text to match
#     the vocabulary used in the old folia files.
#     """
#     regex = re.compile('^(.+?)(\d+)$', flags=re.UNICODE)
#     m = regex.match(word)
#     if m:
#         return m.group(1)
#     return word
# 
# Path: cptm/utils/dutchdata.py
# def pos_topic_words():
#     return ['N']
# 
# def pos_opinion_words():
#     return ['ADJ', 'BW', 'WW']
# 
# def word_types():
#     return pos_topic_words() + pos_opinion_words()
# 
# Path: cptm/utils/frog.py
# def get_frogclient(port=8020):
#     try:
#         frogclient = FrogClient('localhost', port)
#         return frogclient
#     except:
#         logger.error('Cannot connect to the Frog server. '
#                      'Is it running at port {}?'.format(port))
#         logger.info('Start the Frog server with "docker run -p '
#                     '127.0.0.1:{}:{} -t -i proycon/lamachine frog '
#                     '-S {}"'.format(port, port, port))
#         sys.exit(1)
# 
# def pos_and_lemmas(text, frogclient):
#     # add timeout functionality (so frog won't keep parsing faulty text
#     # forever)
#     signal.signal(signal.SIGALRM, timeout)
#     signal.alarm(300)
# 
#     regex = re.compile(r'\(.*\)')
# 
#     try:
#         for data in frogclient.process(text):
#             word, lemma, morph, ext_pos = data[:4]
#             if ext_pos:  # ext_pos can be None
#                 pos = regex.sub('', ext_pos)
#                 yield pos, lemma
#     except Exception, e:
#         raise e


# Path: cptm/manifestoproject2cpt_input.py
import pandas as pd
import logging
import argparse
import os
import glob
from cptm.utils.inputgeneration import Perspective, remove_trailing_digits
from cptm.utils.dutchdata import pos_topic_words, pos_opinion_words, word_types
from cptm.utils.frog import get_frogclient, pos_and_lemmas

    args = parser.parse_args()

    dir_in = args.dir_in
    dir_out = args.dir_out

    frogclient = get_frogclient()

    if not os.path.exists(dir_out):
        os.makedirs(dir_out)

    data_files = glob.glob('{}/*.csv'.format(dir_in))

    for i, data_file in enumerate(data_files):
        if i % 5 == 0:
            logger.info('Processing text {} of {}'.format(i + 1,
                        len(data_files)))
        p = Perspective('', pos_topic_words(), pos_opinion_words())
        df = pd.read_csv(data_file, encoding='utf-8')
        text = ' '.join([line for line in df['content']])
        try:
            for pos, lemma in pos_and_lemmas(text, frogclient):
                if pos in word_types():
                    p.add(pos, remove_trailing_digits(lemma))
        except Exception, e:
            logger.warn(str(e))
            del frogclient
            frogclient = get_frogclient()
            logger.info('parsing pseudo sentences instead')
            for text in df['content']:
                for pos, lemma in pos_and_lemmas(text, frogclient):

                    if pos in word_types():