<commit_before># coding: utf-8
"""
Este processamento gera uma tabulação de autores e afiliação de cada artigo da
coleção SciELO.
Formato de saída:
"PID","issn","título","área temática","ano de publicação","tipo de documento","author","instituição","paises de afiliação","estado de afiliação","cidade de afiliação"
"""
import argparse
import logging
import codecs

import utils

logger = logging.getLogger(__name__)

def _config_logging(logging_level='INFO', logging_file=None):

    allowed_levels = {
        'DEBUG': logging.DEBUG,
        'INFO': logging.INFO,
        'WARNING': logging.WARNING,
        'ERROR': logging.ERROR,
        'CRITICAL': logging.CRITICAL
    }

    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')

    logger.setLevel(allowed_levels.get(logging_level, 'INFO'))

    if logging_file:
        hl = logging.FileHandler(logging_file, mode='a')
    else:
        hl = logging.StreamHandler()

    hl.setFormatter(formatter)
    hl.setLevel(allowed_levels.get(logging_level, 'INFO'))

    logger.addHandler(hl)

    return logger


class Dumper(object):

    def __init__(self, collection, issns=None, output_file=None):

        self._ratchet = utils.ratchet_server()
        self._articlemeta = utils.articlemeta_server()
        self.collection = collection
        self.issns = issns
        self.output_file = output_file

    def run(self):

        header = [u"PID",u"ISSN",u"título",u"área temática",u"ano de publicação",u"tipo de documento",u"author",u"instituição",u"paises de afiliação",u"estado de afiliação",u"cidade de afiliação"]

        if not self.issns:
            self.issns = [None]

        if not self.output_file:
            print(header)
            for issn in self.issns:
                for data in self.get_data(issn=issn):
                    print(self.fmt_csv(data))
            exit()

        with codecs.open(self.output_file, 'w', encoding='utf-8') as f:
            f.write('%s\r\n' % ','.join(header))
            for issn in self.issns:
                for data in self.get_data(issn=issn):
                    for item in self.fmt_csv(data):
                        f.write('%s\r\n' % item)
        
    def join_line(self, line):

        return ','.join(['"%s"' % i.replace('"', '""') for i in line])

    def fmt_csv(self, data):
        countries = set()

        affs = {item['index'].upper():item for item in data.mixed_affiliations}

        line = [
            data.publisher_id,
            data.journal.scielo_issn,
            data.journal.title,
            ','.join(data.journal.subject_areas),
            data.publication_date[0:4],
            data.document_type
        ]
        if data.authors:
            for author in data.authors:
                author_line = [' '.join([author.get('given_names', ''), author.get('surname', '')])]
                if 'xref' in author:
                    for index in author['xref']:
                        index = index.upper()
                        aff_line = []
                        aff_line.append(affs.get(index, {}).get('institution', '')),
                        aff_line.append(affs.get(index, {}).get('country', '')),
                        aff_line.append(affs.get(index, {}).get('state', '')),
                        aff_line.append(affs.get(index, {}).get('city', ''))
                        yield self.join_line(line+author_line+aff_line)
                else:
                    yield self.join_line(line+author_line)
        else:
            yield self.join_line(line)

    def get_data(self, issn):
        document = self._articlemeta.document('S0034-72802014000400230', collection=self.collection)
        #for document in self._articlemeta.documents(collection=self.collection, issn=issn):
        logger.debug('Reading document: %s' % document.publisher_id)
        yield document


def main():

    parser = argparse.ArgumentParser(
        description='Dump languages distribution by article'<commit_msg>Correção de bug de match de indice de afiliação<commit_after># coding: utf-8
"""
Este processamento gera uma tabulação de autores e afiliação de cada artigo da
coleção SciELO.
Formato de saída:
"PID","issn","título","área temática","ano de publicação","tipo de documento","author","instituição","paises de afiliação","estado de afiliação","cidade de afiliação"
"""
import argparse
import logging
import codecs

import utils

logger = logging.getLogger(__name__)

def _config_logging(logging_level='INFO', logging_file=None):

    allowed_levels = {
        'DEBUG': logging.DEBUG,
        'INFO': logging.INFO,
        'WARNING': logging.WARNING,
        'ERROR': logging.ERROR,
        'CRITICAL': logging.CRITICAL
    }

    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')

    logger.setLevel(allowed_levels.get(logging_level, 'INFO'))

    if logging_file:
        hl = logging.FileHandler(logging_file, mode='a')
    else:
        hl = logging.StreamHandler()

    hl.setFormatter(formatter)
    hl.setLevel(allowed_levels.get(logging_level, 'INFO'))

    logger.addHandler(hl)

    return logger


class Dumper(object):

    def __init__(self, collection, issns=None, output_file=None):

        self._ratchet = utils.ratchet_server()
        self._articlemeta = utils.articlemeta_server()
        self.collection = collection
        self.issns = issns
        self.output_file = output_file

    def run(self):

        header = [u"PID",u"ISSN",u"título",u"área temática",u"ano de publicação",u"tipo de documento",u"author",u"instituição",u"paises de afiliação",u"estado de afiliação",u"cidade de afiliação"]

        if not self.issns:
            self.issns = [None]

        if not self.output_file:
            print(header)
            for issn in self.issns:
                for data in self.get_data(issn=issn):
                    print(self.fmt_csv(data))
            exit()

        with codecs.open(self.output_file, 'w', encoding='utf-8') as f:
            f.write('%s\r\n' % ','.join(header))
            for issn in self.issns:
                for data in self.get_data(issn=issn):
                    for item in self.fmt_csv(data):
                        f.write('%s\r\n' % item)
        
    def join_line(self, line):

        return ','.join(['"%s"' % i.replace('"', '""') for i in line])

    def fmt_csv(self, data):
        countries = set()

        affs = {item['index'].upper():item for item in data.mixed_affiliations}

        line = [
            data.publisher_id,
            data.journal.scielo_issn,
            data.journal.title,
            ','.join(data.journal.subject_areas),
            data.publication_date[0:4],
            data.document_type
        ]
        if data.authors:
            for author in data.authors:
                author_line = [' '.join([author.get('given_names', ''), author.get('surname', '')])]
                if 'xref' in author:
                    for index in author['xref']:
                        index = index.upper()
                        aff_line = []
                        aff_line.append(affs.get(index, {}).get('institution', '')),
                        aff_line.append(affs.get(index, {}).get('country', '')),
                        aff_line.append(affs.get(index, {}).get('state', '')),
                        aff_line.append(affs.get(index, {}).get('city', ''))
                        yield self.join_line(line+author_line+aff_line)
                else:
                    yield self.join_line(line+author_line)
        else:
            yield self.join_line(line)

    def get_data(self, issn):
        for document in self._articlemeta.documents(collection=self.collection, issn=issn):
            logger.debug('Reading document: %s' % document.publisher_id)
            yield document


def main():

    parser = argparse.ArgumentParser(
        description='Dump languages distribution by article'
{'fraction_non_alphanumeric': 0.13167170191339375, 'fraction_numerical': 0.00365055387713998, 'mean_word_length': 2.2628336755646816, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5734591', 'n_tokens_mistral': 2527, 'n_tokens_neox': 2410, 'n_words': 486}