<commit_before># coding: utf-8
"""
Esse processamento condença os metadados de documentos com os dados de acessos.
"""
import sys
import argparse
import logging
import re
import json
import codecs
import datetime

from legendarium.urlegendarium import URLegendarium

import choices
import utils

__version__ = 0.1

logger = logging.getLogger(__name__)

SUPPLBEG_REGEX = re.compile(r'^0 ')
SUPPLEND_REGEX = re.compile(r' 0$')
REGEX_PDF_PATH = re.compile(r'/pdf.*\.pdf$')
FROM = '1500-01-01'
UNTIL = datetime.datetime.now().isoformat()[0:10]
DAYLY_GRANULARITY = False
OUTPUT_FORMAT = 'csv'


def _config_logging(logging_level='INFO', logging_file=None):

    allowed_levels = {
        'DEBUG': logging.DEBUG,
        'INFO': logging.INFO,
        'WARNING': logging.WARNING,
        'ERROR': logging.ERROR,
        'CRITICAL': logging.CRITICAL
    }

    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')

    logger.setLevel(allowed_levels.get(logging_level, 'INFO'))

    if logging_file:
        hl = logging.FileHandler(logging_file, mode='a')
    else:
        hl = logging.StreamHandler()

    hl.setFormatter(formatter)
    hl.setLevel(allowed_levels.get(logging_level, 'INFO'))

    logger.addHandler(hl)

    return logger


def pdf_keys(fulltexts):

    keys = []

    if not 'pdf' in fulltexts:
        return keys

    for language, url in fulltexts['pdf'].items():
        path = REGEX_PDF_PATH.search(url)
        if path:
            keys.append(path.group().upper())

    return keys


def fbpe_key(code):
    """
    input:
        'S0102-67202009000300001'
    output:
        'S0102-6720(09)000300001'
    """

    begin = code[0:10]
    year = code[12:14]
    end = code[14:]

    return '%s(%s)%s' % (begin, year, end)


def eligible_match_keys(document):
    keys = []

    keys.append(document.publisher_id)
    keys.append(fbpe_key(document.publisher_id))
    if document.doi:
        keys.append(document.doi)
    keys += pdf_keys(document.fulltexts())

    suppl = ''.join([document.issue.supplement_volume or '', document.issue.supplement_number or '']).strip()

    leg = URLegendarium(
        acron=document.journal.acronym,
        year_pub=document.publication_date[:4],
        volume=document.issue.volume,
        number=document.issue.number,
        fpage=document.start_page,
        fpage_sequence=document.start_page_sequence,
        lpage=document.end_page,
        article_id=document.elocation,
        suppl_number=suppl,
        doi=document.doi,
        order=document.issue.order
    )

    url_code = '/%s/' % leg.url_article

    if url_code:
        keys.append(url_code.upper())

    return keys


def country(country):
    if country in choices.ISO_3166:
        return country
    if country in choices.ISO_3166_COUNTRY_AS_KEY:
        return choices.ISO_3166_COUNTRY_AS_KEY[country]
    return 'undefined'


def get_date_timestamp(date):
    try:
        return datetime.datetime.strptime(date, '%Y-%m').isoformat()
    except ValueError:
        try:
            return datetime.datetime.strptime(date, '%Y-%m-%d').isoformat()<commit_msg>Deixa processamento de dump de acessos resiliente<commit_after># coding: utf-8
"""
Esse processamento condença os metadados de documentos com os dados de acessos.
"""
import sys
import argparse
import logging
import re
import json
import codecs
import datetime

from legendarium.urlegendarium import URLegendarium

import choices
import utils

__version__ = 0.1

logger = logging.getLogger(__name__)

SUPPLBEG_REGEX = re.compile(r'^0 ')
SUPPLEND_REGEX = re.compile(r' 0$')
REGEX_PDF_PATH = re.compile(r'/pdf.*\.pdf$')
FROM = '1500-01-01'
UNTIL = datetime.datetime.now().isoformat()[0:10]
DAYLY_GRANULARITY = False
OUTPUT_FORMAT = 'csv'


def _config_logging(logging_level='INFO', logging_file=None):

    allowed_levels = {
        'DEBUG': logging.DEBUG,
        'INFO': logging.INFO,
        'WARNING': logging.WARNING,
        'ERROR': logging.ERROR,
        'CRITICAL': logging.CRITICAL
    }

    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')

    logger.setLevel(allowed_levels.get(logging_level, 'INFO'))

    if logging_file:
        hl = logging.FileHandler(logging_file, mode='a')
    else:
        hl = logging.StreamHandler()

    hl.setFormatter(formatter)
    hl.setLevel(allowed_levels.get(logging_level, 'INFO'))

    logger.addHandler(hl)

    return logger


def pdf_keys(fulltexts):

    keys = []

    if not 'pdf' in fulltexts:
        return keys

    for language, url in fulltexts['pdf'].items():
        path = REGEX_PDF_PATH.search(url)
        if path:
            keys.append(path.group().upper())

    return keys


def fbpe_key(code):
    """
    input:
        'S0102-67202009000300001'
    output:
        'S0102-6720(09)000300001'
    """

    begin = code[0:10]
    year = code[12:14]
    end = code[14:]

    return '%s(%s)%s' % (begin, year, end)


def eligible_match_keys(document):
    keys = []

    keys.append(document.publisher_id)
    keys.append(fbpe_key(document.publisher_id))
    if document.doi:
        keys.append(document.doi)
    keys += pdf_keys(document.fulltexts())

    suppl = ''.join([document.issue.supplement_volume or '', document.issue.supplement_number or '']).strip()

    try:
        leg = URLegendarium(
            acron=document.journal.acronym,
            year_pub=document.publication_date[:4],
            volume=document.issue.volume,
            number=document.issue.number,
            fpage=document.start_page,
            fpage_sequence=document.start_page_sequence,
            lpage=document.end_page,
            article_id=document.elocation,
            suppl_number=suppl,
            doi=document.doi,
            order=document.issue.order
        )
    except ValueError as e:
        logger.error(
            'Fail to build legendarium eligible match key for %s_%s',
            document.collection_acronym, document.publisher_id
        )
        logger.exception(e)
        leg = ''

    url_code = '/%s/' % leg.url_article if leg else ''

    if url_code:
        keys.append(url_code.upper())

    return keys


def country(country):
    if country in choices.ISO_3166:
        return country
    if country in choices.ISO_3166_COUNTRY_AS_KEY:
        return choices.ISO_3166_COUNTRY_AS_KEY[country]
    return 'undefined'


def get_date_timestamp(date):
    try:
        return datetime.datetime.strptime(date, '%Y-%m').isoformat()
    except ValueError:
        try:
            return datetime.datetime.strptime(date, '%Y-%m-%d').isoformat()
{'fraction_non_alphanumeric': 0.12000605326876514, 'fraction_numerical': 0.02360774818401937, 'mean_word_length': 2.936271590232281, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2929800', 'n_tokens_mistral': 2304, 'n_tokens_neox': 2133, 'n_words': 417}