<commit_before># coding: utf-8
"""
This scripts loads the issue section whiting a given article.
The lists of Sections must be given and respect the following format.
"collection acronym","language","code","text"
"""
from datetime import datetime, timedelta
import argparse
import logging
import re
import os
import requests

from pymongo import MongoClient
from articlemeta import utils
from xylose.scielodocument import Article

logger = logging.getLogger(__name__)

FROM = datetime.now() - timedelta(days=15)
FROM.isoformat()[:10]

file_regex = re.compile(r'serial.*.htm|.*.xml')
data_struct_regex = re.compile(r'^fulltexts\.(pdf|html)\.[a-z][a-z]$')

config = utils.Configuration.from_env()
settings = dict(config.items())

try:
    articlemeta_db = MongoClient(settings['app:main']['mongo_uri'])['articlemeta']
except:
    logging.error('Fail to connect to (%s)' % settings['app:main']['mongo_uri'])

def _config_logging(logging_level='INFO', logging_file=None):

    allowed_levels = {
        'DEBUG': logging.DEBUG,
        'INFO': logging.INFO,
        'WARNING': logging.WARNING,
        'ERROR': logging.ERROR,
        'CRITICAL': logging.CRITICAL
    }

    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')

    
    logger.setLevel(allowed_levels.get(logging_level, 'INFO'))

    if logging_file:
        hl = logging.FileHandler(logging_file, mode='a')
    else:
        hl = logging.StreamHandler()

    hl.setFormatter(formatter)
    hl.setLevel(allowed_levels.get(logging_level, 'INFO'))

    logger.addHandler(hl)

    return logger

def collections_acronym():

    collections = articlemeta_db['collections'].find({}, {'_id': 0})

    return [i['code'] for i in collections]

def collection_info(collection):

    info = articlemeta_db['collections'].find_one({'acron': collection}, {'_id': 0})

    return info

def do_request(url, json=True):


    headers = {
        'User-Agent': 'SciELO Processing ArticleMeta: LoadSection'
    }

    document = None
    try:
        document = requests.get(url, headers=headers)
    except:
        logger.error(u'HTTP request error for: %s' % url)

    if document.status_code == 404:
        logger.error(u'Data not found for: %s' % url)
        return None

    if json:
        return document.json()
    else:
        return document

def load_documents(collection, all_records=False):

    fltr = {
        'collection': collection
    }

    if all_records == False:
        fltr['section'] = {'$exists': 0}

    documents = articlemeta_db['articles'].find(
        fltr,
        {'code': 1}
    )

    pids = []
    for document in documents:
        pids.append(document['code'])

    if 'section' in fltr:
        del(fltr['section'])

    for pid in pids:
        fltr['code'] = pid
        document = articlemeta_db['articles'].find_one(
            fltr,
            {'_id': 0, 'citations': 0},
            no_cursor_timeout=True
        )
        yield Article(document)

    documents.close()

class StaticCatalog(object):

    def __init__(self, collection):
        self.catalog = self._load_static_catalog(collection)

    def _load_static_catalog(self, source):
        """
        source: www.scielo.br
        type: in ['section']

        Download the static files text lists from the selected SciELO Domain.
        Parse the txt file to produce a dictionary with the following structure:
        {
            'ven': {
                'code': {
                    'pt': 'section pt',
                    'es': 'section es',
                    ]
                }
            },
            ...
        }
        """

        logger.info(u'Loading static_section_catalog.txt from server %s' % (source['domain']))

        url = '/'.join(['http:/', source['domain'], 'static_section_catalog.txt'])
        content = do_request(url, json=False)

        if not content:
            logger.warning(u'Section catalog not found: %s' % url)
            return None

        content = cotent.iter_lines()

        items = set([i.decode('iso-8859-1') for i in content])

        sections = {}
        for line in sorted(items):
            splited_line = [i.strip() for i in line.split('|')]
            if len(splited_line) != 4:
                continue
            collection = source['code']
            issue = splited_line[0]
            language = splited_line[1]
            code = splited_line[2]
            label = splited_line[3]
<commit_msg>Correção em carga de seções<commit_after># coding: utf-8
"""
This scripts loads the issue section whiting a given article.
The lists of Sections must be given and respect the following format.
"collection acronym","language","code","text"
"""
from datetime import datetime, timedelta
import argparse
import logging
import re
import os
import requests

from pymongo import MongoClient
from articlemeta import utils
from xylose.scielodocument import Article

logger = logging.getLogger(__name__)

FROM = datetime.now() - timedelta(days=15)
FROM.isoformat()[:10]

file_regex = re.compile(r'serial.*.htm|.*.xml')
data_struct_regex = re.compile(r'^fulltexts\.(pdf|html)\.[a-z][a-z]$')

config = utils.Configuration.from_env()
settings = dict(config.items())

try:
    articlemeta_db = MongoClient(settings['app:main']['mongo_uri'])['articlemeta']
except:
    logging.error('Fail to connect to (%s)' % settings['app:main']['mongo_uri'])

def _config_logging(logging_level='INFO', logging_file=None):

    allowed_levels = {
        'DEBUG': logging.DEBUG,
        'INFO': logging.INFO,
        'WARNING': logging.WARNING,
        'ERROR': logging.ERROR,
        'CRITICAL': logging.CRITICAL
    }

    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')

    
    logger.setLevel(allowed_levels.get(logging_level, 'INFO'))

    if logging_file:
        hl = logging.FileHandler(logging_file, mode='a')
    else:
        hl = logging.StreamHandler()

    hl.setFormatter(formatter)
    hl.setLevel(allowed_levels.get(logging_level, 'INFO'))

    logger.addHandler(hl)

    return logger

def collections_acronym():

    collections = articlemeta_db['collections'].find({}, {'_id': 0})

    return [i['code'] for i in collections]

def collection_info(collection):

    info = articlemeta_db['collections'].find_one({'acron': collection}, {'_id': 0})

    return info

def do_request(url, json=True):


    headers = {
        'User-Agent': 'SciELO Processing ArticleMeta: LoadSection'
    }

    document = None
    try:
        document = requests.get(url, headers=headers)
    except:
        logger.error(u'HTTP request error for: %s' % url)

    if document.status_code == 404:
        logger.error(u'Data not found for: %s' % url)
        return None

    if json:
        return document.json()
    else:
        return document

def load_documents(collection, all_records=False):

    fltr = {
        'collection': collection
    }

    if all_records == False:
        fltr['section'] = {'$exists': 0}

    documents = articlemeta_db['articles'].find(
        fltr,
        {'code': 1}
    )

    pids = []
    for document in documents:
        pids.append(document['code'])

    if 'section' in fltr:
        del(fltr['section'])

    for pid in pids:
        fltr['code'] = pid
        document = articlemeta_db['articles'].find_one(
            fltr,
            {'_id': 0, 'citations': 0},
            no_cursor_timeout=True
        )
        yield Article(document)

    documents.close()

class StaticCatalog(object):

    def __init__(self, collection):
        self.catalog = self._load_static_catalog(collection)

    def _load_static_catalog(self, source):
        """
        source: www.scielo.br
        type: in ['section']

        Download the static files text lists from the selected SciELO Domain.
        Parse the txt file to produce a dictionary with the following structure:
        {
            'ven': {
                'code': {
                    'pt': 'section pt',
                    'es': 'section es',
                    ]
                }
            },
            ...
        }
        """

        logger.info(u'Loading static_section_catalog.txt from server %s' % (source['domain']))

        url = '/'.join(['http:/', source['domain'], 'static_section_catalog.txt'])
        content = do_request(url, json=False)

        if not content:
            logger.warning(u'Section catalog not found: %s' % url)
            return None

        content = content.iter_lines()

        items = set([i.decode('iso-8859-1') for i in content])

        sections = {}
        for line in sorted(items):
            splited_line = [i.strip() for i in line.split('|')]
            if len(splited_line) != 4:
                continue
            collection = source['code']
            issue = splited_line[0]
            language = splited_line[1]
            code = splited_line[2]
            label = splited_line[3]

{'fraction_non_alphanumeric': 0.12803038766618255, 'fraction_numerical': 0.005362529326332253, 'mean_word_length': 2.5092120736965895, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6510707', 'n_tokens_mistral': 2754, 'n_tokens_neox': 2603, 'n_words': 648}