<commit_before># coding: utf-8

import os
import urllib2
import json
import datetime
import urlparse
import re
import logging

import apachelog

from logger import utils
from articlemeta.client import ThriftClient

logger = logging.getLogger(__name__)

MONTH_DICT = {
    'JAN': '01',
    'FEB': '02',
    'MAR': '03',
    'APR': '04',
    'MAY': '05',
    'JUN': '06',
    'JUL': '07',
    'AUG': '08',
    'SEP': '09',
    'OCT': '10',
    'NOV': '11',
    'DEC': '12',
}

ROBOTS = [i.strip() for i in open(utils.settings.get('robots_file', 'robots.txt'))]
APACHE_LOG_FORMAT = utils.settings.get(
    'log_format',
    r'= %h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"')
COMPILED_ROBOTS = [re.compile(i.lower()) for i in ROBOTS]
REGEX_ISSN = re.compile(
    "^[0-9]{4}-[0-9]{3}[0-9xX]$")
REGEX_ISSUE = re.compile(
    "^[0-9]{4}-[0-9]{3}[0-9xX][0-2][0-9]{3}[0-9]{4}$")
REGEX_ARTICLE = re.compile(
    "^[0-9]{4}-[0-9]{3}[0-9xX][0-2][0-9]{3}[0-9]{4}[0-9]{5}$")
REGEX_FBPE = re.compile(
    "^[0-9]{4}-[0-9]{3}[0-9xX]\([0-9]{2}\)[0-9]{8}$")

am_client = ThriftClient(domain='articlemeta.scielo.org:11621')


def _allowed_collections():
    """Obtém a lista das coleções a partir do ArticleMeta
    """
    allowed_collections = []

    try:
        collections = am_client.collections()
    except:
        logger.error('Fail to retrieve collections from thrift server')

    return [i.code for i in collections]
<commit_msg>Usa a classe Collections para obter as coleções "permitidas"<commit_after># coding: utf-8

import datetime
import urlparse
import re
import logging

import apachelog

from logger import utils
from articlemeta.client import ThriftClient

logger = logging.getLogger(__name__)

MONTH_DICT = {
    'JAN': '01',
    'FEB': '02',
    'MAR': '03',
    'APR': '04',
    'MAY': '05',
    'JUN': '06',
    'JUL': '07',
    'AUG': '08',
    'SEP': '09',
    'OCT': '10',
    'NOV': '11',
    'DEC': '12',
}

ROBOTS = [i.strip() for i in open(utils.settings.get('robots_file', 'robots.txt'))]
APACHE_LOG_FORMAT = utils.settings.get(
    'log_format',
    r'= %h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"')
COMPILED_ROBOTS = [re.compile(i.lower()) for i in ROBOTS]
REGEX_ISSN = re.compile(
    "^[0-9]{4}-[0-9]{3}[0-9xX]$")
REGEX_ISSUE = re.compile(
    "^[0-9]{4}-[0-9]{3}[0-9xX][0-2][0-9]{3}[0-9]{4}$")
REGEX_ARTICLE = re.compile(
    "^[0-9]{4}-[0-9]{3}[0-9xX][0-2][0-9]{3}[0-9]{4}[0-9]{5}$")
REGEX_FBPE = re.compile(
    "^[0-9]{4}-[0-9]{3}[0-9xX]\([0-9]{2}\)[0-9]{8}$")

am_client = ThriftClient(domain='articlemeta.scielo.org:11621')


def _allowed_collections():
    COLLECTIONS = utils.Collections(am_client)
    return COLLECTIONS.codes()

{'fraction_non_alphanumeric': 0.2318840579710145, 'fraction_numerical': 0.0650315867707172, 'mean_word_length': 3.6898954703832754, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7185326', 'n_tokens_mistral': 1232, 'n_tokens_neox': 1183, 'n_words': 225}