<commit_before># coding: utf-8

import os
import urllib2
import json
import datetime
import urlparse
import re

from logaccess_config import *
import apachelog

MONTH_DICT = {
    'JAN': '01',
    'FEB': '02',
    'MAR': '03',
    'APR': '04',
    'MAY': '05',
    'JUN': '06',
    'JUL': '07',
    'AUG': '08',
    'SEP': '09',
    'OCT': '10',
    'NOV': '11',
    'DEC': '12',
}

ROBOTS = [i.strip() for i in open('/'.join([os.path.dirname(__file__), ROBOTS_FILE]))]
COMPILED_ROBOTS = [re.compile(i.lower()) for i in ROBOTS]
REGEX_ISSN = re.compile("^[0-9]{4}-[0-9]{3}[0-9xX]$")
REGEX_ISSUE = re.compile("^[0-9]{4}-[0-9]{3}[0-9xX][0-2][0-9]{3}[0-9]{4}$")
REGEX_ARTICLE = re.compile("^[0-9]{4}-[0-9]{3}[0-9xX][0-2][0-9]{3}[0-9]{4}[0-9]{5}$")
REGEX_FBPE = re.compile("^[0-9]{4}-[0-9]{3}[0-9xX]\([0-9]{2}\)[0-9]{8}$")


class AccessChecker(object):

    def __init__(self, collection=None, counter_compliant=False):
        self._parser = apachelog.parser(APACHE_LOG_FORMAT)
        allowed_collections = self._allowed_collections()

        if not collection in allowed_collections:
            raise ValueError('Invalid collection id ({0}), you must select one of these {1}'.format(collection, str(allowed_collections)))

        self.collection = collection
        self.acronym_to_issn_dict = self._acronym_to_issn_dict()
        self.allowed_issns = self._allowed_issns(self.acronym_to_issn_dict)

    def _allowed_collections(self):
        allowed_collections = []
        try:
            json_network = urllib2.urlopen('{0}/api/v1/collection'.format(ARTICLE_META_URL), timeout=3).read()
        except urllib2.URLError:
            raise urllib2.URLError('Was not possible to connect to articlemeta api, try again later!')

        network = json.loads(json_network)

        for collection in network:
            allowed_collections.append(collection['acron'])

        return allowed_collections

    def _acronym_to_issn_dict(self):
        """
        Create a acronym dictionay with valid issns. The issn's are the issn's
        used as id in the SciELO Website.
        """
        query_url = '{0}/api/v1/journal?collection={1}'.format(ARTICLE_META_URL, self.collection)

        try:
            titles_json = urllib2.urlopen(query_url, timeout=3).read()
        except urllib2.URLError:
            raise urllib2.URLError('Was not possible to connect to webservices.scielo.org, try again later!')

        titles = json.loads(titles_json)

        title_dict = {}
        for title in titles:
            title_dict[title['v68'][0]['_']] = title['v400'][0]['_']

        return title_dict

    def _allowed_issns(self, acronym_to_issn):
        issns = []
        for issn in acronym_to_issn.values():
            issns.append(issn)

        return issns

    def _parse_line(self, raw_line):
        try:
            return self._parser.parse(raw_line)
        except:
            return None

    def _query_string(self, url):
        """
        Given a request from a access log line in these formats:
            'GET /scielo.php?script=sci_nlinks&ref=000144&pid=S0103-4014200000020001300010&lng=pt HTTP/1.1'
            'GET http://www.scielo.br/scielo.php?script=sci_nlinks&ref=000144&pid=S0103-4014200000020001300010&lng=pt HTTP/1.1'
        The method must retrieve the query_string dictionary.

        """
        try:
            url = url.split(' ')[1]
        except IndexError:
            return None

        qs = dict((k,v[0]) for k,v in urlparse.parse_qs(urlparse.urlparse(url).query).items())

        if len(qs) > 0:
            return qs

    def _access_date(self, access_date):
        """
        Given a date from a access log line in this format: [30/Dec/2012:23:59:57 -0200]
        The method must retrieve a valid iso date 2012-12-30 or None
        """

        try:
            return datetime.datetime.strptime(access_date[1:21], '%d/%b/%Y:%H:%M:%S')
        except:
            return None

    def _pdf_or_html_access(self, get):
        if "GET" in get and ".pdf" in get:
            return "PDF"

        if "GET" in get and "scielo.php" in get and "script" in get and "pid" in get:
            return "HTML"

        return None

    def _is_valid_html_request(self, script, pid):

        pid = pid.upper().replace('S', '')

        if not pid[0:9] in self.allowed_issns:
            return False

        if script == u"sci_arttext" and (REGEX_ARTICLE.search(pid) or REGEX_FBPE.search(pid)):
            return True

        if script == u"sci_abstract" and (REGEX_ARTICLE.search(pid) or REGEX_FBPE.search(pid)):
            return True

        if script == u"sci_pdf" and (REGEX_ARTICLE.search(pid) or REGEX_FBPE.search(pid)):
            return True

        if script == u"sci_serial" and REGEX_ISSN.search(pid):
            return True

        if script == u"sci_issuetoc" and REGEX_ISSUE.search(pid):
            return True

        if script == u"sci_issues" and REGEX_ISSN.search(pid):
            return True
<commit_msg>Ajustes<commit_after># coding: utf-8

import os
import urllib2
import json
import datetime
import urlparse
import re

from logaccess_config import *
import apachelog

MONTH_DICT = {
    'JAN': '01',
    'FEB': '02',
    'MAR': '03',
    'APR': '04',
    'MAY': '05',
    'JUN': '06',
    'JUL': '07',
    'AUG': '08',
    'SEP': '09',
    'OCT': '10',
    'NOV': '11',
    'DEC': '12',
}

ROBOTS = [i.strip() for i in open('/'.join([os.path.dirname(__file__), ROBOTS_FILE]))]
COMPILED_ROBOTS = [re.compile(i.lower()) for i in ROBOTS]
REGEX_ISSN = re.compile("^[0-9]{4}-[0-9]{3}[0-9xX]$")
REGEX_ISSUE = re.compile("^[0-9]{4}-[0-9]{3}[0-9xX][0-2][0-9]{3}[0-9]{4}$")
REGEX_ARTICLE = re.compile("^[0-9]{4}-[0-9]{3}[0-9xX][0-2][0-9]{3}[0-9]{4}[0-9]{5}$")
REGEX_FBPE = re.compile("^[0-9]{4}-[0-9]{3}[0-9xX]\([0-9]{2}\)[0-9]{8}$")


class AccessChecker(object):

    def __init__(self, collection=None, counter_compliant=False):
        self._parser = apachelog.parser(APACHE_LOG_FORMAT)
        allowed_collections = self._allowed_collections()

        if not collection in allowed_collections:
            raise ValueError('Invalid collection id ({0}), you must select one of these {1}'.format(collection, str(allowed_collections)))

        self.collection = collection
        self.acronym_to_issn_dict = self._acronym_to_issn_dict()
        self.allowed_issns = self._allowed_issns(self.acronym_to_issn_dict)

    def _allowed_collections(self):
        allowed_collections = []
        try:
            json_network = urllib2.urlopen('{0}/api/v1/collection'.format(ARTICLE_META_URL), timeout=3).read()
        except urllib2.URLError:
            raise urllib2.URLError('Was not possible to connect to articlemeta api, try again later!')

        network = json.loads(json_network)

        for collection in network:
            allowed_collections.append(collection['acron'])

        return allowed_collections

    def _acronym_to_issn_dict(self):
        """
        Create a acronym dictionay with valid issns. The issn's are the issn's
        used as id in the SciELO Website.
        """
        query_url = '{0}/api/v1/journal?collection={1}'.format(ARTICLE_META_URL, self.collection)

        try:
            titles_json = urllib2.urlopen(query_url, timeout=3).read()
        except urllib2.URLError:
            raise urllib2.URLError('Was not possible to connect to webservices.scielo.org, try again later!')

        titles = json.loads(titles_json)

        title_dict = {}
        for title in titles:
            title_dict[title['v68'][0]['_']] = title['v400'][0]['_']

        return title_dict

    def _allowed_issns(self, acronym_to_issn):
        issns = []
        for issn in acronym_to_issn.values():
            issns.append(issn)

        return issns

    def _parse_line(self, raw_line):
        try:
            return self._parser.parse(raw_line)
        except:
            return None

    def _query_string(self, url):
        """
        Given a request from a access log line in these formats:
            'GET /scielo.php?script=sci_nlinks&ref=000144&pid=S0103-4014200000020001300010&lng=pt HTTP/1.1'
            'GET http://www.scielo.br/scielo.php?script=sci_nlinks&ref=000144&pid=S0103-4014200000020001300010&lng=pt HTTP/1.1'
        The method must retrieve the query_string dictionary.

        """
        try:
            url = url.split(' ')[1]
        except IndexError:
            return None

        qs = dict((k,v[0]) for k,v in urlparse.parse_qs(urlparse.urlparse(url).query).items())

        if len(qs) > 0:
            return qs

    def _access_date(self, access_date):
        """
        Given a date from a access log line in this format: [30/Dec/2012:23:59:57 -0200]
        The method must retrieve a valid iso date 2012-12-30 or None
        """

        try:
            return datetime.datetime.strptime(access_date[1:21], '%d/%b/%Y:%H:%M:%S')
        except:
            return None

    def _pdf_or_html_access(self, get):
        if "GET" in get and ".pdf" in get:
            return "PDF"

        if "GET" in get and "scielo.php" in get and "script" in get and "pid" in get:
            return "HTML"

        return None

    def _is_valid_html_request(self, script, pid):

        pid = pid.upper().replace('S', '')

        if not pid[0:9] in self.allowed_issns:
            return False

        if script == "sci_arttext" and (REGEX_ARTICLE.search(pid) or REGEX_FBPE.search(pid)):
            return True

        if script == "sci_abstract" and (REGEX_ARTICLE.search(pid) or REGEX_FBPE.search(pid)):
            return True

        if script == "sci_pdf" and (REGEX_ARTICLE.search(pid) or REGEX_FBPE.search(pid)):
            return True

        if script == "sci_serial" and REGEX_ISSN.search(pid):
            return True

        if script == "sci_issuetoc" and REGEX_ISSUE.search(pid):
            return True

        if script == "sci_issues" and REGEX_ISSN.search(pid):
            return True

{'fraction_non_alphanumeric': 0.1365824308062575, 'fraction_numerical': 0.041115122342559166, 'mean_word_length': 2.5630582350839584, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '886124', 'n_tokens_mistral': 3689, 'n_tokens_neox': 3424, 'n_words': 752}