<commit_before># This file is part of Indico.
# Copyright (C) 2002 - 2015 European Organization for Nuclear Research (CERN).
#
# Indico is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# Indico is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Indico; if not, see <http://www.gnu.org/licenses/>.

"""
Base export interface
"""

# python stdlib imports
import itertools
import pytz
import re
import urllib
from datetime import datetime, timedelta, time
from types import GeneratorType

import transaction
from flask import request
from ZEO.Exceptions import ClientDisconnected
from ZODB.POSException import ConflictError

# indico imports
from MaKaC.common.mail import GenericMailer
from indico.core.db import DBMgr
from indico.core.db.util import flush_after_commit_queue
from indico.util.date_time import nowutc
from indico.util.fossilize import fossilize
from indico.web.http_api.metadata import Serializer
from indico.web.http_api.metadata.html import HTML4Serializer
from indico.web.http_api.metadata.jsonp import JSONPSerializer
from indico.web.http_api.metadata.ical import ICalSerializer
from indico.web.http_api.metadata.atom import AtomSerializer
from indico.web.http_api.responses import HTTPAPIError
from indico.web.http_api.util import get_query_parameter
from indico.web.http_api.exceptions import ArgumentParseError, LimitExceededException

# indico legacy imports
from MaKaC.common.info import HelperMaKaCInfo
from MaKaC.plugins.base import PluginsHolder
from MaKaC.common.logger import Logger


class HTTPAPIHook(object):
    """This class is the hook between the query (path+params) and the generator of the results (fossil).
       It is also in charge of checking the parameters and the access rights.
    """

    HOOK_LIST = []
    TYPES = None  # abstract
    PREFIX = 'export'  # url prefix. must exist in indico.web.flask.blueprints.api, too! also used as function prefix
    RE = None  # abstract
    METHOD_NAME = None  # overrides method name derived from prefix+type
    DEFAULT_DETAIL = None  # abstract
    MAX_RECORDS = {}
    SERIALIZER_TYPE_MAP = {}  # maps fossil type names to friendly names (useful for plugins e.g. RoomCERN --> Room)
    VALID_FORMATS = None  # None = all formats
    GUEST_ALLOWED = True  # When False, it forces authentication
    COMMIT = False  # commit database changes
    HTTP_POST = False  # require (and allow) HTTP POST
    NO_CACHE = False

    @classmethod
    def parseRequest(cls, path, queryParams):
        """Parse a request path and return a hook and the requested data type."""
        path = urllib.unquote(path)
        hooks = itertools.chain(cls.HOOK_LIST, cls._getPluginHooks())
        for expCls in hooks:
            Logger.get('HTTPAPIHook.parseRequest').debug(expCls)
            m = expCls._matchPath(path)
            if m:
                gd = m.groupdict()
                g = m.groups()
                type = g[0]
                format = g[-1]
                if format not in DataFetcher.getAllowedFormats():
                    return None, None
                elif expCls.VALID_FORMATS and format not in expCls.VALID_FORMATS:
                    return None, None
                return expCls(queryParams, type, gd), format
        return None, None

    @staticmethod
    def register(cls):
        """Register a hook that is not part of a plugin.

        To use it, simply decorate the hook class with this method."""
        assert cls.RE is not None
        HTTPAPIHook.HOOK_LIST.append(cls)
        return cls

    @classmethod
    def _matchPath(cls, path):
        if not hasattr(cls, '_RE'):
            types = '|'.join(cls.TYPES)
            cls._RE = re.compile(r'/' + cls.PREFIX + '/(' + types + r')/' + cls.RE + r'\.(\w+)$')
        return cls._RE.match(path)

    @classmethod
    def _getPluginHooks(cls):
        for plugin in PluginsHolder().getPluginTypes():
            for expClsName in plugin.getHTTPAPIHookList():
                yield getattr(plugin.getModule().http_api, expClsName)

    def __init__(self, queryParams, type, pathParams):
        self._queryParams = queryParams
        self._type = type
        self._pathParams = pathParams

    def _getParams(self):
        self._offset = get_query_parameter(self._queryParams, ['O', 'offset'], 0, integer=True)
        if self._offset < 0:
            raise HTTPAPIError('Offset must be a positive number', 400)
        self._orderBy = get_query_parameter(self._queryParams, ['o', 'order'])
        self._descending = get_query_parameter(self._queryParams, ['c', 'descending'], 'no') == 'yes'
        self._detail = get_query_parameter(self._queryParams, ['d', 'detail'], self.DEFAULT_DETAIL)
        tzName = get_query_parameter(self._queryParams, ['tz'], None)

        info = HelperMaKaCInfo.getMaKaCInfoInstance()
        self._serverTZ = info.getTimezone()

        if tzName is None:
            tzName = self._serverTZ
        try:
            self._tz = pytz.timezone(tzName)
        except pytz.UnknownTimeZoneError, e:
            raise HTTPAPIError("Bad timezone: '%s'" % e.message, 400)
        max = self.MAX_RECORDS.get(self._detail, 1000)
        self._userLimit = get_query_parameter(self._queryParams, ['n', 'limit'], 0, integer=True)
        if self._userLimit > max:
            raise HTTPAPIError("You can only request up to %d records per request with the detail level '%s'" %
                               (max, self._detail), 400)
        self._limit = self._userLimit if self._userLimit > 0 else max

        fromDT = get_query_parameter(self._queryParams, ['f', 'from'])
        toDT = get_query_parameter(self._queryParams, ['t', 'to'])
        dayDT = get_query_parameter(self._queryParams, ['day'])

        if (fromDT or toDT) and dayDT:
            raise HTTPAPIError("'day' can only be used without 'from' and 'to'", 400)
        elif dayDT:
            fromDT = toDT = dayDT

        self._fromDT = DataFetcher._getDateTime('from', fromDT, self._tz) if fromDT else None
        self._toDT = DataFetcher._getDateTime('to', toDT, self._tz, aux=self._fromDT) if toDT else None

    def _hasAccess(self, aw):
        return True

    @property
    def serializer_args(self):
        return {}

    def _getMethodName(self):
        if self.METHOD_NAME:
            return self.METHOD_NAME
        return self.PREFIX + '_' + self._type

    def _performCall(self, func, aw):
        resultList = []
        complete = True
        try:
            res = func(aw)
            if isinstance(res, GeneratorType):
                for obj in res:
                    resultList.append(obj)
            else:
                resultList = res
        except LimitExceededException:
            complete = (self._limit == self._userLimit)
        return resultList, complete

    def _perform(self, aw, func, extra_func):
        self._getParams()
        if not self._hasAccess(aw):
            raise HTTPAPIError('Access to this resource is restricted.', 403)
        resultList, complete = self._performCall(func, aw)
        extra = extra_func(aw, resultList) if extra_func else None
        return resultList, complete, extra

    def __call__(self, aw):<commit_msg>Support API hooks with empty RE<commit_after># This file is part of Indico.
# Copyright (C) 2002 - 2015 European Organization for Nuclear Research (CERN).
#
# Indico is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# Indico is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Indico; if not, see <http://www.gnu.org/licenses/>.

"""
Base export interface
"""

# python stdlib imports
import itertools
import pytz
import re
import urllib
from datetime import datetime, timedelta, time
from types import GeneratorType

import transaction
from flask import request
from ZEO.Exceptions import ClientDisconnected
from ZODB.POSException import ConflictError

# indico imports
from MaKaC.common.mail import GenericMailer
from indico.core.db import DBMgr
from indico.core.db.util import flush_after_commit_queue
from indico.util.date_time import nowutc
from indico.util.fossilize import fossilize
from indico.web.http_api.metadata import Serializer
from indico.web.http_api.metadata.html import HTML4Serializer
from indico.web.http_api.metadata.jsonp import JSONPSerializer
from indico.web.http_api.metadata.ical import ICalSerializer
from indico.web.http_api.metadata.atom import AtomSerializer
from indico.web.http_api.responses import HTTPAPIError
from indico.web.http_api.util import get_query_parameter
from indico.web.http_api.exceptions import ArgumentParseError, LimitExceededException

# indico legacy imports
from MaKaC.common.info import HelperMaKaCInfo
from MaKaC.plugins.base import PluginsHolder
from MaKaC.common.logger import Logger


class HTTPAPIHook(object):
    """This class is the hook between the query (path+params) and the generator of the results (fossil).
       It is also in charge of checking the parameters and the access rights.
    """

    HOOK_LIST = []
    TYPES = None  # abstract
    PREFIX = 'export'  # url prefix. must exist in indico.web.flask.blueprints.api, too! also used as function prefix
    RE = None  # abstract
    METHOD_NAME = None  # overrides method name derived from prefix+type
    DEFAULT_DETAIL = None  # abstract
    MAX_RECORDS = {}
    SERIALIZER_TYPE_MAP = {}  # maps fossil type names to friendly names (useful for plugins e.g. RoomCERN --> Room)
    VALID_FORMATS = None  # None = all formats
    GUEST_ALLOWED = True  # When False, it forces authentication
    COMMIT = False  # commit database changes
    HTTP_POST = False  # require (and allow) HTTP POST
    NO_CACHE = False

    @classmethod
    def parseRequest(cls, path, queryParams):
        """Parse a request path and return a hook and the requested data type."""
        path = urllib.unquote(path)
        hooks = itertools.chain(cls.HOOK_LIST, cls._getPluginHooks())
        for expCls in hooks:
            Logger.get('HTTPAPIHook.parseRequest').debug(expCls)
            m = expCls._matchPath(path)
            if m:
                gd = m.groupdict()
                g = m.groups()
                type = g[0]
                format = g[-1]
                if format not in DataFetcher.getAllowedFormats():
                    return None, None
                elif expCls.VALID_FORMATS and format not in expCls.VALID_FORMATS:
                    return None, None
                return expCls(queryParams, type, gd), format
        return None, None

    @staticmethod
    def register(cls):
        """Register a hook that is not part of a plugin.

        To use it, simply decorate the hook class with this method."""
        assert cls.RE is not None
        HTTPAPIHook.HOOK_LIST.append(cls)
        return cls

    @classmethod
    def _matchPath(cls, path):
        if not hasattr(cls, '_RE'):
            types = '|'.join(cls.TYPES)
            cls._RE = re.compile(r'/' + cls.PREFIX + '/(' + types + r')' + ('/' + cls.RE).rstrip('/') + r'\.(\w+)$')
        return cls._RE.match(path)

    @classmethod
    def _getPluginHooks(cls):
        for plugin in PluginsHolder().getPluginTypes():
            for expClsName in plugin.getHTTPAPIHookList():
                yield getattr(plugin.getModule().http_api, expClsName)

    def __init__(self, queryParams, type, pathParams):
        self._queryParams = queryParams
        self._type = type
        self._pathParams = pathParams

    def _getParams(self):
        self._offset = get_query_parameter(self._queryParams, ['O', 'offset'], 0, integer=True)
        if self._offset < 0:
            raise HTTPAPIError('Offset must be a positive number', 400)
        self._orderBy = get_query_parameter(self._queryParams, ['o', 'order'])
        self._descending = get_query_parameter(self._queryParams, ['c', 'descending'], 'no') == 'yes'
        self._detail = get_query_parameter(self._queryParams, ['d', 'detail'], self.DEFAULT_DETAIL)
        tzName = get_query_parameter(self._queryParams, ['tz'], None)

        info = HelperMaKaCInfo.getMaKaCInfoInstance()
        self._serverTZ = info.getTimezone()

        if tzName is None:
            tzName = self._serverTZ
        try:
            self._tz = pytz.timezone(tzName)
        except pytz.UnknownTimeZoneError, e:
            raise HTTPAPIError("Bad timezone: '%s'" % e.message, 400)
        max = self.MAX_RECORDS.get(self._detail, 1000)
        self._userLimit = get_query_parameter(self._queryParams, ['n', 'limit'], 0, integer=True)
        if self._userLimit > max:
            raise HTTPAPIError("You can only request up to %d records per request with the detail level '%s'" %
                               (max, self._detail), 400)
        self._limit = self._userLimit if self._userLimit > 0 else max

        fromDT = get_query_parameter(self._queryParams, ['f', 'from'])
        toDT = get_query_parameter(self._queryParams, ['t', 'to'])
        dayDT = get_query_parameter(self._queryParams, ['day'])

        if (fromDT or toDT) and dayDT:
            raise HTTPAPIError("'day' can only be used without 'from' and 'to'", 400)
        elif dayDT:
            fromDT = toDT = dayDT

        self._fromDT = DataFetcher._getDateTime('from', fromDT, self._tz) if fromDT else None
        self._toDT = DataFetcher._getDateTime('to', toDT, self._tz, aux=self._fromDT) if toDT else None

    def _hasAccess(self, aw):
        return True

    @property
    def serializer_args(self):
        return {}

    def _getMethodName(self):
        if self.METHOD_NAME:
            return self.METHOD_NAME
        return self.PREFIX + '_' + self._type.replace('-', '_')

    def _performCall(self, func, aw):
        resultList = []
        complete = True
        try:
            res = func(aw)
            if isinstance(res, GeneratorType):
                for obj in res:
                    resultList.append(obj)
            else:
                resultList = res
        except LimitExceededException:
            complete = (self._limit == self._userLimit)
        return resultList, complete

    def _perform(self, aw, func, extra_func):
        self._getParams()
        if not self._hasAccess(aw):
            raise HTTPAPIError('Access to this resource is restricted.', 403)
        resultList, complete = self._performCall(func, aw)
        extra = extra_func(aw, resultList) if extra_func else None
        return resultList, complete, extra

    def __call__(self, aw):
{'fraction_non_alphanumeric': 0.089665554028405, 'fraction_numerical': 0.004581451665684927, 'mean_word_length': 2.7478538140789794, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7228082', 'n_tokens_mistral': 4475, 'n_tokens_neox': 4319, 'n_words': 1398}