<commit_before># This file is part of Indico.
# Copyright (C) 2002 - 2016 European Organization for Nuclear Research (CERN).
#
# Indico is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# Indico is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Indico; if not, see <http://www.gnu.org/licenses/>.

"""
HTTP API - Handlers
"""

import hashlib
import hmac
import posixpath
import re
import time
import urllib
from uuid import UUID

import transaction
from flask import request, session, g, current_app
from urlparse import parse_qs
from werkzeug.exceptions import NotFound, BadRequest

from indico.core.db import DBMgr
from indico.core.config import Config
from indico.core.logger import Logger
from indico.modules.api import APIMode
from indico.modules.api import settings as api_settings
from indico.modules.api.models.keys import APIKey
from indico.modules.oauth import oauth
from indico.modules.oauth.provider import load_token
from indico.util.contextManager import ContextManager
from indico.util.string import to_unicode
from indico.web.http_api import HTTPAPIHook
from indico.web.http_api.responses import HTTPAPIResult, HTTPAPIError
from indico.web.http_api.util import get_query_parameter
from indico.web.http_api.fossils import IHTTPAPIExportResultFossil
from indico.web.http_api.metadata.serializer import Serializer
from indico.web.flask.util import ResponseUtil

from MaKaC.common.fossilize import fossilize, clearCache
from MaKaC.accessControl import AccessWrapper
from MaKaC.common.cache import GenericCache


# Remove the extension at the end or before the querystring
RE_REMOVE_EXTENSION = re.compile(r'\.(\w+)(?:$|(?=\?))')


def normalizeQuery(path, query, remove=('signature',), separate=False):
    """Normalize request path and query so it can be used for caching and signing

    Returns a string consisting of path and sorted query string.
    Dynamic arguments like signature and timestamp are removed from the query string.
    """
    qparams = parse_qs(query)
    sorted_params = []

    for key, values in sorted(qparams.items(), key=lambda x: x[0].lower()):
        key = key.lower()
        if key not in remove:
            for v in sorted(values):
                sorted_params.append((key, v))

    if separate:
        return path, sorted_params and urllib.urlencode(sorted_params)
    elif sorted_params:
        return '%s?%s' % (path, urllib.urlencode(sorted_params))
    else:
        return path


def validateSignature(ak, signature, timestamp, path, query):
    ttl = api_settings.get('signature_ttl')
    if not timestamp and not (ak.is_persistent_allowed and api_settings.get('allow_persistent')):
        raise HTTPAPIError('Signature invalid (no timestamp)', 403)
    elif timestamp and abs(timestamp - int(time.time())) > ttl:
        raise HTTPAPIError('Signature invalid (bad timestamp)', 403)
    digest = hmac.new(ak.secret, normalizeQuery(path, query), hashlib.sha1).hexdigest()
    if signature != digest:
        raise HTTPAPIError('Signature invalid', 403)


def checkAK(apiKey, signature, timestamp, path, query):
    apiMode = api_settings.get('security_mode')
    if not apiKey:
        if apiMode in {APIMode.ONLYKEY, APIMode.ONLYKEY_SIGNED, APIMode.ALL_SIGNED}:
            raise HTTPAPIError('API key is missing', 403)
        return None, True
    try:
        UUID(hex=apiKey)
    except ValueError:
        raise HTTPAPIError('Malformed API key', 400)
    ak = APIKey.find_first(token=apiKey, is_active=True)
    if not ak:
        raise HTTPAPIError('Invalid API key', 403)
    if ak.is_blocked:
        raise HTTPAPIError('API key is blocked', 403)
    # Signature validation
    onlyPublic = False
    if signature:
        validateSignature(ak, signature, timestamp, path, query)
    elif apiMode == APIMode.ALL_SIGNED:
        raise HTTPAPIError('Signature missing', 403)
    elif apiMode in {APIMode.SIGNED, APIMode.ONLYKEY_SIGNED}:
        onlyPublic = True
    return ak, onlyPublic


def buildAW(ak, onlyPublic=False):
    aw = AccessWrapper()
    if ak and not onlyPublic:
        # If we have an authenticated request, require HTTPS
        # Dirty hack: Google calendar converts HTTP API requests from https to http
        # Therefore, not working with Indico setup (requiring https for HTTP API authenticated)
        if not request.is_secure and api_settings.get('require_https') and request.user_agent.browser != 'google':
            raise HTTPAPIError('HTTPS is required', 403)
        aw.setUser(ak.user.as_avatar)
    return aw


def handler(prefix, path):
    path = posixpath.join('/', prefix, path)
    ContextManager.destroy()
    clearCache()  # init fossil cache
    logger = Logger.get('httpapi')
    if request.method == 'POST':
        # Convert POST data to a query string
        queryParams = dict((key, value.encode('utf-8')) for key, value in request.form.iteritems())
        query = urllib.urlencode(queryParams)
    else:
        # Parse the actual query string
        queryParams = dict((key, value.encode('utf-8')) for key, value in request.args.iteritems())<commit_msg>Fix API signature validation with multiple values<commit_after># This file is part of Indico.
# Copyright (C) 2002 - 2016 European Organization for Nuclear Research (CERN).
#
# Indico is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# Indico is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Indico; if not, see <http://www.gnu.org/licenses/>.

"""
HTTP API - Handlers
"""

import hashlib
import hmac
import posixpath
import re
import time
import urllib
from uuid import UUID

import transaction
from flask import request, session, g, current_app
from urlparse import parse_qs
from werkzeug.exceptions import NotFound, BadRequest

from indico.core.db import DBMgr
from indico.core.config import Config
from indico.core.logger import Logger
from indico.modules.api import APIMode
from indico.modules.api import settings as api_settings
from indico.modules.api.models.keys import APIKey
from indico.modules.oauth import oauth
from indico.modules.oauth.provider import load_token
from indico.util.contextManager import ContextManager
from indico.util.string import to_unicode
from indico.web.http_api import HTTPAPIHook
from indico.web.http_api.responses import HTTPAPIResult, HTTPAPIError
from indico.web.http_api.util import get_query_parameter
from indico.web.http_api.fossils import IHTTPAPIExportResultFossil
from indico.web.http_api.metadata.serializer import Serializer
from indico.web.flask.util import ResponseUtil

from MaKaC.common.fossilize import fossilize, clearCache
from MaKaC.accessControl import AccessWrapper
from MaKaC.common.cache import GenericCache


# Remove the extension at the end or before the querystring
RE_REMOVE_EXTENSION = re.compile(r'\.(\w+)(?:$|(?=\?))')


def normalizeQuery(path, query, remove=('signature',), separate=False):
    """Normalize request path and query so it can be used for caching and signing

    Returns a string consisting of path and sorted query string.
    Dynamic arguments like signature and timestamp are removed from the query string.
    """
    qparams = parse_qs(query)
    sorted_params = []

    for key, values in sorted(qparams.items(), key=lambda x: x[0].lower()):
        key = key.lower()
        if key not in remove:
            for v in sorted(values):
                sorted_params.append((key, v))

    if separate:
        return path, sorted_params and urllib.urlencode(sorted_params)
    elif sorted_params:
        return '%s?%s' % (path, urllib.urlencode(sorted_params))
    else:
        return path


def validateSignature(ak, signature, timestamp, path, query):
    ttl = api_settings.get('signature_ttl')
    if not timestamp and not (ak.is_persistent_allowed and api_settings.get('allow_persistent')):
        raise HTTPAPIError('Signature invalid (no timestamp)', 403)
    elif timestamp and abs(timestamp - int(time.time())) > ttl:
        raise HTTPAPIError('Signature invalid (bad timestamp)', 403)
    digest = hmac.new(ak.secret, normalizeQuery(path, query), hashlib.sha1).hexdigest()
    if signature != digest:
        raise HTTPAPIError('Signature invalid', 403)


def checkAK(apiKey, signature, timestamp, path, query):
    apiMode = api_settings.get('security_mode')
    if not apiKey:
        if apiMode in {APIMode.ONLYKEY, APIMode.ONLYKEY_SIGNED, APIMode.ALL_SIGNED}:
            raise HTTPAPIError('API key is missing', 403)
        return None, True
    try:
        UUID(hex=apiKey)
    except ValueError:
        raise HTTPAPIError('Malformed API key', 400)
    ak = APIKey.find_first(token=apiKey, is_active=True)
    if not ak:
        raise HTTPAPIError('Invalid API key', 403)
    if ak.is_blocked:
        raise HTTPAPIError('API key is blocked', 403)
    # Signature validation
    onlyPublic = False
    if signature:
        validateSignature(ak, signature, timestamp, path, query)
    elif apiMode == APIMode.ALL_SIGNED:
        raise HTTPAPIError('Signature missing', 403)
    elif apiMode in {APIMode.SIGNED, APIMode.ONLYKEY_SIGNED}:
        onlyPublic = True
    return ak, onlyPublic


def buildAW(ak, onlyPublic=False):
    aw = AccessWrapper()
    if ak and not onlyPublic:
        # If we have an authenticated request, require HTTPS
        # Dirty hack: Google calendar converts HTTP API requests from https to http
        # Therefore, not working with Indico setup (requiring https for HTTP API authenticated)
        if not request.is_secure and api_settings.get('require_https') and request.user_agent.browser != 'google':
            raise HTTPAPIError('HTTPS is required', 403)
        aw.setUser(ak.user.as_avatar)
    return aw


def handler(prefix, path):
    path = posixpath.join('/', prefix, path)
    ContextManager.destroy()
    clearCache()  # init fossil cache
    logger = Logger.get('httpapi')
    if request.method == 'POST':
        # Convert POST data to a query string
        queryParams = [(key, [x.encode('utf-8') for x in values]) for key, values in request.form.iterlists()]
        query = urllib.urlencode(queryParams, doseq=1)
        # we only need/keep multiple values so we can properly validate the signature.
        # the legacy code below expects a dict with just the first value.
        # if you write a new api endpoint that needs multiple values get them from
        # ``request.values.getlist()`` directly
        queryParams = {key: values[0] for key, values in queryParams}
    else:
        # Parse the actual query string
        queryParams = dict((key, value.encode('utf-8')) for key, value in request.args.iteritems())
{'fraction_non_alphanumeric': 0.08468657237646238, 'fraction_numerical': 0.0071590710668762, 'mean_word_length': 3.7178747940691927, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1186642', 'n_tokens_mistral': 3318, 'n_tokens_neox': 3203, 'n_words': 1172}