<commit_before>import ConfigParser
import datetime
import json
import logging
import MySQLdb
import os
import urllib2

from dirq.queue import Queue, QueueError
from rest_framework.pagination import PaginationSerializer
from django.conf import settings
from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
from rest_framework.response import Response
from rest_framework.views import APIView


class CloudRecordView(APIView):
    """
    Submit Cloud Accounting Records.
    .../cloud/record

    Will save Cloud Accounting Records for later loading.
    """

    def post(self, request, format=None):
        """
        Submit Cloud Accounting Records.

        .../cloud/record

        Will save Cloud Accounting Records for later loading.
        """
        logger = logging.getLogger(__name__)

        try:
            empaid = request.META['HTTP_EMPA_ID']
        except KeyError:
            empaid = 'noid'

        logger.info("Received message. ID = %s", empaid)

        try:
            signer = request.META['SSL_CLIENT_S_DN']
        except KeyError:
            logger.error("No DN supplied in header")
            return Response(status=401)
 
        # authorise DNs here
        if not self._signer_is_valid(signer):
            logger.error("%s not a valid provider" % signer)
            return Response(status=403)

        if "_content" in request.POST.dict():
            # then POST likely to come via the rest api framework
            # hence use the content of request.POST as message
            body = request.POST.get('_content')

        else:
            # POST likely to comes through a browser client or curl
            # hence use request.body as message
            body = request.body

        logger.debug("Message body received: %s", body)

        for header in request.META:
            logger.debug("%s: %s", header, request.META[header])

        # taken from ssm2
        QSCHEMA = {'body': 'string',
                   'signer': 'string',
                   'empaid': 'string?'}

        inqpath = os.path.join(settings.QPATH, 'incoming')
        inq = Queue(inqpath, schema=QSCHEMA)

        try:
            name = inq.add({'body': body,
                            'signer': signer,
                            'empaid': empaid})
        except QueueError as err:
            logger.error("Could not save %s/%s: %s", inqpath, name, err)

            response = "Data could not be saved to disk, please try again."
            return Response(response, status=500)

        logger.info("Message saved to in queue as %s/%s", inqpath, name)

        response = "Data successfully saved for future loading."
        return Response(response, status=202)

###############################################################################
#                                                                             #
# Helper methods                                                              #
#                                                                             #
###############################################################################

    def _signer_is_valid(self, signer):
        site_list = urllib2.Request('http://indigo.cloud.plgrid.pl/cmdb/service/list')
        site_list = urllib2.urlopen(site_list)

        site_json = json.loads(site_list.read())

        for site in range(len(site_json['rows'])):
            try:
                if signer in site_json['rows'][site]['value']['hostname']:
                    return True
            except KeyError:
                pass

        # If we have not returned while in for loop
        # then site must be invalid
        return False
<commit_msg>Raise an exception if site data cannot be found<commit_after>import ConfigParser
import datetime
import httplib
import json
import logging
import MySQLdb
import os
import urllib2

from dirq.queue import Queue, QueueError
from rest_framework.pagination import PaginationSerializer
from django.conf import settings
from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
from rest_framework.response import Response
from rest_framework.views import APIView


class CloudRecordView(APIView):
    """
    Submit Cloud Accounting Records.
    .../cloud/record

    Will save Cloud Accounting Records for later loading.
    """

    def post(self, request, format=None):
        """
        Submit Cloud Accounting Records.

        .../cloud/record

        Will save Cloud Accounting Records for later loading.
        """
        logger = logging.getLogger(__name__)

        try:
            empaid = request.META['HTTP_EMPA_ID']
        except KeyError:
            empaid = 'noid'

        logger.info("Received message. ID = %s", empaid)

        try:
            signer = request.META['SSL_CLIENT_S_DN']
        except KeyError:
            logger.error("No DN supplied in header")
            return Response(status=401)
 
        # authorise DNs here
        if not self._signer_is_valid(signer):
            logger.error("%s not a valid provider" % signer)
            return Response(status=403)

        if "_content" in request.POST.dict():
            # then POST likely to come via the rest api framework
            # hence use the content of request.POST as message
            body = request.POST.get('_content')

        else:
            # POST likely to comes through a browser client or curl
            # hence use request.body as message
            body = request.body

        logger.debug("Message body received: %s", body)

        for header in request.META:
            logger.debug("%s: %s", header, request.META[header])

        # taken from ssm2
        QSCHEMA = {'body': 'string',
                   'signer': 'string',
                   'empaid': 'string?'}

        inqpath = os.path.join(settings.QPATH, 'incoming')
        inq = Queue(inqpath, schema=QSCHEMA)

        try:
            name = inq.add({'body': body,
                            'signer': signer,
                            'empaid': empaid})
        except QueueError as err:
            logger.error("Could not save %s/%s: %s", inqpath, name, err)

            response = "Data could not be saved to disk, please try again."
            return Response(response, status=500)

        logger.info("Message saved to in queue as %s/%s", inqpath, name)

        response = "Data successfully saved for future loading."
        return Response(response, status=202)

###############################################################################
#                                                                             #
# Helper methods                                                              #
#                                                                             #
###############################################################################

    def _signer_is_valid(self, signer):
        logger = logging.getLogger(__name__)      
  
        site_list = urllib2.Request('http://indigo.cloud.plgrid.pl/cmdb/service/list')
        site_list = urllib2.urlopen(site_list)

        try:
            site_json = json.loads(site_list.read())
        except ValueError:
            logger.error("List of providers could not be retrieved.")
            return False

        for site in range(len(site_json['rows'])):
            try:
                if signer in site_json['rows'][site]['value']['hostname']:
                    return True
            except KeyError:
                pass

        # If we have not returned while in for loop
        # then site must be invalid
        logger.info('Site is not found on list of providers')
        return False

{'fraction_non_alphanumeric': 0.12334286456979464, 'fraction_numerical': 0.004159085001299714, 'mean_word_length': 1.8531701890989989, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '94345', 'n_tokens_mistral': 1964, 'n_tokens_neox': 1867, 'n_words': 589}