<commit_before># coding=utf-8

#  Copyright (c) 2001-2014, Canal TP and/or its affiliates. All rights reserved.
#
# This file is part of Navitia,
#     the software to build cool stuff with public transport.
#
# Hope you'll enjoy and contribute to this project,
#     powered by Canal TP (www.canaltp.fr).
# Help us simplify mobility and open public transport:
#     a non ending quest to the responsive locomotion way of traveling!
#
# LICENCE: This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# Stay tuned using
# twitter @navitia
# IRC #navitia on freenode
# https://groups.google.com/d/forum/navitia
# www.navitia.io

from flask import request, g
from functools import wraps
from navitiacommon import stat_pb2
from datetime import datetime
import logging
from jormungandr import app
from jormungandr.authentication import get_user, get_token
from jormungandr import utils

import time
import sys
import kombu
f_datetime = "%Y%m%dT%H%M%S"

def init_journey(stat_journey):
    stat_journey.requested_date_time = 0
    stat_journey.departure_date_time = 0
    stat_journey.arrival_date_time = 0
    stat_journey.duration = 0
    stat_journey.nb_transfers = 0
    stat_journey.type = ''


class StatManager(object):

    def __init__(self):
        if 'SAVE_STAT' in app.config:
            self.save_stat = app.config['SAVE_STAT']
        else:
            self.save_stat = False
        self.connection = None
        self.producer = None
        if 'BROKER_URL' in app.config:
            self.broker_url = app.config['BROKER_URL']
        else:
            self.broker_url = None
        if 'EXCHANGE_NAME' in app.config:
            self.exchange_name = app.config['EXCHANGE_NAME']
        else:
            self.exchange_name = None

        if self.save_stat:
            self._init_rabbitmq()

    def _init_rabbitmq(self):
        """
        connection to rabbitmq and initialize queues
        """
        try:
            self.connection = kombu.Connection(self.broker_url)
            exchange = kombu.Exchange(self.exchange_name, type="direct")
            self.producer = self.connection.Producer(exchange=exchange)
            self.save_stat = True
        except:
            self.save_stat = False
            logging.getLogger(__name__).warn('Unable to activate the producer of stat')

    def manage_stat(self, start_time, call_result):
        """
        Function to fill stat objects (requests, parameters, journeys et sections) sand send them to Broker
        """
        if not self.save_stat:
            return

        try:
            self._manage_stat(start_time, call_result)
        except Exception as e:
            #if stat are not working we don't want jormungandr to stop.
            logging.getLogger(__name__).exception('Error during stat management')

    def _manage_stat(self, start_time, call_result):
        end_time = time.time()
        stat_request = stat_pb2.StatRequest()
        stat_request.request_duration = int((end_time - start_time) * 1000) #In milliseconds
        self.fill_request(stat_request, call_result)
        self.fill_coverages(stat_request)
        self.fill_parameters(stat_request)
        self.fill_result(stat_request, call_result)
        self.publish_request(stat_request)

    def fill_result(self, stat_request, call_result):
        if 'error' in call_result[0]:
            self.fill_error(stat_request, call_result[0]['error'])

        #We do not save informations of journeys and sections for a request
        #Isochron (parameter "&to" is absent for API Journeys )
        if 'journeys' in request.endpoint and \
                'to' in request.args:
            self.fill_journeys(stat_request, call_result)

    def fill_request(self, stat_request, call_result):
        """
        Remplir requests
        """
        dt = datetime.utcnow()
        stat_request.request_date = int(time.mktime(dt.timetuple()))
        # Note: for stat we don't want to abort if no token has been
        # given (it's up to the authentication process)
        user = get_user(token=get_token(), abort_if_no_token=False)
        if user is not None:
            stat_request.user_id = user.id
            stat_request.user_name = user.login

        stat_request.application_id = -1
        stat_request.application_name = ''
        stat_request.api = request.endpoint
        stat_request.host = request.host_url
        if request.remote_addr and \
                not request.headers.getlist("X-Forwarded-For"):
            stat_request.client = request.remote_addr
        elif request.headers.getlist("X-Forwarded-For"):
            stat_request.client = request.headers.getlist("X-Forwarded-For")[0]

        stat_request.response_size = sys.getsizeof(call_result[0])

    def fill_parameters(self, stat_request):
        for item in request.args.iteritems():
            stat_parameter = stat_request.parameters.add()
            stat_parameter.key = item[0]
            stat_parameter.value = item[1]

    def fill_coverages(self, stat_request):
        if 'region' in request.view_args:
            stat_coverage = stat_request.coverages.add()
            stat_coverage.region_id = request.view_args['region']
        elif hasattr(g, 'regions_called'):
            for region_id in g.regions_called:
                stat_coverage = stat_request.coverages.add()
                stat_coverage.region_id = g.regions_called[0]

    def fill_error(self, stat_request, error):
        stat_error = stat_request.error
        if 'id' in error:
            stat_error.id = error['id']
        if 'message' in error:<commit_msg>Jormungandr: handle empty coverage in stats<commit_after># coding=utf-8

#  Copyright (c) 2001-2014, Canal TP and/or its affiliates. All rights reserved.
#
# This file is part of Navitia,
#     the software to build cool stuff with public transport.
#
# Hope you'll enjoy and contribute to this project,
#     powered by Canal TP (www.canaltp.fr).
# Help us simplify mobility and open public transport:
#     a non ending quest to the responsive locomotion way of traveling!
#
# LICENCE: This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# Stay tuned using
# twitter @navitia
# IRC #navitia on freenode
# https://groups.google.com/d/forum/navitia
# www.navitia.io

from flask import request, g
from functools import wraps
from navitiacommon import stat_pb2
from datetime import datetime
import logging
from jormungandr import app
from jormungandr.authentication import get_user, get_token
from jormungandr import utils

import time
import sys
import kombu
f_datetime = "%Y%m%dT%H%M%S"

def init_journey(stat_journey):
    stat_journey.requested_date_time = 0
    stat_journey.departure_date_time = 0
    stat_journey.arrival_date_time = 0
    stat_journey.duration = 0
    stat_journey.nb_transfers = 0
    stat_journey.type = ''


class StatManager(object):

    def __init__(self):
        if 'SAVE_STAT' in app.config:
            self.save_stat = app.config['SAVE_STAT']
        else:
            self.save_stat = False
        self.connection = None
        self.producer = None
        if 'BROKER_URL' in app.config:
            self.broker_url = app.config['BROKER_URL']
        else:
            self.broker_url = None
        if 'EXCHANGE_NAME' in app.config:
            self.exchange_name = app.config['EXCHANGE_NAME']
        else:
            self.exchange_name = None

        if self.save_stat:
            self._init_rabbitmq()

    def _init_rabbitmq(self):
        """
        connection to rabbitmq and initialize queues
        """
        try:
            self.connection = kombu.Connection(self.broker_url)
            exchange = kombu.Exchange(self.exchange_name, type="direct")
            self.producer = self.connection.Producer(exchange=exchange)
            self.save_stat = True
        except:
            self.save_stat = False
            logging.getLogger(__name__).warn('Unable to activate the producer of stat')

    def manage_stat(self, start_time, call_result):
        """
        Function to fill stat objects (requests, parameters, journeys et sections) sand send them to Broker
        """
        if not self.save_stat:
            return

        try:
            self._manage_stat(start_time, call_result)
        except Exception as e:
            #if stat are not working we don't want jormungandr to stop.
            logging.getLogger(__name__).exception('Error during stat management')

    def _manage_stat(self, start_time, call_result):
        end_time = time.time()
        stat_request = stat_pb2.StatRequest()
        stat_request.request_duration = int((end_time - start_time) * 1000) #In milliseconds
        self.fill_request(stat_request, call_result)
        self.fill_coverages(stat_request)
        self.fill_parameters(stat_request)
        self.fill_result(stat_request, call_result)
        self.publish_request(stat_request)

    def fill_result(self, stat_request, call_result):
        if 'error' in call_result[0]:
            self.fill_error(stat_request, call_result[0]['error'])

        #We do not save informations of journeys and sections for a request
        #Isochron (parameter "&to" is absent for API Journeys )
        if 'journeys' in request.endpoint and \
                'to' in request.args:
            self.fill_journeys(stat_request, call_result)

    def fill_request(self, stat_request, call_result):
        """
        Remplir requests
        """
        dt = datetime.utcnow()
        stat_request.request_date = int(time.mktime(dt.timetuple()))
        # Note: for stat we don't want to abort if no token has been
        # given (it's up to the authentication process)
        user = get_user(token=get_token(), abort_if_no_token=False)
        if user is not None:
            stat_request.user_id = user.id
            stat_request.user_name = user.login

        stat_request.application_id = -1
        stat_request.application_name = ''
        stat_request.api = request.endpoint
        stat_request.host = request.host_url
        if request.remote_addr and \
                not request.headers.getlist("X-Forwarded-For"):
            stat_request.client = request.remote_addr
        elif request.headers.getlist("X-Forwarded-For"):
            stat_request.client = request.headers.getlist("X-Forwarded-For")[0]

        stat_request.response_size = sys.getsizeof(call_result[0])

    def fill_parameters(self, stat_request):
        for item in request.args.iteritems():
            stat_parameter = stat_request.parameters.add()
            stat_parameter.key = item[0]
            stat_parameter.value = item[1]

    def fill_coverages(self, stat_request):
        if 'region' in request.view_args:
            stat_coverage = stat_request.coverages.add()
            stat_coverage.region_id = request.view_args['region']
        elif hasattr(g, 'regions_called'):
            for region_id in g.regions_called:
                stat_coverage = stat_request.coverages.add()
                stat_coverage.region_id = g.regions_called[0]
        else:
            # We need an empty coverage.
            stat_request.coverages.add()

    def fill_error(self, stat_request, error):
        stat_error = stat_request.error
        if 'id' in error:
            stat_error.id = error['id']
        if 'message' in error:
{'fraction_non_alphanumeric': 0.08232291001914487, 'fraction_numerical': 0.004626675175494576, 'mean_word_length': 2.6711566617862372, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 6, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7478894', 'n_tokens_mistral': 3711, 'n_tokens_neox': 3578, 'n_words': 1035}