<commit_before>#!python

import sys

import datetime
import functools
import glob
import json
import logging
import os
import signal
import time
import uuid

from pkg_resources import Requirement, resource_filename

import clize
from clize import Parameter
from flask import Flask, render_template, request, jsonify # Response

from ambassador.config import Config
from ambassador.VERSION import Version
from ambassador.utils import RichStatus, SystemInfo, PeriodicTrigger

from .envoy import EnvoyStats

__version__ = Version

boot_time = datetime.datetime.now()
last_scout_update = datetime.datetime.now() - datetime.timedelta(hours=24)

logging.basicConfig(
    level=logging.INFO,
    format="%%(asctime)s diagd %s %%(levelname)s: %%(message)s" % __version__,
    datefmt="%Y-%m-%d %H:%M:%S"
)

# Shut up Werkzeug's standard request logs -- they're just too noisy.
logging.getLogger("werkzeug").setLevel(logging.CRITICAL)

ambassador_targets = {
    'mapping': 'https://www.getambassador.io/reference/configuration#mappings',
    'module': 'https://www.getambassador.io/reference/configuration#modules',
}

envoy_targets = {
    'route': 'https://envoyproxy.github.io/envoy/configuration/http_conn_man/route_config/route.html',
    'cluster': 'https://envoyproxy.github.io/envoy/configuration/cluster_manager/cluster.html',
}

######## DECORATORS

def standard_handler(f):
    func_name = getattr(f, '__name__', '<anonymous>')

    @functools.wraps(f)
    def wrapper(*args, **kwds):
        reqid = str(uuid.uuid4()).upper()
        prefix = "%s: %s \"%s %s\"" % (reqid, request.remote_addr, request.method, request.path)

        start = datetime.datetime.now()

        app.logger.debug("%s handler %s" % (prefix, func_name))

        result = ("impossible error", 500)
        status_to_log = 500
        result_to_log = "impossible error"
        result_log_level = logging.ERROR

        try:
            result = f(*args, reqid=reqid, **kwds)

            if not isinstance(result, tuple):
                result = (result, 200)

            status_to_log = result[1]

            if (status_to_log // 100) == 2:
                result_log_level = logging.DEBUG
                result_to_log = "success"
            else:
                result_log_level = logging.ERROR
                result_to_log = "failure"
        except Exception as e:
            result_to_log = "server error"
            status_to_log = 500
            result_log_level = logging.ERROR
            result = (result_to_log, status_to_log)

            app.logger.exception(e)

        end = datetime.datetime.now()
        ms = int(((end - start).total_seconds() * 1000) + .5)

        app.logger.log(result_log_level, "%s %dms %d %s" % (prefix, ms, status_to_log, result_to_log))

        return result

    return wrapper

# Get the Flask app defined early.
app = Flask(__name__,
            template_folder=resource_filename(Requirement.parse("ambassador"), "templates"))

# Next, various helpers.
def aconf(app):
    configs = glob.glob("%s-*" % app.config_dir_prefix)

    if configs:
        configs.sort(key=lambda x: int(x.split("-")[-1]))
        latest = configs[-1]
    else:
        latest = app.config_dir_prefix

    aconf = Config(latest)

    # How long since the last Scout update? If it's been more than an hour, 
    # check Scout again.

    now = datetime.datetime.now()

    if (now - last_scout_update) > datetime.timedelta(hours=1):
        uptime = now - boot_time
        hr_uptime = td_format(uptime)

        result = Config.scout_report(mode="diagd", runtime=Config.runtime,
                                     uptime=uptime, hr_uptime=hr_uptime)

        app.logger.debug("Scout reports %s" % json.dumps(result))

    return aconf

def td_format(td_object):
    seconds = int(td_object.total_seconds())
    periods = [
        ('year',   60*60*24*365),
        ('month',  60*60*24*30),
        ('day',    60*60*24),
        ('hour',   60*60),
        ('minute', 60),
        ('second', 1)
    ]

    strings=[]
    for period_name,period_seconds in periods:
        if seconds > period_seconds:
            period_value, seconds = divmod(seconds,period_seconds)

            strings.append("%d %s%s" % 
                           (period_value, period_name, "" if (period_value == 1) else "s"))

    return ", ".join(strings)<commit_msg>Remember to send diagd uptime in seconds.<commit_after>#!python

import sys

import datetime
import functools
import glob
import json
import logging
import os
import signal
import time
import uuid

from pkg_resources import Requirement, resource_filename

import clize
from clize import Parameter
from flask import Flask, render_template, request, jsonify # Response

from ambassador.config import Config
from ambassador.VERSION import Version
from ambassador.utils import RichStatus, SystemInfo, PeriodicTrigger

from .envoy import EnvoyStats

__version__ = Version

boot_time = datetime.datetime.now()
last_scout_update = datetime.datetime.now() - datetime.timedelta(hours=24)

logging.basicConfig(
    level=logging.INFO,
    format="%%(asctime)s diagd %s %%(levelname)s: %%(message)s" % __version__,
    datefmt="%Y-%m-%d %H:%M:%S"
)

# Shut up Werkzeug's standard request logs -- they're just too noisy.
logging.getLogger("werkzeug").setLevel(logging.CRITICAL)

ambassador_targets = {
    'mapping': 'https://www.getambassador.io/reference/configuration#mappings',
    'module': 'https://www.getambassador.io/reference/configuration#modules',
}

envoy_targets = {
    'route': 'https://envoyproxy.github.io/envoy/configuration/http_conn_man/route_config/route.html',
    'cluster': 'https://envoyproxy.github.io/envoy/configuration/cluster_manager/cluster.html',
}

######## DECORATORS

def standard_handler(f):
    func_name = getattr(f, '__name__', '<anonymous>')

    @functools.wraps(f)
    def wrapper(*args, **kwds):
        reqid = str(uuid.uuid4()).upper()
        prefix = "%s: %s \"%s %s\"" % (reqid, request.remote_addr, request.method, request.path)

        start = datetime.datetime.now()

        app.logger.debug("%s handler %s" % (prefix, func_name))

        result = ("impossible error", 500)
        status_to_log = 500
        result_to_log = "impossible error"
        result_log_level = logging.ERROR

        try:
            result = f(*args, reqid=reqid, **kwds)

            if not isinstance(result, tuple):
                result = (result, 200)

            status_to_log = result[1]

            if (status_to_log // 100) == 2:
                result_log_level = logging.DEBUG
                result_to_log = "success"
            else:
                result_log_level = logging.ERROR
                result_to_log = "failure"
        except Exception as e:
            result_to_log = "server error"
            status_to_log = 500
            result_log_level = logging.ERROR
            result = (result_to_log, status_to_log)

            app.logger.exception(e)

        end = datetime.datetime.now()
        ms = int(((end - start).total_seconds() * 1000) + .5)

        app.logger.log(result_log_level, "%s %dms %d %s" % (prefix, ms, status_to_log, result_to_log))

        return result

    return wrapper

# Get the Flask app defined early.
app = Flask(__name__,
            template_folder=resource_filename(Requirement.parse("ambassador"), "templates"))

# Next, various helpers.
def aconf(app):
    configs = glob.glob("%s-*" % app.config_dir_prefix)

    if configs:
        configs.sort(key=lambda x: int(x.split("-")[-1]))
        latest = configs[-1]
    else:
        latest = app.config_dir_prefix

    aconf = Config(latest)

    # How long since the last Scout update? If it's been more than an hour, 
    # check Scout again.

    now = datetime.datetime.now()

    if (now - last_scout_update) > datetime.timedelta(hours=1):
        uptime = now - boot_time
        hr_uptime = td_format(uptime)

        result = Config.scout_report(mode="diagd", runtime=Config.runtime,
                                     uptime=int(uptime.total_seconds()),
                                     hr_uptime=hr_uptime)

        app.logger.debug("Scout reports %s" % json.dumps(result))

    return aconf

def td_format(td_object):
    seconds = int(td_object.total_seconds())
    periods = [
        ('year',   60*60*24*365),
        ('month',  60*60*24*30),
        ('day',    60*60*24),
        ('hour',   60*60),
        ('minute', 60),
        ('second', 1)
    ]

    strings=[]
    for period_name,period_seconds in periods:
        if seconds > period_seconds:
            period_value, seconds = divmod(seconds,period_seconds)

            strings.append("%d %s%s" % 
                           (period_value, period_name, "" if (period_value == 1) else "s"))

    return ", ".join(strings)
{'fraction_non_alphanumeric': 0.12343714480563764, 'fraction_numerical': 0.013412139122527847, 'mean_word_length': 2.715793918918919, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 8, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3849815', 'n_tokens_mistral': 2864, 'n_tokens_neox': 2690, 'n_words': 617}