<commit_before>
import os
import re
import imp
import json
import time
import types
import socket
import logging
import xml.sax.saxutils
import urlparse

import genshi.template

import classad
htcondor = None

import htcondorce.rrd
import htcondorce.web_utils

_initialized = None
_loader = None
_view = None
_plugins = []
g_is_multice = False
OK_STATUS = '200 OK'

log = logging.getLogger(__name__)
log.setLevel(logging.DEBUG)


def validate_plugin(name, plugin):
    try:
        for regex, callback in plugin.urls:
            if not hasattr(regex, "match"):
                log.warn("plugin %s: %r is not a compiled regex (real type: %s)" % (name, regex, type(regex)))
                return False
            if not callable(callback):
                log.warn("plugin %s: regex %r does not have a valid callback (real type: %s)"
                      % (name, regex.pattern, type(callback)))
                return False
    except (ValueError, AttributeError) as err:
        log.warn("plugin %s: %s" % (name, err))
        return False
    return True


def check_initialized(environ):
    global _initialized
    global _loader
    global _cp
    global _plugins
    global htcondor

    if not _initialized:
        if 'htcondorce.templates' in environ:
            _loader = genshi.template.TemplateLoader(environ['htcondorce.templates'], auto_reload=True)
        else:
            _loader = genshi.template.TemplateLoader('/usr/share/condor-ce/templates', auto_reload=True)
        ce_config = environ.get('htcondorce.config', '/etc/condor-ce/condor_config')
        htcondor = htcondorce.web_utils.check_htcondor()

        plugins_dir = htcondor.param.get("HTCONDORCE_VIEW_PLUGINS_DIR", "/usr/share/condor-ce/ceview-plugins")
        if os.path.isdir(plugins_dir):
            for filename in sorted(os.listdir(plugins_dir)):
                if not filename.endswith(".py"):
                    continue
                name = filename[:-3]
                plugin = imp.load_source(name, os.path.join(plugins_dir, filename))
                if validate_plugin(name, plugin):
                    log.debug("plugin %s: loaded ok", name)
                    _plugins.append(plugin)

        _initialized = True


def _headers(content_type):
    return [('Content-type', content_type),
            ('Cache-Control', 'max-age=60, public')]


class ClassadEncoder(json.JSONEncoder):
    def default(self, obj):
        if isinstance(obj, classad.ExprTree):
            return str(obj)
        else:
            return json.JSONEncoder.default(self, obj)


def ad_to_json(ad):
    result = {}
    for (key, val) in ad.items():
        if isinstance(val, classad.ExprTree):
            val = val.eval()
        if val == classad.Value.Undefined:
            val = None
        result[key] = val
    return result


def schedds(environ, start_response):
    ads = htcondorce.web_utils.get_schedd_ads(environ)
    results = {}
    for ad in ads:
        if 'Name' not in ad:
            continue
        results[ad['Name']] = ad_to_json(ad)

    start_response(OK_STATUS, _headers('application/json'))

    return [ json.dumps(results, cls=ClassadEncoder) ]
<commit_msg>CEView: Handle classad.Value.Undefined in ClassAd to JSON<commit_after>
import os
import re
import imp
import json
import time
import types
import socket
import logging
import xml.sax.saxutils
import urlparse

import genshi.template

import classad
htcondor = None

import htcondorce.rrd
import htcondorce.web_utils

_initialized = None
_loader = None
_view = None
_plugins = []
g_is_multice = False
OK_STATUS = '200 OK'

log = logging.getLogger(__name__)
log.setLevel(logging.DEBUG)


def validate_plugin(name, plugin):
    try:
        for regex, callback in plugin.urls:
            if not hasattr(regex, "match"):
                log.warn("plugin %s: %r is not a compiled regex (real type: %s)" % (name, regex, type(regex)))
                return False
            if not callable(callback):
                log.warn("plugin %s: regex %r does not have a valid callback (real type: %s)"
                      % (name, regex.pattern, type(callback)))
                return False
    except (ValueError, AttributeError) as err:
        log.warn("plugin %s: %s" % (name, err))
        return False
    return True


def check_initialized(environ):
    global _initialized
    global _loader
    global _cp
    global _plugins
    global htcondor

    if not _initialized:
        if 'htcondorce.templates' in environ:
            _loader = genshi.template.TemplateLoader(environ['htcondorce.templates'], auto_reload=True)
        else:
            _loader = genshi.template.TemplateLoader('/usr/share/condor-ce/templates', auto_reload=True)
        ce_config = environ.get('htcondorce.config', '/etc/condor-ce/condor_config')
        htcondor = htcondorce.web_utils.check_htcondor()

        plugins_dir = htcondor.param.get("HTCONDORCE_VIEW_PLUGINS_DIR", "/usr/share/condor-ce/ceview-plugins")
        if os.path.isdir(plugins_dir):
            for filename in sorted(os.listdir(plugins_dir)):
                if not filename.endswith(".py"):
                    continue
                name = filename[:-3]
                plugin = imp.load_source(name, os.path.join(plugins_dir, filename))
                if validate_plugin(name, plugin):
                    log.debug("plugin %s: loaded ok", name)
                    _plugins.append(plugin)

        _initialized = True


def _headers(content_type):
    return [('Content-type', content_type),
            ('Cache-Control', 'max-age=60, public')]


def ad_to_json(ad):
    result = classad.ClassAd()
    for (key, val) in ad.items():
        # Evaluate Condor expressions
        if isinstance(val, classad.ExprTree):
            val = val.eval()
        result[key] = val
    # Unfortunately, classad.Value.Undefined is of type int,
    # and json.dumps() converts it to "2".
    # Use HTCondor JSON conversion, then back to a Python object instead.
    return json.loads(result.printJson())


def schedds(environ, start_response):
    ads = htcondorce.web_utils.get_schedd_ads(environ)
    results = {}
    for ad in ads:
        if 'Name' not in ad:
            continue
        results[ad['Name']] = ad_to_json(ad)

    start_response(OK_STATUS, _headers('application/json'))

    return [ json.dumps(results) ]

{'fraction_non_alphanumeric': 0.10440944881889763, 'fraction_numerical': 0.002047244094488189, 'mean_word_length': 2.5302946081156197, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3671772', 'n_tokens_mistral': 1948, 'n_tokens_neox': 1907, 'n_words': 489}