<commit_before>"""
Utils to get schedresources info from dedicated information system (CRIC)
"""
import urllib3
import json
import logging

from django.core.cache import cache

from core.schedresource.models import SchedconfigJson
from core.settings.config import CRIC_API_URL, DEPLOYMENT

_logger = logging.getLogger('bigpandamon')


def get_CRIC_panda_queues():
    """Get PanDA queues config from CRIC and put to cache"""
    panda_queues_dict = cache.get(f'pandaQueues{DEPLOYMENT}')
    if not panda_queues_dict:
        panda_queues_dict = {}
        url = CRIC_API_URL
        http = urllib3.PoolManager()
        try:
            r = http.request('GET', url)
            data = json.loads(r.data.decode('utf-8'))
            for pq, params in data.items():
                if DEPLOYMENT == 'ORACLE_ATLAS':
                    if 'vo_name' in params and params['vo_name'] == 'atlas':
                        panda_queues_dict[pq] = params
                if DEPLOYMENT == 'ORACLE_DOMA':
                    if 'vo_name' in params and params['vo_name'] in ['osg', 'atlas']:
                        panda_queues_dict[pq] = params
        except Exception as exc:
            print (exc)
        cache.set(f'pandaQueues{DEPLOYMENT}', panda_queues_dict, 3600)
    return panda_queues_dict


def get_panda_queues():
    """
    Get PanDA queues info from available sources, priority: CRIC -> SchedconfigJson table -> Schedconfig
    :return: dict of PQs
    """
    # try get info from CRIC
    try:
        panda_queues_dict = get_CRIC_panda_queues()
    except:
        panda_queues_dict = None
        _logger.error("[JSR] cannot get json from CRIC")

    if not panda_queues_dict:
        # get data from new SCHEDCONFIGJSON table
        panda_queues_list = []
        panda_queues_dict = {}
        panda_queues_list.extend(SchedconfigJson.objects.values())
        if len(panda_queues_list) > 0:
            for pq in panda_queues_list:
                try:
                    panda_queues_dict[pq['pandaqueue']] = json.loads(pq['data'])
                except:
                    panda_queues_dict[pq['pandaqueue']] = None
                    _logger.error("cannot load json from SCHEDCONFIGJSON table for {} PanDA queue".format(pq['pandaqueue']))<commit_msg>decrease time to store CRIC data in cache<commit_after>"""
Utils to get schedresources info from dedicated information system (CRIC)
"""
import urllib3
import json
import logging

from django.core.cache import cache

from core.schedresource.models import SchedconfigJson
from core.settings.config import CRIC_API_URL, DEPLOYMENT

_logger = logging.getLogger('bigpandamon')


def get_CRIC_panda_queues():
    """Get PanDA queues config from CRIC and put to cache"""
    panda_queues_dict = cache.get(f'pandaQueues{DEPLOYMENT}')
    if not panda_queues_dict:
        panda_queues_dict = {}
        url = CRIC_API_URL
        http = urllib3.PoolManager()
        try:
            r = http.request('GET', url)
            data = json.loads(r.data.decode('utf-8'))
            for pq, params in data.items():
                if DEPLOYMENT == 'ORACLE_ATLAS':
                    if 'vo_name' in params and params['vo_name'] == 'atlas':
                        panda_queues_dict[pq] = params
                if DEPLOYMENT == 'ORACLE_DOMA':
                    if 'vo_name' in params and params['vo_name'] in ['osg', 'atlas']:
                        panda_queues_dict[pq] = params
        except Exception as exc:
            print (exc)
        cache.set(f'pandaQueues{DEPLOYMENT}', panda_queues_dict, 60*20)
    return panda_queues_dict


def get_panda_queues():
    """
    Get PanDA queues info from available sources, priority: CRIC -> SchedconfigJson table -> Schedconfig
    :return: dict of PQs
    """
    # try get info from CRIC
    try:
        panda_queues_dict = get_CRIC_panda_queues()
    except:
        panda_queues_dict = None
        _logger.error("[JSR] cannot get json from CRIC")

    if not panda_queues_dict:
        # get data from new SCHEDCONFIGJSON table
        panda_queues_list = []
        panda_queues_dict = {}
        panda_queues_list.extend(SchedconfigJson.objects.values())
        if len(panda_queues_list) > 0:
            for pq in panda_queues_list:
                try:
                    panda_queues_dict[pq['pandaqueue']] = json.loads(pq['data'])
                except:
                    panda_queues_dict[pq['pandaqueue']] = None
                    _logger.error("cannot load json from SCHEDCONFIGJSON table for {} PanDA queue".format(pq['pandaqueue']))
{'fraction_non_alphanumeric': 0.09433136353687897, 'fraction_numerical': 0.0035018603633180127, 'mean_word_length': 2.415545590433483, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4709618', 'n_tokens_mistral': 1484, 'n_tokens_neox': 1420, 'n_words': 350}