<commit_before>import time
import datetime
import json
import functools
import traceback
import copy
import statistics

from pandacommon.pandalogger.PandaLogger import PandaLogger
from pandacommon.pandalogger import logger_utils

from pandaserver.config import panda_config


# logger
main_logger = PandaLogger().getLogger('metric_collector')


# list of metrics in FetchData to fetch data and update to DB
metric_list = [
    ('analy_pmerge_jobs_wait_time', 'site'),
]


def get_now_time_str():
    """
    Return string of nowtime that can be stored in DB
    """
    now_time = datetime.datetime.utcnow()
    ts_str = now_time.strftime('%Y-%m-%d %H:%M:%S')
    return ts_str


class MetricsDB(object):
    """
    Proxy to access the metrics table in DB
    """

    def __init__(self, tbuf):
        self.tbuf = tbuf

    def _decor(method):
        def _decorator(_method, *args, **kwargs):
            @functools.wraps(_method)
            def _wrapped_method(self, *args, **kwargs):
                try:
                    _method(self, *args, **kwargs)
                except Exception as exc:
                    pass
            return _wrapped_method
        return _decorator(method)

    def update(self, key, value, site, gshare):
        tmp_log = logger_utils.make_logger(main_logger, 'MetricsDB')
        tmp_log.debug('start key={0} site={1}, gshare={2}'.format(key, site, gshare))
        # sql
        sql_update_tmp = (
            """UPDATE ATLAS_PANDA.Metrics SET """
                """data_json = json_mergepatch(data_json, '{patch_data_json}'), """
                """timestamp_json = json_mergepatch(timestamp_json, '{patch_timestamp_json}') """
            """WHERE computingSite=:site AND gshare=:gshare """
        )
        sql_insert_tmp = (
            """INSERT INTO ATLAS_PANDA.Metrics """
                """VALUES ( """
                    """:site, :gshare, '{patch_data_json}', '{patch_timestamp_json}' """
                """) """
        )
        # var map
        varMap = {
            ':site': site,
            ':gshare': gshare,
        }
        # json string evaluated
        patch_data_dict = {}
        patch_timestamp_dict = {}
        try:
            patch_data_dict[key] = value
            patch_data_json = json.dumps(patch_data_dict)
        except Exception:
            tmp_log.error(traceback.format_exc())
            return
        try:
            now_time_str = get_now_time_str()
            patch_timestamp_dict[key] = now_time_str
            patch_timestamp_json = json.dumps(patch_timestamp_dict)
        except Exception:
            tmp_log.error(traceback.format_exc())
            return
        # json in sql
        sql_update = sql_update_tmp.format(patch_data_json=patch_data_json, patch_timestamp_json=patch_timestamp_json)
        sql_insert = sql_insert_tmp.format(patch_data_json=patch_data_json, patch_timestamp_json=patch_timestamp_json)
        # update
        n_row = self.tbuf.querySQL(sql_update, varMap)
        # try insert if no row updated
        if n_row == 0:
            try:
                tmp_log.debug('no row to update about site={site}, gshare={gshare} ; trying insert'.format(site=site, gshare=gshare))
                self.tbuf.querySQL(sql_insert, varMap)
                tmp_log.debug('inserted site={site}, gshare={gshare}'.format(site=site, gshare=gshare))
            except Exception:
                tmp_log.warning('failed to insert site={site}, gshare={gshare}'.format(site=site, gshare=gshare))
        else:
            tmp_log.debug('updated site={site}, gshare={gshare}'.format(site=site, gshare=gshare))
        # done
        tmp_log.debug('done key={0} site={1}, gshare={2}'.format(key, site, gshare))

    def update_site(self, key, value, site):
        return self.update(key, value, site=site, gshare='NULL')

    def update_gshare(self, key, value, gshare):
        return self.update(key, value, site='NULL', gshare=gshare)


class FetchData(object):<commit_msg>metric collector: table schema change<commit_after>import time
import datetime
import json
import functools
import traceback
import copy
import statistics

from pandacommon.pandalogger.PandaLogger import PandaLogger
from pandacommon.pandalogger import logger_utils

from pandaserver.config import panda_config


# logger
main_logger = PandaLogger().getLogger('metric_collector')


# list of metrics in FetchData to fetch data and update to DB
metric_list = [
    ('analy_pmerge_jobs_wait_time', 'site'),
]


def get_now_time_str():
    """
    Return string of nowtime that can be stored in DB
    """
    now_time = datetime.datetime.utcnow()
    ts_str = now_time.strftime('%Y-%m-%d %H:%M:%S')
    return ts_str


class MetricsDB(object):
    """
    Proxy to access the metrics table in DB
    """

    def __init__(self, tbuf):
        self.tbuf = tbuf

    def _decor(method):
        def _decorator(_method, *args, **kwargs):
            @functools.wraps(_method)
            def _wrapped_method(self, *args, **kwargs):
                try:
                    _method(self, *args, **kwargs)
                except Exception as exc:
                    pass
            return _wrapped_method
        return _decorator(method)

    def update(self, key, value, site, gshare):
        tmp_log = logger_utils.make_logger(main_logger, 'MetricsDB')
        tmp_log.debug('start key={0} site={1}, gshare={2}'.format(key, site, gshare))
        # sql
        sql_update_tmp = (
            """UPDATE ATLAS_PANDA.Metrics SET """
                """data_json = json_mergepatch(data_json, '{patch_data_json}') """
            """WHERE computingSite=:site AND gshare=:gshare """
        )
        sql_insert_tmp = (
            """INSERT INTO ATLAS_PANDA.Metrics """
                """VALUES ( """
                    """:site, :gshare, '{patch_data_json}' """
                """) """
        )
        # var map
        varMap = {
            ':site': site,
            ':gshare': gshare,
        }
        # json string evaluated
        try:
            now_time_str = get_now_time_str()
            patch_data_dict = dict()
            patch_data_dict[key] = {
                "value": value,
                "timestamp": now_time_str,
            }
            patch_data_json = json.dumps(patch_data_dict)
        except Exception:
            tmp_log.error(traceback.format_exc())
            return
        # json in sql
        sql_update = sql_update_tmp.format(patch_data_json=patch_data_json)
        sql_insert = sql_insert_tmp.format(patch_data_json=patch_data_json)
        # update
        n_row = self.tbuf.querySQL(sql_update, varMap)
        # try insert if no row updated
        if n_row == 0:
            try:
                tmp_log.debug('no row to update about site={site}, gshare={gshare} ; trying insert'.format(site=site, gshare=gshare))
                self.tbuf.querySQL(sql_insert, varMap)
                tmp_log.debug('inserted site={site}, gshare={gshare}'.format(site=site, gshare=gshare))
            except Exception:
                tmp_log.warning('failed to insert site={site}, gshare={gshare}'.format(site=site, gshare=gshare))
        else:
            tmp_log.debug('updated site={site}, gshare={gshare}'.format(site=site, gshare=gshare))
        # done
        tmp_log.debug('done key={0} site={1}, gshare={2}'.format(key, site, gshare))

    def update_site(self, key, value, site):
        return self.update(key, value, site=site, gshare='NULL')

    def update_gshare(self, key, value, gshare):
        return self.update(key, value, site='NULL', gshare=gshare)


class FetchData(object):
{'fraction_non_alphanumeric': 0.11422950819672131, 'fraction_numerical': 0.0018360655737704918, 'mean_word_length': 2.4181981174361273, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '126862', 'n_tokens_mistral': 2345, 'n_tokens_neox': 2283, 'n_words': 499}