<commit_before>import datetime
import json
import sys
import time
import os
import re

sys.path.append(os.environ.get('STACKTACH_INSTALL_DIR', '/stacktach'))
from stacktach import datetime_to_decimal as dt
from stacktach import image_type
from stacktach import models


if __name__ != '__main__':
    sys.exit(1)

# To mask unique identifiers for categorizing notifications
def mask_msg(text):
    masking_regex = {
        'REQ_ID': r"req-[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}",
        'UUID': r"[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}",
        'LG_NUM': r"\b\d{3}\d+\b",
        'HOST_ADDRESS': r"\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b"
    }
    masked = str(text)
    for mask, regex in masking_regex.iteritems():
        masked = re.sub(regex, "$%s" % str(mask), masked)
    return masked


# Assemble message from exception object
def build_exc_msg(exc=None, separator=", "):
    """
    If you get queasy with bad code smells, you might want to glaze over this
    function. Exception objects don't appear to be standardized yet. So,
    white-list the things we know we want about the exceptions we're counting
    and ignore the rest.
        -TMaddox
    """

    if exc is None:
        return exc

    message = []
    if exc.get('kwargs', False):
        kwargs = exc['kwargs']
        if kwargs.get('value', False):
            value = kwargs['value']
            trcbk_index = value.rfind("Traceback")
            if trcbk_index > 0:
                value = str(value[:trcbk_index]) + "$TRACEBACK"
            message.append("value: %s" % value)

        # kwargs: generic message components that don't require more filter
        misc_list = ['reason', 'method', 'topic', 'exc_type',
                     'actual', 'code']
        for key in misc_list:
            if kwargs.get(key, False):
                message.append("%s: %s" % (key, kwargs[key]))
        # END generic message components in kwargs

        if kwargs.get('expected', False):
            message.append("expected: %s" % kwargs['expected'][0])

    if exc.get('details', False):
        details = exc['details']
        if type(details) is list:
            for item in details:
                message.append(str(item))
        elif type(details) is dict:
            for k, v in details.iteritems():
                message.append("%s: %s" % (k, v))
        elif type(details) is str:
            message.append(details)

    # exc: generic messages that don't require more filter
    misc_list = ['message', 'cmd', 'stderr', 'exit_code',
                 'code', 'description']
    for key in misc_list:
        if exc.get(key, False):
            message.append("%s: %s" % (key, exc[key]))

    if exc.get('stdout', False):
        if exc['stdout'] != "":
            message.append("stdout: %s" % exc['stdout'])
    #END generic message components in exc

    if len(message) == 0:
        for k, v in exc.iteritems():
            message.append("%s: %s" % (k, v))
    return separator.join(message)

if __name__ == '__main__':

    # Start report
    yesterday = datetime.datetime.utcnow().date() - datetime.timedelta(days=1)
    if len(sys.argv) == 2:
        try:
            t = time.strptime(sys.argv[1], "%Y-%m-%d")
            yesterday = datetime.datetime(*t[:6])
        except Exception, e:
            print e
            print "Usage: python error_details.py YYYY-MM-DD (the end date)"
            sys.exit(1)

    hours = 0
    length = 24

    start = datetime.datetime(year=yesterday.year, month=yesterday.month,
                              day=yesterday.day)
    end = start + datetime.timedelta(hours=length-1, minutes=59, seconds=59)

    instance_map = {}  # { uuid : [request_id, request_id, ...] }
    exception_counts = {}  # { exception_message : count }
    event_counts = {}  # { event_name : count }
    metadata = {'report_format': 'json',
                'instances': instance_map,
                'exception_counts': exception_counts,
                'event_counts': event_counts
                }

    # Tell Stacky to format as JSON and set placeholders for various summaries
    report = [metadata]

    dstart = dt.dt_to_decimal(start)
    dend = dt.dt_to_decimal(end)

    codes = {}

    deployments = {}
    for deploy in models.Deployment.objects.all():
        deployments[deploy.id] = deploy.name

    # Get all the instances that have changed in the last N hours ...
    updates = models.RawData.objects.filter(event='compute.instance.update',
                                            when__gt=dstart, when__lte=dend)\
                                    .values('instance').distinct()

    expiry = 60 * 60  # 1 hour
    cmds = ['create', 'rebuild', 'rescue', 'resize', 'snapshot']

    failures = {}
    causes = {}
    durations = {}
    successes = {}
    tenant_issues = {}

    for uuid_dict in updates:
        uuid = uuid_dict['instance']

        # All the unique Request ID's for this instance during that timespan.
        reqs = models.RawData.objects.filter(instance=uuid,
                                             when__gt=dstart, when__lte=dend)\
                                     .values('request_id').distinct()

        req_list = []<commit_msg>changed to tuple for REGEX configuration<commit_after>import datetime
import json
import sys
import time
import os
import re

sys.path.append(os.environ.get('STACKTACH_INSTALL_DIR', '/stacktach'))
from stacktach import datetime_to_decimal as dt
from stacktach import image_type
from stacktach import models


if __name__ != '__main__':
    sys.exit(1)

# To mask unique identifiers for categorizing notifications
def mask_msg(text):
    masking_regex = (
        (1, 'REQ_ID', r"req-[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}"),
        (2, 'UUID', r"[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}"),
        (3, 'HOST_ADDRESS', r"\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b"),
        (4, 'LG_NUM', r"\b\d{3}\d+\b")
    )
    masked = str(text)
    for config in masking_regex:
        masked = re.sub(config[2], "$%s" % str(config[1]), masked)
    return masked


# Assemble message from exception object
def build_exc_msg(exc=None, separator=", "):
    """
    If you get queasy with bad code smells, you might want to glaze over this
    function. Exception objects don't appear to be standardized yet. So,
    white-list the things we know we want about the exceptions we're counting
    and ignore the rest.
        -TMaddox
    """

    if exc is None:
        return exc

    message = []
    if exc.get('kwargs', False):
        kwargs = exc['kwargs']
        if kwargs.get('value', False):
            value = kwargs['value']
            trcbk_index = value.rfind("Traceback")
            if trcbk_index > 0:
                value = str(value[:trcbk_index]) + "$TRACEBACK"
            message.append("value: %s" % value)

        # kwargs: generic message components that don't require more filter
        misc_list = ['reason', 'method', 'topic', 'exc_type',
                     'actual', 'code']
        for key in misc_list:
            if kwargs.get(key, False):
                message.append("%s: %s" % (key, kwargs[key]))
        # END generic message components in kwargs

        if kwargs.get('expected', False):
            message.append("expected: %s" % kwargs['expected'][0])

    if exc.get('details', False):
        details = exc['details']
        if type(details) is list:
            for item in details:
                message.append(str(item))
        elif type(details) is dict:
            for k, v in details.iteritems():
                message.append("%s: %s" % (k, v))
        elif type(details) is str:
            message.append(details)

    # exc: generic messages that don't require more filter
    misc_list = ['message', 'cmd', 'stderr', 'exit_code',
                 'code', 'description']
    for key in misc_list:
        if exc.get(key, False):
            message.append("%s: %s" % (key, exc[key]))

    if exc.get('stdout', False):
        if exc['stdout'] != "":
            message.append("stdout: %s" % exc['stdout'])
    #END generic message components in exc

    if len(message) == 0:
        for k, v in exc.iteritems():
            message.append("%s: %s" % (k, v))
    return separator.join(message)

if __name__ == '__main__':

    # Start report
    yesterday = datetime.datetime.utcnow().date() - datetime.timedelta(days=1)
    if len(sys.argv) == 2:
        try:
            t = time.strptime(sys.argv[1], "%Y-%m-%d")
            yesterday = datetime.datetime(*t[:6])
        except Exception, e:
            print e
            print "Usage: python error_details.py YYYY-MM-DD (the end date)"
            sys.exit(1)

    hours = 0
    length = 24

    start = datetime.datetime(year=yesterday.year, month=yesterday.month,
                              day=yesterday.day)
    end = start + datetime.timedelta(hours=length-1, minutes=59, seconds=59)

    instance_map = {}  # { uuid : [request_id, request_id, ...] }
    exception_counts = {}  # { exception_message : count }
    event_counts = {}  # { event_name : count }
    metadata = {'report_format': 'json',
                'instances': instance_map,
                'exception_counts': exception_counts,
                'event_counts': event_counts
                }

    # Tell Stacky to format as JSON and set placeholders for various summaries
    report = [metadata]

    dstart = dt.dt_to_decimal(start)
    dend = dt.dt_to_decimal(end)

    codes = {}
    deployments = {}
    for deploy in models.Deployment.objects.all():
        deployments[deploy.id] = deploy.name

    # Get all the instances that have changed in the last N hours ...
    updates = models.RawData.objects.filter(event='compute.instance.update',
                                            when__gt=dstart, when__lte=dend)\
                                    .values('instance').distinct()

    expiry = 60 * 60  # 1 hour
    cmds = ['create', 'rebuild', 'rescue', 'resize', 'snapshot']

    failures = {}
    causes = {}
    durations = {}
    successes = {}
    tenant_issues = {}

    for uuid_dict in updates:
        uuid = uuid_dict['instance']

        # All the unique Request ID's for this instance during that timespan.
        reqs = models.RawData.objects.filter(instance=uuid,
                                             when__gt=dstart, when__lte=dend)\
                                     .values('request_id').distinct()

        req_list = []
{'fraction_non_alphanumeric': 0.13830392714854867, 'fraction_numerical': 0.012521343198634035, 'mean_word_length': 2.1670171222589367, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5596716', 'n_tokens_mistral': 3349, 'n_tokens_neox': 3249, 'n_words': 832}