<commit_before>import collections
import json
import logging
import os
import string
import sys
import threading
import traceback
from datetime import datetime, timezone
from enum import IntEnum
import random

import requests
from flask import request, Response

from google.api.monitored_resource_pb2 import MonitoredResource
from google.cloud import logging as gcp_logging
from google.cloud import logging_v2 as gcp_logging_v2
from google.logging.type import http_request_pb2 as gcp_http_request_pb2
from google.protobuf import json_format as gcp_json_format, any_pb2 as gcp_any_pb2

from werkzeug.exceptions import HTTPException

# Do not remove this import.
from rdr_service.services import gcp_request_log_pb2  # pylint: disable=unused-import
from rdr_service.config import GAE_PROJECT

# https://pypi.org/project/google-cloud-logging/
# https://cloud.google.com/logging/docs/reference/v2/rpc/google.logging.v2
# https://developers.google.com/resources/api-libraries/documentation/logging/v2/python/latest/logging_v2.entries.html

# How many log lines should be batched before pushing them to StackDriver.
_LOG_BUFFER_SIZE = 24

GAE_LOGGING_MODULE_ID = 'app-' + os.environ.get('GAE_SERVICE', 'default')
GAE_LOGGING_VERSION_ID = os.environ.get('GAE_VERSION', 'devel')

# This is where we save all data that is tied to a specific execution thread.
_thread_store = threading.local()

class LogCompletionStatusEnum(IntEnum):
    """
    Indicator for log entry completion status, which can span multiple log entries.
    """
    COMPLETE = 0
    PARTIAL_BEGIN = 1
    PARTIAL_MORE = 2
    PARTIAL_FINISHED = 3


def setup_logging_zone():
    """
    Attempt to get the project zone information.
    return: Zone pb2 structure.
    """
    zone = 'local-machine'
    if 'GAE_SERVICE' in os.environ:
        try:
            resp = requests.get('http://metadata.google.internal/computeMetadata/v1/instance/zone', timeout=15.0)
            if resp.status_code == 200:
                zone = resp.text.strip()
        # pylint: disable=broad-except
        except Exception:
            zone = 'unknown'
    return zone

# Safe to use for all threads.
logging_zone_pb2 = setup_logging_zone()


def setup_logging_resource():
    """
    Set the values for the Google Logging Resource object. Thread safe.
    :return: MonitoredResource pb2 structure.
    """
    labels = {
        "project_id": GAE_PROJECT,
        "module_id": GAE_LOGGING_MODULE_ID,
        "version_id": GAE_LOGGING_VERSION_ID,
        "zone": logging_zone_pb2
    }

    # https://cloud.google.com/logging/docs/reference/v2/rpc/google.api#google.api.MonitoredResource
    resource_pb2 = MonitoredResource(type='gae_app', labels=labels)
    return resource_pb2

# Safe to use for all threads.
logging_resource_pb2 = setup_logging_resource()


# pylint: disable=unused-argument
def setup_log_line(record: logging.LogRecord, resource=None, method=None):
    """
    Prepare a log event for sending to GCP StackDriver.  Thread safe.
    :param record: Log event record.
    :param resource: request resource
    :param method: request method
    :return: LogLine proto buffer object
    """
    event_ts = datetime.utcfromtimestamp(record.created)
    event_ts = event_ts.replace(tzinfo=timezone.utc)
    event_ts = event_ts.isoformat()
    severity = gcp_logging._helpers._normalize_severity(record.levelno)
    message = record.msg if record.msg else ''

    # Look for embedded traceback source location override information
    if '%%' in message:
        tmp_sl = message[message.find('%%'):message.rfind('%%')+2]
        message = message.replace(tmp_sl, '')
        tmp_sl = tmp_sl.replace('%%', '')
        source_location = json.loads(tmp_sl)

    else:
        funcname = record.funcName if record.funcName else ''
        file = record.pathname if record.pathname else ''
        lineno = record.lineno if record.lineno else 0
        source_location = {
            "file": file,<commit_msg>Handle dict logging messages (#1691)<commit_after>import collections
import json
import logging
import os
import string
import sys
import threading
import traceback
from datetime import datetime, timezone
from enum import IntEnum
import random

import requests
from flask import request, Response

from google.api.monitored_resource_pb2 import MonitoredResource
from google.cloud import logging as gcp_logging
from google.cloud import logging_v2 as gcp_logging_v2
from google.logging.type import http_request_pb2 as gcp_http_request_pb2
from google.protobuf import json_format as gcp_json_format, any_pb2 as gcp_any_pb2

from werkzeug.exceptions import HTTPException

# Do not remove this import.
from rdr_service.services import gcp_request_log_pb2  # pylint: disable=unused-import
from rdr_service.config import GAE_PROJECT

# https://pypi.org/project/google-cloud-logging/
# https://cloud.google.com/logging/docs/reference/v2/rpc/google.logging.v2
# https://developers.google.com/resources/api-libraries/documentation/logging/v2/python/latest/logging_v2.entries.html

# How many log lines should be batched before pushing them to StackDriver.
_LOG_BUFFER_SIZE = 24

GAE_LOGGING_MODULE_ID = 'app-' + os.environ.get('GAE_SERVICE', 'default')
GAE_LOGGING_VERSION_ID = os.environ.get('GAE_VERSION', 'devel')

# This is where we save all data that is tied to a specific execution thread.
_thread_store = threading.local()

class LogCompletionStatusEnum(IntEnum):
    """
    Indicator for log entry completion status, which can span multiple log entries.
    """
    COMPLETE = 0
    PARTIAL_BEGIN = 1
    PARTIAL_MORE = 2
    PARTIAL_FINISHED = 3


def setup_logging_zone():
    """
    Attempt to get the project zone information.
    return: Zone pb2 structure.
    """
    zone = 'local-machine'
    if 'GAE_SERVICE' in os.environ:
        try:
            resp = requests.get('http://metadata.google.internal/computeMetadata/v1/instance/zone', timeout=15.0)
            if resp.status_code == 200:
                zone = resp.text.strip()
        # pylint: disable=broad-except
        except Exception:
            zone = 'unknown'
    return zone

# Safe to use for all threads.
logging_zone_pb2 = setup_logging_zone()


def setup_logging_resource():
    """
    Set the values for the Google Logging Resource object. Thread safe.
    :return: MonitoredResource pb2 structure.
    """
    labels = {
        "project_id": GAE_PROJECT,
        "module_id": GAE_LOGGING_MODULE_ID,
        "version_id": GAE_LOGGING_VERSION_ID,
        "zone": logging_zone_pb2
    }

    # https://cloud.google.com/logging/docs/reference/v2/rpc/google.api#google.api.MonitoredResource
    resource_pb2 = MonitoredResource(type='gae_app', labels=labels)
    return resource_pb2

# Safe to use for all threads.
logging_resource_pb2 = setup_logging_resource()


# pylint: disable=unused-argument
def setup_log_line(record: logging.LogRecord, resource=None, method=None):
    """
    Prepare a log event for sending to GCP StackDriver.  Thread safe.
    :param record: Log event record.
    :param resource: request resource
    :param method: request method
    :return: LogLine proto buffer object
    """
    event_ts = datetime.utcfromtimestamp(record.created)
    event_ts = event_ts.replace(tzinfo=timezone.utc)
    event_ts = event_ts.isoformat()
    severity = gcp_logging._helpers._normalize_severity(record.levelno)
    message = record.msg if record.msg else ''
    if isinstance(message, dict):
        message = json.dumps(message)

    # Look for embedded traceback source location override information
    if '%%' in message:
        tmp_sl = message[message.find('%%'):message.rfind('%%')+2]
        message = message.replace(tmp_sl, '')
        tmp_sl = tmp_sl.replace('%%', '')
        source_location = json.loads(tmp_sl)

    else:
        funcname = record.funcName if record.funcName else ''
        file = record.pathname if record.pathname else ''
        lineno = record.lineno if record.lineno else 0
        source_location = {
            "file": file,
{'fraction_non_alphanumeric': 0.09004857391954166, 'fraction_numerical': 0.009216589861751152, 'mean_word_length': 3.6309111880046134, 'pattern_counts': {'":': 10, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 8, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7040668', 'n_tokens_mistral': 2475, 'n_tokens_neox': 2414, 'n_words': 646}