<commit_before># Copyright 2011 The greplin-exception-catcher Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Queue system for aggregating exceptions.

DESIGN:

Step 1:

A new error instance is reported.  The instance is added to the Queue data store and the instances queue.
(this step is a remnant of an old strategy for dealing with incoming instances and will be removed in the future)

Step 2:

The "instances" queue handler is called.  It puts the LoggedErrorInstance, adds the instance key to the
"aggregation" queue, and adds a new task to the "aggregationWorker" queue.

Step 3:

The aggregationWorker queue handler is called.  It pulls from the "aggregation" queue, getting as many instances as
possible.  It groups them by the Error they are an Instance of.  New stats are computed, a lock is acquired on the
error, and a get/set is performed.  If the lock can't be acquired, the tasks are abandoned and should be picked up by
the next worker.  In this case, the worker throws an exception so that it is rerun.
"""

# pylint: disable=E0611
from google.appengine.api import memcache, taskqueue
# pylint: disable=E0611
from google.appengine.ext import webapp

import backtrace
import collections
from datetime import datetime
import hashlib
try:
  from django.utils import simplejson as json
except ImportError:
  import json
import logging

from common import AttrDict, getProject, parseDate
from datamodel import LoggedError, LoggedErrorInstance, Queue


AGGREGATION_ID = 'currentAggregationId'


def getEndpoints():
  """Returns endpoints needed for queue processing."""
  return [
    ('/reportWorker', ReportWorker),
    ('/aggregationWorker', AggregationWorker)
  ]


def generateHash(exceptionType, backtraceText):
  """Generates a hash for the given exception type and backtrace."""
  hasher = hashlib.md5()
  if exceptionType:
    hasher.update(exceptionType.encode('utf-8'))
  if backtraceText:
    hasher.update(backtrace.normalizeBacktrace(backtraceText.encode('utf-8')))
  return hasher.hexdigest()


def getAggregatedError(project, backtraceText, errorHash):
  """Gets (and updates) the error matching the given report, or None if no matching error is found."""
  error = None

  project = getProject(project)

  q = LoggedError.all().ancestor(project).filter('hash =', errorHash).filter('active =', True)

  for possibility in q:
    if backtrace.normalizeBacktrace(possibility.backtrace) == backtrace.normalizeBacktrace(backtraceText):
      error = possibility
      break

  return error


def aggregate(destination, count, first, last, lastMessage, backtraceText, environments, servers):
  """Aggregates in to the given destination."""
  destination.count += count

  if destination.firstOccurrence:
    destination.firstOccurrence = min(destination.firstOccurrence, first)
  else:
    destination.firstOccurrence = first

  if not destination.lastOccurrence or last > destination.lastOccurrence:
    destination.lastOccurrence = last
    destination.backtrace = backtraceText
    destination.lastMessage = lastMessage

  destination.environments = list(set(destination.environments) | set(environments))
  destination.servers = list(set(destination.servers) | set(servers))


def aggregateSingleInstance(instance, backtraceText):
  """Aggregates a single instance into an "aggregate" object."""
  return {
    'count': 1,
    'firstOccurrence': str(instance.date),
    'lastOccurrence': str(instance.date),
    'lastMessage': instance.message[:300],
    'backtrace': backtraceText,
    'environments': (instance.environment,),
    'servers': (instance.server,),
  }


def aggregateInstances(instances):
  """Aggregates instances in to a meta instance."""
  result = AttrDict(
    count = 0,
    firstOccurrence = None,
    lastOccurrence = None,
    lastMessage = None,
    backtrace = None,
    environments = set(),
    servers = set()
  )

  for instance in instances:
    aggregate(result,
              int(instance['count']),
              parseDate(instance['firstOccurrence']),
              parseDate(instance['lastOccurrence']),
              instance['lastMessage'],
              instance['backtrace'],
              instance['environments'],
              instance['servers'])

  return result


def queueException(serializedException):
  """Enqueues the given exception."""
  task = Queue(payload = serializedException)
  task.put()
  taskqueue.add(queue_name='instances', url='/reportWorker', params={'key': task.key()})


def queueAggregation(error, instance, backtraceText):
  """Enqueues a task to aggregate the given instance in to the given error."""
  payload = {'error': str(error.key()), 'instance': str(instance.key()), 'backtrace': backtraceText}
  taskqueue.Queue('aggregation').add([
    taskqueue.Task(payload = json.dumps(payload), method='PULL')
  ])
  queueAggregationWorker()


def queueAggregationWorker():
  """Enqueues a task to aggregate available instances."""
  workerId = memcache.incr(AGGREGATION_ID, initial_value=0)
  taskqueue.add(queue_name='aggregationWorker',
                url='/aggregationWorker',
                params={'id': workerId})


def _putInstance(exception):
  """Put an exception in the data store."""
  backtraceText = exception['backtrace'] or ''
  environment = exception.get('environment', 'Unknown')
  message = exception['message'] or ''
  project = exception['project']
  server = exception['serverName']
  timestamp = datetime.fromtimestamp(exception['timestamp'])
  exceptionType = exception['type'] or ''
  logMessage = exception.get('logMessage')
  context = exception.get('context')
  errorLevel = exception.get('errorLevel')

  errorHash = generateHash(exceptionType, backtraceText)

  error = getAggregatedError(project, backtraceText, errorHash)

  exceptionType = exceptionType.replace('\n', ' ')
  if len(exceptionType) > 500:
    exceptionType = exceptionType[:500]
  exceptionType = exceptionType.replace('\n', ' ')

  needsAggregation = True
  if not error:
    error = LoggedError(parent = getProject(project))
    error.backtrace = backtraceText
    error.type = exceptionType
    error.hash = errorHash
    error.active = True
    error.errorLevel = errorLevel
    error.count = 1
    error.firstOccurrence = timestamp
    error.lastOccurrence = timestamp
    error.lastMessage = message[:300]
    error.environments = [str(environment)]
    error.servers = [server]
    error.put()
    needsAggregation = False

  instance = LoggedErrorInstance(parent = error)
  instance.environment = environment
  instance.type = exceptionType
  instance.errorLevel = errorLevel<commit_msg>Trust the hash instead of checking backtraces<commit_after># Copyright 2011 The greplin-exception-catcher Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Queue system for aggregating exceptions.

DESIGN:

Step 1:

A new error instance is reported.  The instance is added to the Queue data store and the instances queue.
(this step is a remnant of an old strategy for dealing with incoming instances and will be removed in the future)

Step 2:

The "instances" queue handler is called.  It puts the LoggedErrorInstance, adds the instance key to the
"aggregation" queue, and adds a new task to the "aggregationWorker" queue.

Step 3:

The aggregationWorker queue handler is called.  It pulls from the "aggregation" queue, getting as many instances as
possible.  It groups them by the Error they are an Instance of.  New stats are computed, a lock is acquired on the
error, and a get/set is performed.  If the lock can't be acquired, the tasks are abandoned and should be picked up by
the next worker.  In this case, the worker throws an exception so that it is rerun.
"""

# pylint: disable=E0611
from google.appengine.api import memcache, taskqueue
# pylint: disable=E0611
from google.appengine.ext import webapp

import backtrace
import collections
from datetime import datetime
import hashlib
try:
  from django.utils import simplejson as json
except ImportError:
  import json
import logging

from common import AttrDict, getProject, parseDate
from datamodel import LoggedError, LoggedErrorInstance, Queue


AGGREGATION_ID = 'currentAggregationId'


def getEndpoints():
  """Returns endpoints needed for queue processing."""
  return [
    ('/reportWorker', ReportWorker),
    ('/aggregationWorker', AggregationWorker)
  ]


def generateHash(exceptionType, backtraceText):
  """Generates a hash for the given exception type and backtrace."""
  hasher = hashlib.md5()
  if exceptionType:
    hasher.update(exceptionType.encode('utf-8'))
  if backtraceText:
    hasher.update(backtrace.normalizeBacktrace(backtraceText.encode('utf-8')))
  return hasher.hexdigest()


def getAggregatedError(project, errorHash):
  """Gets (and updates) the error matching the given report, or None if no matching error is found."""
  error = None

  project = getProject(project)

  q = LoggedError.all().ancestor(project).filter('hash =', errorHash).filter('active =', True)

  for possibility in q:
    return possibility

  return error


def aggregate(destination, count, first, last, lastMessage, backtraceText, environments, servers):
  """Aggregates in to the given destination."""
  destination.count += count

  if destination.firstOccurrence:
    destination.firstOccurrence = min(destination.firstOccurrence, first)
  else:
    destination.firstOccurrence = first

  if not destination.lastOccurrence or last > destination.lastOccurrence:
    destination.lastOccurrence = last
    destination.backtrace = backtraceText
    destination.lastMessage = lastMessage

  destination.environments = list(set(destination.environments) | set(environments))
  destination.servers = list(set(destination.servers) | set(servers))


def aggregateSingleInstance(instance, backtraceText):
  """Aggregates a single instance into an "aggregate" object."""
  return {
    'count': 1,
    'firstOccurrence': str(instance.date),
    'lastOccurrence': str(instance.date),
    'lastMessage': instance.message[:300],
    'backtrace': backtraceText,
    'environments': (instance.environment,),
    'servers': (instance.server,),
  }


def aggregateInstances(instances):
  """Aggregates instances in to a meta instance."""
  result = AttrDict(
    count = 0,
    firstOccurrence = None,
    lastOccurrence = None,
    lastMessage = None,
    backtrace = None,
    environments = set(),
    servers = set()
  )

  for instance in instances:
    aggregate(result,
              int(instance['count']),
              parseDate(instance['firstOccurrence']),
              parseDate(instance['lastOccurrence']),
              instance['lastMessage'],
              instance['backtrace'],
              instance['environments'],
              instance['servers'])

  return result


def queueException(serializedException):
  """Enqueues the given exception."""
  task = Queue(payload = serializedException)
  task.put()
  taskqueue.add(queue_name='instances', url='/reportWorker', params={'key': task.key()})


def queueAggregation(error, instance, backtraceText):
  """Enqueues a task to aggregate the given instance in to the given error."""
  payload = {'error': str(error.key()), 'instance': str(instance.key()), 'backtrace': backtraceText}
  taskqueue.Queue('aggregation').add([
    taskqueue.Task(payload = json.dumps(payload), method='PULL')
  ])
  queueAggregationWorker()


def queueAggregationWorker():
  """Enqueues a task to aggregate available instances."""
  workerId = memcache.incr(AGGREGATION_ID, initial_value=0)
  taskqueue.add(queue_name='aggregationWorker',
                url='/aggregationWorker',
                params={'id': workerId})


def _putInstance(exception):
  """Put an exception in the data store."""
  backtraceText = exception['backtrace'] or ''
  environment = exception.get('environment', 'Unknown')
  message = exception['message'] or ''
  project = exception['project']
  server = exception['serverName']
  timestamp = datetime.fromtimestamp(exception['timestamp'])
  exceptionType = exception['type'] or ''
  logMessage = exception.get('logMessage')
  context = exception.get('context')
  errorLevel = exception.get('errorLevel')

  errorHash = generateHash(exceptionType, backtraceText)

  error = getAggregatedError(project, errorHash)

  exceptionType = exceptionType.replace('\n', ' ')
  if len(exceptionType) > 500:
    exceptionType = exceptionType[:500]
  exceptionType = exceptionType.replace('\n', ' ')

  needsAggregation = True
  if not error:
    error = LoggedError(parent = getProject(project))
    error.backtrace = backtraceText
    error.type = exceptionType
    error.hash = errorHash
    error.active = True
    error.errorLevel = errorLevel
    error.count = 1
    error.firstOccurrence = timestamp
    error.lastOccurrence = timestamp
    error.lastMessage = message[:300]
    error.environments = [str(environment)]
    error.servers = [server]
    error.put()
    needsAggregation = False

  instance = LoggedErrorInstance(parent = error)
  instance.environment = environment
  instance.type = exceptionType
  instance.errorLevel = errorLevel
{'fraction_non_alphanumeric': 0.10061062198239136, 'fraction_numerical': 0.005396194262993468, 'mean_word_length': 4.3231292517006805, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '179643', 'n_tokens_mistral': 3906, 'n_tokens_neox': 3814, 'n_words': 1249}