<commit_before>#!/usr/bin/python
try:
  import json
except ImportError:
  import simplejson as json
import itertools
import sys
import heapq
import threading

import socket

from thrift import Thrift  #get Thrift modue
from thrift.transport import TSocket, TTransport #get modules
from thrift.protocol import TBinaryProtocol #get module
from hbase import ttypes #get module
from hbase.hbase import Client, ColumnDescriptor, Mutation #get classes from module

import socorro.lib.util as utl

class HBaseClientException(Exception):
  pass

class BadOoidException(HBaseClientException):
  def __init__(self, wrapped_exception_class, reason=''):
    #super(BadOoidException, self).__init__("Bad OOID: %s-%s" % (str(wrapped_exception_class), str(reason)))
    HBaseClientException.__init__(self, "Bad OOID: %s-%s" % (str(wrapped_exception_class), str(reason)))

class OoidNotFoundException(HBaseClientException):
  def __init__(self, reason=''):
    #super(OoidNotFoundException, self).__init__("OOID not found: %s" % str(reason))
    HBaseClientException.__init__(self, "OOID not found: %s" % str(reason))

#class NotInJsonFormatException(HBaseClientException):
  #def __init__(self, wrapped_exception_class, reason=''):
    ##super(NotInJsonFormatException, self).__init__("Improper JSON format: %s" % str(reason))
    #HBaseClientException.__init__(self, "Improper JSON format: %s" % str(reason))

class FatalException(HBaseClientException):
  def __init__(self, wrapped_exception_class, reason=''):
    #super(FatalException, self).__init__("Improper JSON format: %s" % str(reason))
    HBaseClientException.__init__(self, "the connection is not viable.  retries fail: %s" % str(reason))

class NoConnectionException(FatalException):
  def __init__(self, wrapped_exception_class, reason='', tries=0):
    #super(NoConnectionException, self).__init__("No connection was made to HBase (%d tries): %s-%s" % (tries, str(wrapped_exception_class), str(reason)))
    FatalException.__init__(self, NoConnectionException, "No connection was made to HBase (%d tries): %s-%s" % (tries, str(wrapped_exception_class), str(reason)))

class UnhandledInternalException(HBaseClientException):
  def __init__(self, wrapped_exception_class, reason=''):
    #super(UnhandledInternalException, self).__init__("An internal exception was not handled: %s-%s" % (str(wrapped_exception_class), str(reason)))
    HBaseClientException.__init__(self, "An internal exception was not handled: %s-%s" % (str(wrapped_exception_class), str(reason)))

def exception_wrapper(xClass):
  """This decorator ensures that no exception escapes that isn't from the
  HBaseClientException hierarchy.  Any unexpected exceptions are wrapped in the
  exception class passed into this function.  The original exception is preserved
  as the text of the wrapping expression.  Traceback info of the original
  exception is also preserved as the traceback for the wrapping exception."""
  def wrapper (fn):
    def f(*args, **kwargs):
      try:
        result = fn(*args, **kwargs)
        return result
      except HBaseClientException, x:
        raise
      except Exception, x:
        txClass, tx, txtb = sys.exc_info()
        raise xClass, xClass(txClass,tx), txtb
    f.__name__ = fn.__name__
    return f
  return wrapper

def exception_wrapper_for_generators(xClass):
  """This decorator ensures that no exception escapes that isn't from the
  HBaseClientException hierarchy.  Any unexpected exceptions are wrapped in the
  exception class passed into this function.  The original exception is preserved
  as the text of the wrapping expression.  Traceback info of the original
  exception is also preserved as the traceback for the wrapping exception."""
  def wrapper (fn):
    def f(*args, **kwargs):
      try:
        for x in fn(*args, **kwargs):
          yield x
      except HBaseClientException, x:
        raise
      except Exception, x:
        txClass, tx, txtb = sys.exc_info()
        raise xClass, xClass(txClass,tx), txtb
    f.__name__ = fn.__name__
    return f
  return wrapper

def retry_wrapper_for_generators(fn):
  """a decorator to add retry symantics to any generator that uses hbase.  Don't wrap iterators
  that themselves wrap iterators.  In other words, don't nest these."""
  def f(self, *args, **kwargs):
    self.logger.debug('%s - retry_wrapper_for_generators: trying first time, %s', threading.currentThread().getName(), fn.__name__)
    fail_counter = 0
    while True:  #we have to loop forever, we don't know the length of the wrapped iterator
      try:
        for x in fn(self, *args, **kwargs):
          fail_counter = 0
          yield x
        self.logger.debug('%s - retry_wrapper_for_generators: completed without trouble, %s', threading.currentThread().getName(), fn.__name__)
        break # this is the sucessful exit from this function
      except self.hbaseThriftExceptions, x:
        self.logger.debug('%s - retry_wrapper_for_generators: handled exception, threading.currentThread().getName(), %s', threading.currentThread().getName(), str(x))
        fail_counter += 1
        if fail_counter > 1:
          self.logger.error('%s - retry_wrapper_for_generators: failed too many times on this one operation, %s', threading.currentThread().getName(), fn.__name__)
          txClass, tx, txtb = sys.exc_info()
          raise FatalException, FatalException(tx), txtb
        try:
          self.close()
        except self.hbaseThriftExceptions:
          pass
        self.logger.debug('%s - retry_wrapper_for_generators: about to retry connection', threading.currentThread().getName())
        self.make_connection(timeout=self.timeout)
        self.logger.debug('%s - retry_wrapper_for_generators: about to retry function, %s', threading.currentThread().getName(), fn.__name__)
      except Exception, x:  #lars
        self.logger.debug('%s - retry_wrapper_for_generators: unhandled exception, %s', threading.currentThread().getName(), str(x)) #lars
        raise
  f.__name__ = fn.__name__
  return f

def optional_retry_wrapper(fn):
  """a decorator to add retry symantics to any method that uses hbase"""
  def f(self, *args, **kwargs):
    number_of_retries = kwargs.setdefault('number_of_retries', 0)
    del kwargs['number_of_retries']
    wait_between_retries = kwargs.setdefault('wait_between_retries', 0)
    del kwargs['wait_between_retries']
    countdown = number_of_retries + 1
    while countdown:
      countdown -= 1
      try:
        self.logger.debug('%s - retry_wrapper: %s, try number %s', threading.currentThread().getName(), fn.__name__, number_of_retries + 1 - countdown)
        result = fn(self, *args, **kwargs)
        self.logger.debug('%s - retry_wrapper: completed without trouble, %s', threading.currentThread().getName(), fn.__name__)
        return result
      # drop and remake connection
      except self.hbaseThriftExceptions, x:
        self.logger.debug('%s - retry_wrapper: handled exception, %s', threading.currentThread().getName(), str(x))<commit_msg>changed default number of retries from 0 to 1 in the retry decorator.  Internal calls that were not having retry number explicitly set, were not, of course, being retried.<commit_after>#!/usr/bin/python
try:
  import json
except ImportError:
  import simplejson as json
import itertools
import sys
import heapq
import threading
import time

import socket

from thrift import Thrift  #get Thrift modue
from thrift.transport import TSocket, TTransport #get modules
from thrift.protocol import TBinaryProtocol #get module
from hbase import ttypes #get module
from hbase.hbase import Client, ColumnDescriptor, Mutation #get classes from module

import socorro.lib.util as utl

class HBaseClientException(Exception):
  pass

class BadOoidException(HBaseClientException):
  def __init__(self, wrapped_exception_class, reason=''):
    #super(BadOoidException, self).__init__("Bad OOID: %s-%s" % (str(wrapped_exception_class), str(reason)))
    HBaseClientException.__init__(self, "Bad OOID: %s-%s" % (str(wrapped_exception_class), str(reason)))

class OoidNotFoundException(HBaseClientException):
  def __init__(self, reason=''):
    #super(OoidNotFoundException, self).__init__("OOID not found: %s" % str(reason))
    HBaseClientException.__init__(self, "OOID not found: %s" % str(reason))

#class NotInJsonFormatException(HBaseClientException):
  #def __init__(self, wrapped_exception_class, reason=''):
    ##super(NotInJsonFormatException, self).__init__("Improper JSON format: %s" % str(reason))
    #HBaseClientException.__init__(self, "Improper JSON format: %s" % str(reason))

class FatalException(HBaseClientException):
  def __init__(self, wrapped_exception_class, reason=''):
    #super(FatalException, self).__init__("Improper JSON format: %s" % str(reason))
    HBaseClientException.__init__(self, "the connection is not viable.  retries fail: %s" % str(reason))

class NoConnectionException(FatalException):
  def __init__(self, wrapped_exception_class, reason='', tries=0):
    #super(NoConnectionException, self).__init__("No connection was made to HBase (%d tries): %s-%s" % (tries, str(wrapped_exception_class), str(reason)))
    FatalException.__init__(self, NoConnectionException, "No connection was made to HBase (%d tries): %s-%s" % (tries, str(wrapped_exception_class), str(reason)))

class UnhandledInternalException(HBaseClientException):
  def __init__(self, wrapped_exception_class, reason=''):
    #super(UnhandledInternalException, self).__init__("An internal exception was not handled: %s-%s" % (str(wrapped_exception_class), str(reason)))
    HBaseClientException.__init__(self, "An internal exception was not handled: %s-%s" % (str(wrapped_exception_class), str(reason)))

def exception_wrapper(xClass):
  """This decorator ensures that no exception escapes that isn't from the
  HBaseClientException hierarchy.  Any unexpected exceptions are wrapped in the
  exception class passed into this function.  The original exception is preserved
  as the text of the wrapping expression.  Traceback info of the original
  exception is also preserved as the traceback for the wrapping exception."""
  def wrapper (fn):
    def f(*args, **kwargs):
      try:
        result = fn(*args, **kwargs)
        return result
      except HBaseClientException, x:
        raise
      except Exception, x:
        txClass, tx, txtb = sys.exc_info()
        raise xClass, xClass(txClass,tx), txtb
    f.__name__ = fn.__name__
    return f
  return wrapper

def exception_wrapper_for_generators(xClass):
  """This decorator ensures that no exception escapes that isn't from the
  HBaseClientException hierarchy.  Any unexpected exceptions are wrapped in the
  exception class passed into this function.  The original exception is preserved
  as the text of the wrapping expression.  Traceback info of the original
  exception is also preserved as the traceback for the wrapping exception."""
  def wrapper (fn):
    def f(*args, **kwargs):
      try:
        for x in fn(*args, **kwargs):
          yield x
      except HBaseClientException, x:
        raise
      except Exception, x:
        txClass, tx, txtb = sys.exc_info()
        raise xClass, xClass(txClass,tx), txtb
    f.__name__ = fn.__name__
    return f
  return wrapper

def retry_wrapper_for_generators(fn):
  """a decorator to add retry symantics to any generator that uses hbase.  Don't wrap iterators
  that themselves wrap iterators.  In other words, don't nest these."""
  def f(self, *args, **kwargs):
    self.logger.debug('%s - retry_wrapper_for_generators: trying first time, %s', threading.currentThread().getName(), fn.__name__)
    fail_counter = 0
    while True:  #we have to loop forever, we don't know the length of the wrapped iterator
      try:
        for x in fn(self, *args, **kwargs):
          fail_counter = 0
          yield x
        self.logger.debug('%s - retry_wrapper_for_generators: completed without trouble, %s', threading.currentThread().getName(), fn.__name__)
        break # this is the sucessful exit from this function
      except self.hbaseThriftExceptions, x:
        self.logger.debug('%s - retry_wrapper_for_generators: handled exception, threading.currentThread().getName(), %s', threading.currentThread().getName(), str(x))
        fail_counter += 1
        if fail_counter > 1:
          self.logger.error('%s - retry_wrapper_for_generators: failed too many times on this one operation, %s', threading.currentThread().getName(), fn.__name__)
          txClass, tx, txtb = sys.exc_info()
          raise FatalException, FatalException(tx), txtb
        try:
          self.close()
        except self.hbaseThriftExceptions:
          pass
        self.logger.debug('%s - retry_wrapper_for_generators: about to retry connection', threading.currentThread().getName())
        self.make_connection(timeout=self.timeout)
        self.logger.debug('%s - retry_wrapper_for_generators: about to retry function, %s', threading.currentThread().getName(), fn.__name__)
      except Exception, x:  #lars
        self.logger.debug('%s - retry_wrapper_for_generators: unhandled exception, %s', threading.currentThread().getName(), str(x)) #lars
        raise
  f.__name__ = fn.__name__
  return f

def optional_retry_wrapper(fn):
  """a decorator to add retry symantics to any method that uses hbase"""
  def f(self, *args, **kwargs):
    number_of_retries = kwargs.setdefault('number_of_retries', 1)
    del kwargs['number_of_retries']
    wait_between_retries = kwargs.setdefault('wait_between_retries', 1)
    del kwargs['wait_between_retries']
    countdown = number_of_retries + 1
    while countdown:
      countdown -= 1
      try:
        self.logger.debug('%s - retry_wrapper: %s, try number %s', threading.currentThread().getName(), fn.__name__, number_of_retries + 1 - countdown)
        result = fn(self, *args, **kwargs)
        self.logger.debug('%s - retry_wrapper: completed without trouble, %s', threading.currentThread().getName(), fn.__name__)
        return result
      # drop and remake connection
      except self.hbaseThriftExceptions, x:
        self.logger.debug('%s - retry_wrapper: handled exception, %s', threading.currentThread().getName(), str(x))
{'fraction_non_alphanumeric': 0.1045774647887324, 'fraction_numerical': 0.0015492957746478873, 'mean_word_length': 4.2363569321533925, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3282015', 'n_tokens_mistral': 4114, 'n_tokens_neox': 4057, 'n_words': 1257}