<commit_before>from __future__ import absolute_import

import re
import six

from symsynd import demangle_symbol, SymbolicationError, get_cpu_name, \
    ImageLookup, Symbolizer as SymsyndSymbolizer

from sentry.lang.native.dsymcache import dsymcache
from sentry.utils.safe import trim
from sentry.utils.compat import implements_to_string
from sentry.models import EventError
from sentry.constants import MAX_SYM, NATIVE_UNKNOWN_STRING


FATAL_ERRORS = (
    EventError.NATIVE_MISSING_DSYM,
    EventError.NATIVE_BAD_DSYM,
)
USER_FIXABLE_ERRORS = (
    EventError.NATIVE_MISSING_DSYM,
    EventError.NATIVE_MISSING_OPTIONALLY_BUNDLED_DSYM,
    EventError.NATIVE_BAD_DSYM,
    EventError.NATIVE_MISSING_SYMBOL,
)
APP_BUNDLE_PATHS = (
    '/var/containers/Bundle/Application/',
    '/private/var/containers/Bundle/Application/',
)
_sim_platform_re = re.compile(r'/\w+?Simulator\.platform/')
_support_framework = re.compile(r'''(?x)
    /Frameworks/(
            libswift([a-zA-Z0-9]+)\.dylib$
        |   (KSCrash|SentrySwift|Sentry)\.framework/
    )
''')
SIM_PATH = '/Developer/CoreSimulator/Devices/'
SIM_APP_PATH = '/Containers/Bundle/Application/'

KNOWN_GARBAGE_SYMBOLS = set([
    '_mh_execute_header',
    '<redacted>',
    NATIVE_UNKNOWN_STRING,
])


@implements_to_string
class SymbolicationFailed(Exception):
    message = None

    def __init__(self, message=None, type=None, image=None):
        Exception.__init__(self)
        self.message = six.text_type(message)
        self.type = type
        if image is not None:
            self.image_uuid = image['uuid'].lower()
            self.image_path = image['name']
            self.image_name = image['name'].rsplit('/', 1)[-1]
            self.image_arch = get_cpu_name(image['cpu_type'],
                                           image['cpu_subtype'])
        else:
            self.image_uuid = None
            self.image_name = None
            self.image_path = None
            self.image_arch = None

    @property
    def is_user_fixable(self):
        """These are errors that a user can fix themselves."""
        return self.type in USER_FIXABLE_ERRORS

    @property
    def is_fatal(self):
        """If this is true then a processing issues has to be reported."""
        return self.type in FATAL_ERRORS

    @property
    def is_sdk_failure(self):
        """An error that most likely happened because of a bad SDK."""
        return self.type == EventError.NATIVE_UNKNOWN_IMAGE

    def __str__(self):
        rv = []
        if self.type is not None:
            rv.append(u'%s: ' % self.type)
        rv.append(self.message or 'no information available')
        if self.image_uuid is not None:
            rv.append(' image-uuid=%s' % self.image_uuid)
        if self.image_name is not None:
            rv.append(' image-name=%s' % self.image_name)
        return u''.join(rv)


class Symbolizer(object):
    """This symbolizer dispatches to both symsynd and the system symbols
    we have in the database and reports errors slightly differently.
    """

    def __init__(self, project, binary_images, referenced_images=None,
                 cpu_name=None, on_dsym_file_referenced=None):
        if isinstance(binary_images, ImageLookup):
            self.image_lookup = binary_images
        else:
            self.image_lookup = ImageLookup(binary_images)

        self._symbolizer = SymsyndSymbolizer()

        to_load = referenced_images
        if to_load is None:
            to_load = self.image_lookup.get_uuids()

        self.dsym_paths = dsymcache.fetch_dsyms(
            project, to_load, on_dsym_file_referenced=on_dsym_file_referenced)

        self.cpu_name = cpu_name

    def close(self):
        self._symbolizer.close()

    def _process_frame(self, frame, img):
        symbol = trim(frame['symbol'], MAX_SYM)
        function = trim(demangle_symbol(frame['symbol'], simplified=True),
                        MAX_SYM)

        frame['function'] = function
        if function != symbol:
            frame['symbol'] = symbol
        else:
            frame['symbol'] = None

        frame['filename'] = trim(frame.get('filename'), 256)
        frame['abs_path'] = trim(frame.get('abs_path'), 256)

        return frame

    def is_image_from_app_bundle(self, img):
        fn = img['name']
        if not (fn.startswith(APP_BUNDLE_PATHS) or
                (SIM_PATH in fn and SIM_APP_PATH in fn)):
            return False
        return True

    def _is_support_framework(self, img):
        """True if the frame is from a framework that is known and app
        bundled.  Those are frameworks which are specifically not frameworks
        that are ever in_app.
        """
        return _support_framework.search(img['name']) is not None

    def _is_app_bundled_framework(self, img):
        fn = img['name']
        return fn.startswith(APP_BUNDLE_PATHS) and '/Frameworks/' in fn

    def _is_app_frame(self, instruction_addr, img):
        """Given a frame derives the value of `in_app` by discarding the
        original value of the frame.
        """
        if not self.is_image_from_app_bundle(img):
            return False
        return not self._is_app_bundled_framework(img)

    def _is_optional_dsym(self, img):
        """Checks if this is a dsym that is optional."""
        # Frames that are not in the app are not considered optional.  In
        # theory we should never reach this anyways.
        if not self.is_image_from_app_bundle(img):
            return False

        # If we're dealing with an app bundled framework that is also
        # considered optional.
        if self._is_app_bundled_framework(img):
            return True

        # Frameworks that are known to sentry and bundled helpers are always
        # optional for now.  In theory this should always be False here
        # because we should catch it with the last branch already.
        if self._is_support_framework(img):
            return True

        return False

    def _is_simulator_frame(self, frame, img):
        return _sim_platform_re.search(img['name']) is not None

    def _symbolize_app_frame(self, instruction_addr, img):
        dsym_path = self.dsym_paths.get(img['uuid'])
        if dsym_path is None:
            if self._is_optional_dsym(img):
                type = EventError.NATIVE_MISSING_OPTIONALLY_BUNDLED_DSYM
            else:
                type = EventError.NATIVE_MISSING_DSYM
            raise SymbolicationFailed(type=type, image=img)

        try:
            rv = self._symbolizer.symbolize(
                dsym_path, img['image_vmaddr'], img['image_addr'],
                instruction_addr, self.cpu_name, symbolize_inlined=True)
        except SymbolicationError as e:
            raise
            raise SymbolicationFailed(
                type=EventError.NATIVE_BAD_DSYM,
                message=six.text_type(e),
                image=img
            )

        if not rv:
            raise SymbolicationFailed(
                type=EventError.NATIVE_MISSING_SYMBOL,
                image=img
            )
        return [self._process_frame(nf, img) for nf in reversed(rv)]

    def _convert_symbolserver_match(self, symbolserver_match, img):
        """Symbolizes a frame with system symbols only."""
        if symbolserver_match is None:
            return []

        symbol = symbolserver_match['symbol']
        if symbol[:1] == '_':
            symbol = symbol[1:]

        return [self._process_frame(dict(
            symbol=symbol, filename=None, abs_path=None, lineno=0,
            colno=0, package=symbolserver_match['object_name']), img)]<commit_msg>Removed a debug reraise that should not be there<commit_after>from __future__ import absolute_import

import re
import six

from symsynd import demangle_symbol, SymbolicationError, get_cpu_name, \
    ImageLookup, Symbolizer as SymsyndSymbolizer

from sentry.lang.native.dsymcache import dsymcache
from sentry.utils.safe import trim
from sentry.utils.compat import implements_to_string
from sentry.models import EventError
from sentry.constants import MAX_SYM, NATIVE_UNKNOWN_STRING


FATAL_ERRORS = (
    EventError.NATIVE_MISSING_DSYM,
    EventError.NATIVE_BAD_DSYM,
)
USER_FIXABLE_ERRORS = (
    EventError.NATIVE_MISSING_DSYM,
    EventError.NATIVE_MISSING_OPTIONALLY_BUNDLED_DSYM,
    EventError.NATIVE_BAD_DSYM,
    EventError.NATIVE_MISSING_SYMBOL,
)
APP_BUNDLE_PATHS = (
    '/var/containers/Bundle/Application/',
    '/private/var/containers/Bundle/Application/',
)
_sim_platform_re = re.compile(r'/\w+?Simulator\.platform/')
_support_framework = re.compile(r'''(?x)
    /Frameworks/(
            libswift([a-zA-Z0-9]+)\.dylib$
        |   (KSCrash|SentrySwift|Sentry)\.framework/
    )
''')
SIM_PATH = '/Developer/CoreSimulator/Devices/'
SIM_APP_PATH = '/Containers/Bundle/Application/'

KNOWN_GARBAGE_SYMBOLS = set([
    '_mh_execute_header',
    '<redacted>',
    NATIVE_UNKNOWN_STRING,
])


@implements_to_string
class SymbolicationFailed(Exception):
    message = None

    def __init__(self, message=None, type=None, image=None):
        Exception.__init__(self)
        self.message = six.text_type(message)
        self.type = type
        if image is not None:
            self.image_uuid = image['uuid'].lower()
            self.image_path = image['name']
            self.image_name = image['name'].rsplit('/', 1)[-1]
            self.image_arch = get_cpu_name(image['cpu_type'],
                                           image['cpu_subtype'])
        else:
            self.image_uuid = None
            self.image_name = None
            self.image_path = None
            self.image_arch = None

    @property
    def is_user_fixable(self):
        """These are errors that a user can fix themselves."""
        return self.type in USER_FIXABLE_ERRORS

    @property
    def is_fatal(self):
        """If this is true then a processing issues has to be reported."""
        return self.type in FATAL_ERRORS

    @property
    def is_sdk_failure(self):
        """An error that most likely happened because of a bad SDK."""
        return self.type == EventError.NATIVE_UNKNOWN_IMAGE

    def __str__(self):
        rv = []
        if self.type is not None:
            rv.append(u'%s: ' % self.type)
        rv.append(self.message or 'no information available')
        if self.image_uuid is not None:
            rv.append(' image-uuid=%s' % self.image_uuid)
        if self.image_name is not None:
            rv.append(' image-name=%s' % self.image_name)
        return u''.join(rv)


class Symbolizer(object):
    """This symbolizer dispatches to both symsynd and the system symbols
    we have in the database and reports errors slightly differently.
    """

    def __init__(self, project, binary_images, referenced_images=None,
                 cpu_name=None, on_dsym_file_referenced=None):
        if isinstance(binary_images, ImageLookup):
            self.image_lookup = binary_images
        else:
            self.image_lookup = ImageLookup(binary_images)

        self._symbolizer = SymsyndSymbolizer()

        to_load = referenced_images
        if to_load is None:
            to_load = self.image_lookup.get_uuids()

        self.dsym_paths = dsymcache.fetch_dsyms(
            project, to_load, on_dsym_file_referenced=on_dsym_file_referenced)

        self.cpu_name = cpu_name

    def close(self):
        self._symbolizer.close()

    def _process_frame(self, frame, img):
        symbol = trim(frame['symbol'], MAX_SYM)
        function = trim(demangle_symbol(frame['symbol'], simplified=True),
                        MAX_SYM)

        frame['function'] = function
        if function != symbol:
            frame['symbol'] = symbol
        else:
            frame['symbol'] = None

        frame['filename'] = trim(frame.get('filename'), 256)
        frame['abs_path'] = trim(frame.get('abs_path'), 256)

        return frame

    def is_image_from_app_bundle(self, img):
        fn = img['name']
        if not (fn.startswith(APP_BUNDLE_PATHS) or
                (SIM_PATH in fn and SIM_APP_PATH in fn)):
            return False
        return True

    def _is_support_framework(self, img):
        """True if the frame is from a framework that is known and app
        bundled.  Those are frameworks which are specifically not frameworks
        that are ever in_app.
        """
        return _support_framework.search(img['name']) is not None

    def _is_app_bundled_framework(self, img):
        fn = img['name']
        return fn.startswith(APP_BUNDLE_PATHS) and '/Frameworks/' in fn

    def _is_app_frame(self, instruction_addr, img):
        """Given a frame derives the value of `in_app` by discarding the
        original value of the frame.
        """
        if not self.is_image_from_app_bundle(img):
            return False
        return not self._is_app_bundled_framework(img)

    def _is_optional_dsym(self, img):
        """Checks if this is a dsym that is optional."""
        # Frames that are not in the app are not considered optional.  In
        # theory we should never reach this anyways.
        if not self.is_image_from_app_bundle(img):
            return False

        # If we're dealing with an app bundled framework that is also
        # considered optional.
        if self._is_app_bundled_framework(img):
            return True

        # Frameworks that are known to sentry and bundled helpers are always
        # optional for now.  In theory this should always be False here
        # because we should catch it with the last branch already.
        if self._is_support_framework(img):
            return True

        return False

    def _is_simulator_frame(self, frame, img):
        return _sim_platform_re.search(img['name']) is not None

    def _symbolize_app_frame(self, instruction_addr, img):
        dsym_path = self.dsym_paths.get(img['uuid'])
        if dsym_path is None:
            if self._is_optional_dsym(img):
                type = EventError.NATIVE_MISSING_OPTIONALLY_BUNDLED_DSYM
            else:
                type = EventError.NATIVE_MISSING_DSYM
            raise SymbolicationFailed(type=type, image=img)

        try:
            rv = self._symbolizer.symbolize(
                dsym_path, img['image_vmaddr'], img['image_addr'],
                instruction_addr, self.cpu_name, symbolize_inlined=True)
        except SymbolicationError as e:
            raise SymbolicationFailed(
                type=EventError.NATIVE_BAD_DSYM,
                message=six.text_type(e),
                image=img
            )

        if not rv:
            raise SymbolicationFailed(
                type=EventError.NATIVE_MISSING_SYMBOL,
                image=img
            )
        return [self._process_frame(nf, img) for nf in reversed(rv)]

    def _convert_symbolserver_match(self, symbolserver_match, img):
        """Symbolizes a frame with system symbols only."""
        if symbolserver_match is None:
            return []

        symbol = symbolserver_match['symbol']
        if symbol[:1] == '_':
            symbol = symbol[1:]

        return [self._process_frame(dict(
            symbol=symbol, filename=None, abs_path=None, lineno=0,
            colno=0, package=symbolserver_match['object_name']), img)]
{'fraction_non_alphanumeric': 0.0897125270119835, 'fraction_numerical': 0.0018335406980551372, 'mean_word_length': 2.4489611562782296, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7374579', 'n_tokens_mistral': 4623, 'n_tokens_neox': 4455, 'n_words': 1112}