<commit_before>"""
sentry.tasks.fetch_source
~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2012 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

import urllib2
from collections import namedtuple
from urlparse import urljoin

from celery.task import task
from sentry.utils.cache import cache
from sentry.utils.lrucache import lrucache
from sentry.utils.sourcemaps import sourcemap_to_index, find_source

BAD_SOURCE = -1

# number of surrounding lines (on each side) to fetch
LINES_OF_CONTEXT = 5


UrlResult = namedtuple('UrlResult', ['url', 'headers', 'body'])


def trim_line(line):
    line = line.strip('\n')
    if len(line) > 150:
        line = line[:140] + ' [... truncated]'
    return line


def get_source_context(source, lineno, context=LINES_OF_CONTEXT):
    # lineno's in JS are 1-indexed
    lineno -= 1

    lower_bound = max(0, lineno - context)
    upper_bound = min(lineno + 1 + context, len(source))

    try:
        pre_context = map(trim_line, source[lower_bound:lineno])
    except IndexError:
        pre_context = []

    try:
        context_line = trim_line(source[lineno])
    except IndexError:
        context_line = ''

    try:
        post_context = map(trim_line, source[(lineno + 1):upper_bound])
    except IndexError:
        post_context = []

    return pre_context, context_line, post_context


def discover_sourcemap(result, logger=None):
    """
    Given a UrlResult object, attempt to discover a sourcemap.
    """
    sourcemap = result.headers.get('X-SourceMap', None)

    if not sourcemap:
        parsed_body = result.body.splitlines()
        indicator = parsed_body[-1]
        if indicator.startswith('//@'):
            try:
                parsed = dict(v.split('=') for v in indicator[3:].strip().split(' '))
            except Exception:
                if logger:
                    logger.error('Failed parsing source map line for %r (line was %r)', result.url, indicator, exc_info=True)
            else:
                sourcemap = parsed.get('sourceMappingURL')

    if sourcemap:
        # fix url so its absolute
        sourcemap = urljoin(result.url, sourcemap)

    return sourcemap


@lrucache.memoize
def fetch_url(url, logger=None):
    """
    Pull down a URL, returning a UrlResult object.

    Attempts to fetch from the cache.
    """
    import sentry

    cache_key = 'fetch_url:%s' % url
    result = cache.get(cache_key)
    if result is not None:
        return result

    try:
        opener = urllib2.build_opener()
        opener.addheaders = [('User-Agent', 'Sentry/%s' % sentry.VERSION)]
        req = opener.open(url)
        headers = dict(req.headers)
        body = req.read().rstrip('\n')
    except Exception:
        if logger:
            logger.error('Unable to fetch remote source for %r', url, exc_info=True)
        return BAD_SOURCE

    result = UrlResult(url, headers, body)

    cache.set(cache_key, result, 60 * 5)

    return result


@task(ignore_result=True)
def fetch_javascript_source(event, **kwargs):
    """
    Attempt to fetch source code for javascript frames.

    Frames must match the following requirements:

    - lineno >= 0
    - colno >= 0
    - abs_path is the HTTP URI to the source
    - context_line is empty
    """
    import logging
    logger = fetch_javascript_source.get_logger()
    logger.setLevel(logging.INFO)
    logger.addHandler(logging.StreamHandler())

    try:<commit_msg>Remove epxlicit logging changes<commit_after>"""
sentry.tasks.fetch_source
~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2012 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

import urllib2
from collections import namedtuple
from urlparse import urljoin

from celery.task import task
from sentry.utils.cache import cache
from sentry.utils.lrucache import lrucache
from sentry.utils.sourcemaps import sourcemap_to_index, find_source

BAD_SOURCE = -1

# number of surrounding lines (on each side) to fetch
LINES_OF_CONTEXT = 5


UrlResult = namedtuple('UrlResult', ['url', 'headers', 'body'])


def trim_line(line):
    line = line.strip('\n')
    if len(line) > 150:
        line = line[:140] + ' [... truncated]'
    return line


def get_source_context(source, lineno, context=LINES_OF_CONTEXT):
    # lineno's in JS are 1-indexed
    lineno -= 1

    lower_bound = max(0, lineno - context)
    upper_bound = min(lineno + 1 + context, len(source))

    try:
        pre_context = map(trim_line, source[lower_bound:lineno])
    except IndexError:
        pre_context = []

    try:
        context_line = trim_line(source[lineno])
    except IndexError:
        context_line = ''

    try:
        post_context = map(trim_line, source[(lineno + 1):upper_bound])
    except IndexError:
        post_context = []

    return pre_context, context_line, post_context


def discover_sourcemap(result, logger=None):
    """
    Given a UrlResult object, attempt to discover a sourcemap.
    """
    sourcemap = result.headers.get('X-SourceMap', None)

    if not sourcemap:
        parsed_body = result.body.splitlines()
        indicator = parsed_body[-1]
        if indicator.startswith('//@'):
            try:
                parsed = dict(v.split('=') for v in indicator[3:].strip().split(' '))
            except Exception:
                if logger:
                    logger.error('Failed parsing source map line for %r (line was %r)', result.url, indicator, exc_info=True)
            else:
                sourcemap = parsed.get('sourceMappingURL')

    if sourcemap:
        # fix url so its absolute
        sourcemap = urljoin(result.url, sourcemap)

    return sourcemap


@lrucache.memoize
def fetch_url(url, logger=None):
    """
    Pull down a URL, returning a UrlResult object.

    Attempts to fetch from the cache.
    """
    import sentry

    cache_key = 'fetch_url:%s' % url
    result = cache.get(cache_key)
    if result is not None:
        return result

    try:
        opener = urllib2.build_opener()
        opener.addheaders = [('User-Agent', 'Sentry/%s' % sentry.VERSION)]
        req = opener.open(url)
        headers = dict(req.headers)
        body = req.read().rstrip('\n')
    except Exception:
        if logger:
            logger.error('Unable to fetch remote source for %r', url, exc_info=True)
        return BAD_SOURCE

    result = UrlResult(url, headers, body)

    cache.set(cache_key, result, 60 * 5)

    return result


@task(ignore_result=True)
def fetch_javascript_source(event, **kwargs):
    """
    Attempt to fetch source code for javascript frames.

    Frames must match the following requirements:

    - lineno >= 0
    - colno >= 0
    - abs_path is the HTTP URI to the source
    - context_line is empty
    """
    logger = fetch_javascript_source.get_logger()

    try:
{'fraction_non_alphanumeric': 0.11153284671532847, 'fraction_numerical': 0.008759124087591242, 'mean_word_length': 2.7642857142857142, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2342047', 'n_tokens_mistral': 2164, 'n_tokens_neox': 2049, 'n_words': 590}