<commit_before>"""
sentry.tasks.cleanup
~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2012 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from celery.task import task


@task(ignore_result=True)
def cleanup(days=30, project=None, **kwargs):
    """
    Deletes a portion of the trailing data in Sentry based on
    their creation dates. For example, if ``days`` is 30, this
    would attempt to clean up all data thats older than 30 days.

    :param project: limit all deletion scopes to messages that are part
                    of the given project
    """
    import datetime

    from django.utils import timezone

    from sentry.models import (Group, Event, MessageCountByMinute,
        MessageFilterValue, FilterKey, FilterValue, ProjectCountByMinute,
        SearchDocument)
    from sentry.utils.query import RangeQuerySetWrapper

    GENERIC_DELETES = (
        (SearchDocument, 'date_changed'),
        (MessageCountByMinute, 'date'),
        (ProjectCountByMinute, 'date'),
        (Event, 'datetime'),
        (Group, 'last_seen'),
        (MessageFilterValue, 'last_seen'),
    )

    log = cleanup.get_logger()

    ts = timezone.now() - datetime.timedelta(days=days)

    # Remove types which can easily be bound to project + date
    for model, date_col in GENERIC_DELETES:
        log.info("Removing %r for days=%s project=%r" % (model, days, project))
        qs = model.objects.filter(**{'%s__lte' % (date_col,): ts})
        if project:
            qs = qs.filter(project=project)
        qs.delete()

    # We'll need this to confirm deletion of FilterKey and Filtervalue objects.
    mqs = MessageFilterValue.objects.all()
    if project:
        mqs = mqs.filter(project=project)

    # FilterKey
    log.info("Removing %r for days=%s project=%r" % (FilterKey, days, project))
    qs = FilterKey.objects.all()
    if project:
        qs = qs.filter(project=project)
    for obj in RangeQuerySetWrapper(qs):
        if not mqs.filter(key=obj.key).exists():
            log.info("Removing unused filter %s=*", obj.key,)
            qs.filter(key=obj.key).delete()
            obj.delete()

    # FilterValue
    log.info("Removing %r for days=%s project=%r" % (FilterValue, days, project))
    qs = FilterValue.objects.all()<commit_msg>Delete one object at a time<commit_after>"""
sentry.tasks.cleanup
~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2012 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from celery.task import task


@task(ignore_result=True)
def cleanup(days=30, project=None, **kwargs):
    """
    Deletes a portion of the trailing data in Sentry based on
    their creation dates. For example, if ``days`` is 30, this
    would attempt to clean up all data thats older than 30 days.

    :param project: limit all deletion scopes to messages that are part
                    of the given project
    """
    import datetime

    from django.utils import timezone

    from sentry.models import (Group, Event, MessageCountByMinute,
        MessageFilterValue, FilterKey, FilterValue, ProjectCountByMinute,
        SearchDocument)
    from sentry.utils.query import RangeQuerySetWrapper

    GENERIC_DELETES = (
        (SearchDocument, 'date_changed'),
        (MessageCountByMinute, 'date'),
        (ProjectCountByMinute, 'date'),
        (Event, 'datetime'),
        (Group, 'last_seen'),
        (MessageFilterValue, 'last_seen'),
    )

    log = cleanup.get_logger()

    ts = timezone.now() - datetime.timedelta(days=days)

    # Remove types which can easily be bound to project + date
    for model, date_col in GENERIC_DELETES:
        log.info("Removing %r for days=%s project=%r" % (model, days, project))
        qs = model.objects.filter(**{'%s__lte' % (date_col,): ts})
        if project:
            qs = qs.filter(project=project)
        # XXX: we step through because the deletion collector will pull all relations into memory
        for obj in RangeQuerySetWrapper(qs):
            obj.delete()

    # We'll need this to confirm deletion of FilterKey and Filtervalue objects.
    mqs = MessageFilterValue.objects.all()
    if project:
        mqs = mqs.filter(project=project)

    # FilterKey
    log.info("Removing %r for days=%s project=%r" % (FilterKey, days, project))
    qs = FilterKey.objects.all()
    if project:
        qs = qs.filter(project=project)
    for obj in RangeQuerySetWrapper(qs):
        if not mqs.filter(key=obj.key).exists():
            log.info("Removing unused filter %s=*", obj.key,)
            qs.filter(key=obj.key).delete()
            obj.delete()

    # FilterValue
    log.info("Removing %r for days=%s project=%r" % (FilterValue, days, project))
    qs = FilterValue.objects.all()
{'fraction_non_alphanumeric': 0.11726588628762542, 'fraction_numerical': 0.005852842809364548, 'mean_word_length': 2.957816377171216, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6315948', 'n_tokens_mistral': 1459, 'n_tokens_neox': 1377, 'n_words': 434}