<commit_before>"""
sentry.management.commands.cleanup
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2012 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, print_function

from datetime import timedelta
from django.core.management.base import BaseCommand
from django.db import connections
from django.utils import timezone
from optparse import make_option

from sentry.app import nodestore
from sentry.models import (
    Activity, Event, EventMapping, Group, GroupRuleStatus, GroupTagValue,
    LostPasswordHash, TagValue, GroupEmailThread,
)
from sentry.utils import db
from sentry.utils.threadpool import ThreadPool


def delete_object(item):
    item.delete()


class Command(BaseCommand):
    help = 'Deletes a portion of trailing data based on creation date'

    option_list = BaseCommand.option_list + (
        make_option('--days', default='30', type=int, help='Numbers of days to truncate on.'),
        make_option('--project', type=int, help='Limit truncation to only entries from project.'),
        make_option('--concurrency', type=int, default=1, help='The number of concurrent workers to run.'),
    )

    # these models should be safe to delete without cascades, in order
    BULK_DELETES = (
        (GroupRuleStatus, 'date_added'),
        (GroupTagValue, 'last_seen'),
        (Activity, 'datetime'),
        (TagValue, 'last_seen'),
        (GroupEmailThread, 'date'),
    )

    GENERIC_DELETES = (
        (Event, 'datetime'),
        (Group, 'last_seen'),
    )

    def _postgres_bulk_speed_delete(self, model, dtfield, days=None,
                                    chunk_size=100000):
        """
        Cleanup models which we know dont need expensive (in-app) cascades or
        cache handling.

        This chunks up delete statements but still does them in bulk.
        """
        cursor = connections['default'].cursor()
        quote_name = connections['default'].ops.quote_name

        if days is None:
            days = self.days

        if self.project:
            where_extra = 'and project_id = %d' % (self.project,)<commit_msg>Dont bulk delete old activity<commit_after>"""
sentry.management.commands.cleanup
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2012 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, print_function

from datetime import timedelta
from django.core.management.base import BaseCommand
from django.db import connections
from django.utils import timezone
from optparse import make_option

from sentry.app import nodestore
from sentry.models import (
    Event, EventMapping, Group, GroupRuleStatus, GroupTagValue,
    LostPasswordHash, TagValue, GroupEmailThread,
)
from sentry.utils import db
from sentry.utils.threadpool import ThreadPool


def delete_object(item):
    item.delete()


class Command(BaseCommand):
    help = 'Deletes a portion of trailing data based on creation date'

    option_list = BaseCommand.option_list + (
        make_option('--days', default='30', type=int, help='Numbers of days to truncate on.'),
        make_option('--project', type=int, help='Limit truncation to only entries from project.'),
        make_option('--concurrency', type=int, default=1, help='The number of concurrent workers to run.'),
    )

    # these models should be safe to delete without cascades, in order
    BULK_DELETES = (
        (GroupRuleStatus, 'date_added'),
        (GroupTagValue, 'last_seen'),
        (TagValue, 'last_seen'),
        (GroupEmailThread, 'date'),
    )

    GENERIC_DELETES = (
        (Event, 'datetime'),
        (Group, 'last_seen'),
    )

    def _postgres_bulk_speed_delete(self, model, dtfield, days=None,
                                    chunk_size=100000):
        """
        Cleanup models which we know dont need expensive (in-app) cascades or
        cache handling.

        This chunks up delete statements but still does them in bulk.
        """
        cursor = connections['default'].cursor()
        quote_name = connections['default'].ops.quote_name

        if days is None:
            days = self.days

        if self.project:
            where_extra = 'and project_id = %d' % (self.project,)
{'fraction_non_alphanumeric': 0.1109309865678555, 'fraction_numerical': 0.006021306160259379, 'mean_word_length': 3.1608863198458574, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7308558', 'n_tokens_mistral': 1224, 'n_tokens_neox': 1161, 'n_words': 380}