<commit_before>from datetime import datetime

from django.core.management.base import BaseCommand, CommandError
from django.db.models import Q, F

from celery import chord, group

from olympia import amo
from olympia.addons.models import Addon
from olympia.addons.tasks import (
    add_dynamic_theme_tag,
    delete_addons,
    extract_colors_from_static_themes,
    find_inconsistencies_between_es_and_db,
    migrate_webextensions_to_git_storage,
    recreate_theme_previews,
)
from olympia.abuse.models import AbuseReport
from olympia.constants.base import _ADDON_PERSONA, _ADDON_THEME, _ADDON_WEBAPP
from olympia.amo.utils import chunked
from olympia.devhub.tasks import get_preview_sizes, recreate_previews
from olympia.git.tasks import delete_source_git_repositories
from olympia.lib.crypto.tasks import sign_addons
from olympia.reviewers.tasks import recalculate_post_review_weight
from olympia.versions.compare import version_int
from olympia.versions.models import Version


firefox_56_star = version_int('56.*')
current_autoapprovalsummary = '_current_version__autoapprovalsummary__'


def get_recalc_needed_filters():
    summary_modified = F('_current_version__autoapprovalsummary__modified')
    # We don't take deleted reports into account
    valid_abuse_report_states = (
        AbuseReport.STATES.UNTRIAGED, AbuseReport.STATES.VALID,
        AbuseReport.STATES.SUSPICIOUS)
    return [
        # Only recalculate add-ons that received recent abuse reports
        # possibly through their authors.
        Q(
            abuse_reports__state__in=valid_abuse_report_states,
            abuse_reports__created__gte=summary_modified
        ) |
        Q(
            authors__abuse_reports__state__in=valid_abuse_report_states,
            authors__abuse_reports__created__gte=summary_modified
        ) |

        # And check ratings that have a rating of 3 or less
        Q(
            _current_version__ratings__deleted=False,
            _current_version__ratings__created__gte=summary_modified,
            _current_version__ratings__rating__lte=3)
    ]


tasks = {
    'find_inconsistencies_between_es_and_db': {
        'method': find_inconsistencies_between_es_and_db, 'qs': []},
    'get_preview_sizes': {'method': get_preview_sizes, 'qs': []},
    'recalculate_post_review_weight': {
        'method': recalculate_post_review_weight,
        'qs': [
            Q(**{current_autoapprovalsummary + 'verdict': amo.AUTO_APPROVED}) &
            ~Q(**{current_autoapprovalsummary + 'confirmed': True})
        ]},
    'constantly_recalculate_post_review_weight': {
        # This re-calculates the whole post-review weight which can be costly
        # so take that into account. We may want to optimize that later
        # in case we notice things are slower than needed - cgrebs 20190730
        'method': recalculate_post_review_weight,
        'kwargs': {'only_current_version': True},
        'qs': get_recalc_needed_filters()},
    'resign_addons_for_cose': {
        'method': sign_addons,
        'qs': [
            # Only resign public add-ons where the latest version has been
            # created before the 5th of April
            Q(status=amo.STATUS_APPROVED,
              _current_version__created__lt=datetime(2019, 4, 5)) &
            ~Q(type=amo.ADDON_SEARCH)
        ]
    },
    'recreate_previews': {
        'method': recreate_previews,
        'qs': [
            ~Q(type=amo.ADDON_STATICTHEME)
        ]
    },
    'recreate_theme_previews': {
        'method': recreate_theme_previews,
        'qs': [
            Q(type=amo.ADDON_STATICTHEME, status__in=[
                amo.STATUS_APPROVED, amo.STATUS_AWAITING_REVIEW])
        ],
        'kwargs': {'only_missing': False},
    },
    'create_missing_theme_previews': {
        'method': recreate_theme_previews,
        'qs': [
            Q(type=amo.ADDON_STATICTHEME, status__in=[
                amo.STATUS_APPROVED, amo.STATUS_AWAITING_REVIEW])
        ],
        'kwargs': {'only_missing': True},
    },
    'add_dynamic_theme_tag_for_theme_api': {
        'method': add_dynamic_theme_tag,
        'qs': [
            Q(status=amo.STATUS_APPROVED,
              _current_version__files__is_webextension=True)
        ]
    },
    'extract_webextensions_to_git_storage': {
        'method': migrate_webextensions_to_git_storage,
        'qs': [
            Q(type__in=(
                # Ignoring legacy add-ons and lightweight themes
                amo.ADDON_EXTENSION, amo.ADDON_STATICTHEME,
                amo.ADDON_DICT, amo.ADDON_LPAPP, amo.ADDON_SEARCH))
        ],
        'distinct': True,
        'allowed_kwargs': ('channel',),
    },
    'extract_colors_from_static_themes': {
        'method': extract_colors_from_static_themes,
        'qs': [Q(type=amo.ADDON_STATICTHEME)]
    },
    'delete_source_git_repositories': {
        'method': delete_source_git_repositories,
        'distinct': True,
        'qs': [
            Q(
                # Retrieve a list of add-on IDs that have at least one version
                # with a source git hash (which means source files have been
                # git-extracted).
                pk__in=Version.unfiltered.exclude(source_git_hash__exact='')
                .values_list('addon_id', flat=True)
                .distinct()
            )
        ],
        'allowed_kwargs': ('with_deleted',),<commit_msg>Optimize query for deleting source git repositories (#14197)<commit_after>from datetime import datetime

from django.core.management.base import BaseCommand, CommandError
from django.db.models import Q, F

from celery import chord, group

from olympia import amo
from olympia.addons.models import Addon
from olympia.addons.tasks import (
    add_dynamic_theme_tag,
    delete_addons,
    extract_colors_from_static_themes,
    find_inconsistencies_between_es_and_db,
    migrate_webextensions_to_git_storage,
    recreate_theme_previews,
)
from olympia.abuse.models import AbuseReport
from olympia.constants.base import _ADDON_PERSONA, _ADDON_THEME, _ADDON_WEBAPP
from olympia.amo.utils import chunked
from olympia.devhub.tasks import get_preview_sizes, recreate_previews
from olympia.git.tasks import delete_source_git_repositories
from olympia.lib.crypto.tasks import sign_addons
from olympia.reviewers.tasks import recalculate_post_review_weight
from olympia.versions.compare import version_int
from olympia.versions.models import Version


firefox_56_star = version_int('56.*')
current_autoapprovalsummary = '_current_version__autoapprovalsummary__'


def get_recalc_needed_filters():
    summary_modified = F('_current_version__autoapprovalsummary__modified')
    # We don't take deleted reports into account
    valid_abuse_report_states = (
        AbuseReport.STATES.UNTRIAGED, AbuseReport.STATES.VALID,
        AbuseReport.STATES.SUSPICIOUS)
    return [
        # Only recalculate add-ons that received recent abuse reports
        # possibly through their authors.
        Q(
            abuse_reports__state__in=valid_abuse_report_states,
            abuse_reports__created__gte=summary_modified
        ) |
        Q(
            authors__abuse_reports__state__in=valid_abuse_report_states,
            authors__abuse_reports__created__gte=summary_modified
        ) |

        # And check ratings that have a rating of 3 or less
        Q(
            _current_version__ratings__deleted=False,
            _current_version__ratings__created__gte=summary_modified,
            _current_version__ratings__rating__lte=3)
    ]


tasks = {
    'find_inconsistencies_between_es_and_db': {
        'method': find_inconsistencies_between_es_and_db, 'qs': []},
    'get_preview_sizes': {'method': get_preview_sizes, 'qs': []},
    'recalculate_post_review_weight': {
        'method': recalculate_post_review_weight,
        'qs': [
            Q(**{current_autoapprovalsummary + 'verdict': amo.AUTO_APPROVED}) &
            ~Q(**{current_autoapprovalsummary + 'confirmed': True})
        ]},
    'constantly_recalculate_post_review_weight': {
        # This re-calculates the whole post-review weight which can be costly
        # so take that into account. We may want to optimize that later
        # in case we notice things are slower than needed - cgrebs 20190730
        'method': recalculate_post_review_weight,
        'kwargs': {'only_current_version': True},
        'qs': get_recalc_needed_filters()},
    'resign_addons_for_cose': {
        'method': sign_addons,
        'qs': [
            # Only resign public add-ons where the latest version has been
            # created before the 5th of April
            Q(status=amo.STATUS_APPROVED,
              _current_version__created__lt=datetime(2019, 4, 5)) &
            ~Q(type=amo.ADDON_SEARCH)
        ]
    },
    'recreate_previews': {
        'method': recreate_previews,
        'qs': [
            ~Q(type=amo.ADDON_STATICTHEME)
        ]
    },
    'recreate_theme_previews': {
        'method': recreate_theme_previews,
        'qs': [
            Q(type=amo.ADDON_STATICTHEME, status__in=[
                amo.STATUS_APPROVED, amo.STATUS_AWAITING_REVIEW])
        ],
        'kwargs': {'only_missing': False},
    },
    'create_missing_theme_previews': {
        'method': recreate_theme_previews,
        'qs': [
            Q(type=amo.ADDON_STATICTHEME, status__in=[
                amo.STATUS_APPROVED, amo.STATUS_AWAITING_REVIEW])
        ],
        'kwargs': {'only_missing': True},
    },
    'add_dynamic_theme_tag_for_theme_api': {
        'method': add_dynamic_theme_tag,
        'qs': [
            Q(status=amo.STATUS_APPROVED,
              _current_version__files__is_webextension=True)
        ]
    },
    'extract_webextensions_to_git_storage': {
        'method': migrate_webextensions_to_git_storage,
        'qs': [
            Q(type__in=(
                # Ignoring legacy add-ons and lightweight themes
                amo.ADDON_EXTENSION, amo.ADDON_STATICTHEME,
                amo.ADDON_DICT, amo.ADDON_LPAPP, amo.ADDON_SEARCH))
        ],
        'distinct': True,
        'allowed_kwargs': ('channel',),
    },
    'extract_colors_from_static_themes': {
        'method': extract_colors_from_static_themes,
        'qs': [Q(type=amo.ADDON_STATICTHEME)]
    },
    'delete_source_git_repositories': {
        'method': delete_source_git_repositories,
        'distinct': True,
        'qs': [
            Q(
                # Retrieve a list of add-on IDs that have at least one version
                # with a source git hash (which means source files have been
                # git-extracted).
                pk__in=Version.unfiltered.exclude(source_git_hash__exact='')
                # This is needed to make `.distinct()` work, see:
                # https://code.djangoproject.com/ticket/16058
                .order_by()
                .values_list('addon_id', flat=True)
                .distinct()
            )
        ],
        'allowed_kwargs': ('with_deleted',),
{'fraction_non_alphanumeric': 0.08748746924268659, 'fraction_numerical': 0.004738904583978857, 'mean_word_length': 2.789364640883978, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5997408', 'n_tokens_mistral': 3494, 'n_tokens_neox': 3368, 'n_words': 632}