<commit_before>import logging
import requests

import pdcupdater.handlers
import pdcupdater.services
import pdcupdater.utils

from pdc_client import get_paged


log = logging.getLogger(__name__)


class AtomicComponentGroupHandler(pdcupdater.handlers.BaseHandler):
    """ When someone changes the packages list for fedora-atomic.

    https://git.fedorahosted.org/cgit/fedora-atomic.git/tree/fedora-atomic-docker-host.json

    """
    group_type = 'atomic-docker-host'

    def __init__(self, *args, **kwargs):
        super(AtomicComponentGroupHandler, self).__init__(*args, **kwargs)
        self.git_url = self.config['pdcupdater.fedora_atomic_git_url']

    @property
    def topic_suffixes(self):
        return [
            'trac.git.receive',
        ]

    def can_handle(self, msg):
        if not msg['topic'].endswith('trac.git.receive'):
            return False
        if msg['msg']['commit']['repo'] != 'fedora-atomic':
            return False
        return True

    def atomic_component_groups_from_git(self, pdc):
        # First, build a mapping of git branches (from the fedora-atomic
        # fedorahosted repo) to PDC release ids.
        tags = [pdcupdater.utils.rawhide_tag()]
        for release in pdcupdater.utils.bodhi_releases():
            if 'EPEL' in release['id_prefix']:
                # We don't maintain an atomic group for epel.
                continue
            tags.append(release['stable_tag'])

        pdc_releases = [pdcupdater.utils.tag2release(tag) for tag in tags]
        for release_id, release in pdc_releases:
            # First, make sure PDC can handle a group on this release
            pdcupdater.utils.ensure_release_exists(pdc, release_id, release)

            # Then, map the fedorahosted git repo branch to our PDC release
            if release['release_type'] == 'ga':
                branch = 'master'
            else:
                branch = 'f' + release['version']

            # Go, get, and parse the data
            params = dict(h=branch)
            filename = 'fedora-%s.json' % self.group_type
            response = requests.get(self.git_url + filename, params=params)
            data = response.json()

            # Some of the packages listed *could* be sub-packages, but in the
            # PDC component group we want to deal with parent srpms.  So, use
            # mdapi to convert based on whatevers in the repos right now.
            packages = [
                pdcupdater.utils.subpackage2parent(package, release)
                for package in data['packages']
            ]

            # And return formatted component group data
            yield {
                'group_type': self.group_type,
                'release': release_id,
                'description': 'Deps for %s %s' % (
                    self.group_type,
                    self.git_url,
                ),
                'components': [{
                    'release': release_id,
                    'name': package,
                } for package in packages],
            }

    def handle(self, pdc, msg):
        component_groups = self.atomic_component_groups_from_git(pdc)
        for group in component_groups:
            self._update_atomic_component_group(pdc, group)

    def audit(self, pdc):
        # Query the data sources
        git_groups = list(self.atomic_component_groups_from_git(pdc))
        pdc_groups = get_paged(pdc['component-groups']._)<commit_msg>atomic: Remove a duplicate component-groups query<commit_after>import logging
import requests

import pdcupdater.handlers
import pdcupdater.services
import pdcupdater.utils

from pdc_client import get_paged


log = logging.getLogger(__name__)


class AtomicComponentGroupHandler(pdcupdater.handlers.BaseHandler):
    """ When someone changes the packages list for fedora-atomic.

    https://git.fedorahosted.org/cgit/fedora-atomic.git/tree/fedora-atomic-docker-host.json

    """
    group_type = 'atomic-docker-host'

    def __init__(self, *args, **kwargs):
        super(AtomicComponentGroupHandler, self).__init__(*args, **kwargs)
        self.git_url = self.config['pdcupdater.fedora_atomic_git_url']

    @property
    def topic_suffixes(self):
        return [
            'trac.git.receive',
        ]

    def can_handle(self, msg):
        if not msg['topic'].endswith('trac.git.receive'):
            return False
        if msg['msg']['commit']['repo'] != 'fedora-atomic':
            return False
        return True

    def atomic_component_groups_from_git(self, pdc):
        # First, build a mapping of git branches (from the fedora-atomic
        # fedorahosted repo) to PDC release ids.
        tags = [pdcupdater.utils.rawhide_tag()]
        for release in pdcupdater.utils.bodhi_releases():
            if 'EPEL' in release['id_prefix']:
                # We don't maintain an atomic group for epel.
                continue
            tags.append(release['stable_tag'])

        pdc_releases = [pdcupdater.utils.tag2release(tag) for tag in tags]
        for release_id, release in pdc_releases:
            # First, make sure PDC can handle a group on this release
            pdcupdater.utils.ensure_release_exists(pdc, release_id, release)

            # Then, map the fedorahosted git repo branch to our PDC release
            if release['release_type'] == 'ga':
                branch = 'master'
            else:
                branch = 'f' + release['version']

            # Go, get, and parse the data
            params = dict(h=branch)
            filename = 'fedora-%s.json' % self.group_type
            response = requests.get(self.git_url + filename, params=params)
            data = response.json()

            # Some of the packages listed *could* be sub-packages, but in the
            # PDC component group we want to deal with parent srpms.  So, use
            # mdapi to convert based on whatevers in the repos right now.
            packages = [
                pdcupdater.utils.subpackage2parent(package, release)
                for package in data['packages']
            ]

            # And return formatted component group data
            yield {
                'group_type': self.group_type,
                'release': release_id,
                'description': 'Deps for %s %s' % (
                    self.group_type,
                    self.git_url,
                ),
                'components': [{
                    'release': release_id,
                    'name': package,
                } for package in packages],
            }

    def handle(self, pdc, msg):
        component_groups = self.atomic_component_groups_from_git(pdc)
        for group in component_groups:
            self._update_atomic_component_group(pdc, group)

    def audit(self, pdc):
        # Query the data sources
        git_groups = list(self.atomic_component_groups_from_git(pdc))
{'fraction_non_alphanumeric': 0.09184561810795125, 'fraction_numerical': 0.0005803830528148578, 'mean_word_length': 2.0690115761353516, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '944384', 'n_tokens_mistral': 1964, 'n_tokens_neox': 1886, 'n_words': 502}