<commit_before># This file is part of Cockpit.
#
# Copyright (C) 2022 Red Hat, Inc.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <https://www.gnu.org/licenses/>.


import collections
import hashlib
import json
import fnmatch
import logging
import mimetypes
import os
import zipfile

from pathlib import Path

import packaging.version

VERSION = packaging.version.Version("300")
logger = logging.getLogger('cockpit.packages')


# Sorting is important because the checksums are dependent on the order we
# visit these.  This is not the same as sorting on the full pathname, since
# each directory component is considered separately.  We could split the path
# and sort on that, but it ends up not being particularly elegant.
# Changing the approach here will change the checksum.
# We're allowed to change it as we wish, but let's try to match behaviour with
# cockpitpackages.c for the time being, since the unit tests depend on it.
def directory_items(path):
    return sorted(path.iterdir(), key=lambda item: item.name)


class Package:
    def __init__(self, path):
        self.path = path

        with (self.path / 'manifest.json').open(encoding='utf-8') as manifest_file:
            self.manifest = json.load(manifest_file)

        if 'name' in self.manifest:
            self.name = self.manifest['name']
        else:
            self.name = path.name

        self.content_security_policy = None
        self.priority = self.manifest.get('priority', 1)
        self.bridges = self.manifest.get('bridges', [])

        self.files = set()
        for file in self.path.rglob('*'):
            self.files.add(file.relative_to(self.path))

    def walk(self, checksums, path=None):
        if not path:
            path = self.path

        for item in directory_items(path):
            if item.is_dir():
                self.walk(checksums, item)
            elif item.is_file():
                rel = item.relative_to(path)
                with item.open('rb') as file:
                    data = file.read()
                sha = hashlib.sha256(data).hexdigest()
                for context in checksums:
                    context.update(f'{rel}\0{sha}\0'.encode('ascii'))

    def check(self, at_least_prio):
        if 'requires' in self.manifest:
            requires = self.manifest['requires']
            if any(package != 'cockpit' for package in requires):
                return False

            if 'cockpit' in requires and VERSION < packaging.version.Version(requires['cockpit']):
                return False

        if at_least_prio is not None:
            if 'priority' not in self.manifest:
                return False

            if self.manifest['priority'] <= at_least_prio:
                return False

        return True

    @staticmethod
    def filename_variants(filename, locale):
        base, _, ext = filename.rpartition('.')

        while base:
            if locale:
                yield f'{base}.{locale}.{ext}'
                yield f'{base}.{locale}.{ext}.gz'

                if '_' in locale:
                    language, _, _ = locale.partition(' ')
                    yield f'{base}.{language}.{ext}'
                    yield f'{base}.{language}.{ext}.gz'

            yield f'{base}.{ext}'
            yield f'{base}.min.{ext}'
            yield f'{base}.{ext}'
            yield f'{base}.{ext}.gz'
            yield f'{base}.min.{ext}.gz'
<commit_msg>python: use pkg_resources instead of packaging<commit_after># This file is part of Cockpit.
#
# Copyright (C) 2022 Red Hat, Inc.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <https://www.gnu.org/licenses/>.


import collections
import hashlib
import json
import fnmatch
import logging
import mimetypes
import os
import pkg_resources
import zipfile

from pathlib import Path

VERSION = pkg_resources.parse_version('300')
logger = logging.getLogger('cockpit.packages')


# Sorting is important because the checksums are dependent on the order we
# visit these.  This is not the same as sorting on the full pathname, since
# each directory component is considered separately.  We could split the path
# and sort on that, but it ends up not being particularly elegant.
# Changing the approach here will change the checksum.
# We're allowed to change it as we wish, but let's try to match behaviour with
# cockpitpackages.c for the time being, since the unit tests depend on it.
def directory_items(path):
    return sorted(path.iterdir(), key=lambda item: item.name)


class Package:
    def __init__(self, path):
        self.path = path

        with (self.path / 'manifest.json').open(encoding='utf-8') as manifest_file:
            self.manifest = json.load(manifest_file)

        if 'name' in self.manifest:
            self.name = self.manifest['name']
        else:
            self.name = path.name

        self.content_security_policy = None
        self.priority = self.manifest.get('priority', 1)
        self.bridges = self.manifest.get('bridges', [])

        self.files = set()
        for file in self.path.rglob('*'):
            self.files.add(file.relative_to(self.path))

    def walk(self, checksums, path=None):
        if not path:
            path = self.path

        for item in directory_items(path):
            if item.is_dir():
                self.walk(checksums, item)
            elif item.is_file():
                rel = item.relative_to(path)
                with item.open('rb') as file:
                    data = file.read()
                sha = hashlib.sha256(data).hexdigest()
                for context in checksums:
                    context.update(f'{rel}\0{sha}\0'.encode('ascii'))

    def check(self, at_least_prio):
        if 'requires' in self.manifest:
            requires = self.manifest['requires']
            if any(package != 'cockpit' for package in requires):
                return False

            if 'cockpit' in requires and VERSION < pkg_resources.parse_version(requires['cockpit']):
                return False

        if at_least_prio is not None:
            if 'priority' not in self.manifest:
                return False

            if self.manifest['priority'] <= at_least_prio:
                return False

        return True

    @staticmethod
    def filename_variants(filename, locale):
        base, _, ext = filename.rpartition('.')

        while base:
            if locale:
                yield f'{base}.{locale}.{ext}'
                yield f'{base}.{locale}.{ext}.gz'

                if '_' in locale:
                    language, _, _ = locale.partition(' ')
                    yield f'{base}.{language}.{ext}'
                    yield f'{base}.{language}.{ext}.gz'

            yield f'{base}.{ext}'
            yield f'{base}.min.{ext}'
            yield f'{base}.{ext}'
            yield f'{base}.{ext}.gz'
            yield f'{base}.min.{ext}.gz'

{'fraction_non_alphanumeric': 0.09824561403508772, 'fraction_numerical': 0.0037593984962406013, 'mean_word_length': 2.2897774113767517, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '777482', 'n_tokens_mistral': 2254, 'n_tokens_neox': 2105, 'n_words': 765}