<commit_before># Debian packaging tools: Package manipulation.
#
# Author: <NAME> <<EMAIL>>
# Last Change: February 6, 2020
# URL: https://github.com/xolox/python-deb-pkg-tools

"""Functions to build and inspect Debian binary package archives (``*.deb`` files)."""

# Standard library modules.
import collections
import copy
import fnmatch
import logging
import os
import os.path
import pipes
import random
import re
import shutil
import tempfile

# External dependencies.
from debian.deb822 import Deb822
from executor import CommandNotFound, ExternalCommandFailed, execute
from humanfriendly import coerce_boolean, concatenate, format_path, pluralize, Spinner, Timer

# Modules included in our package.
from deb_pkg_tools.control import (deb822_from_string,
                                   parse_control_fields,
                                   patch_control_file)
from deb_pkg_tools.utils import makedirs
from deb_pkg_tools.version import Version

# Initialize a logger.
logger = logging.getLogger(__name__)

# The filename extensions of Debian binary package archives.
BINARY_PACKAGE_ARCHIVE_EXTENSIONS = ('.deb', '.udeb')

# The names of control file fields that specify dependencies.
DEPENDENCY_FIELDS = ('Depends', 'Pre-Depends')

DIRECTORIES_TO_REMOVE = (
    '.bzr',         # Bazaar (version control system).
    '.git',         # Git (version control system).
    '.hg',          # Mercurial (version control system).
    '.svn',         # SVN (version control system).
    '__pycache__',  # Python 3 byte code files.
)
"""
A tuple of strings with :mod:`fnmatch` patterns of directories to remove before
building a package. Used by :func:`clean_package_tree()` which is called by
:func:`build_package()`. Avoids the following Lintian warnings:

- `package-contains-vcs-control-dir <http://lintian.debian.org/tags/package-contains-vcs-control-dir.html>`_
- `package-installs-python-pycache-dir <http://lintian.debian.org/tags/package-installs-python-pycache-dir.html>`_
"""

FILES_TO_REMOVE = (
    '*.pyc',             # Python byte code files.
    '*.pyo',             # Python optimized byte code files.
    '*~',                # Backups created by text editors (Emacs/Vim).
    '.*.s??',            # Vim named swap files.
    '.DS_Store',         # Mac OS X custom folder attributes.
    '.DS_Store.gz',      # Mac OS X custom folder attributes.
    '._*',               # Mac OS X resource forks.
    '.bzrignore',        # Bazaar ignore files.
    '.gitignore',        # Git ignore files.
    '.hg_archival.txt',  # Artefact of `hg archive'.
    '.hgignore',         # Mercurial ignore files.
    '.hgtags',           # Mercurial tags files.
    '.s??',              # Vim anonymous swap files.
)
"""
A tuple of strings with :mod:`fnmatch` patterns of files to remove before
building a package. Used by :func:`clean_package_tree()` which is called by
:func:`build_package()`. Avoids the following Lintian warnings:

- `backup-file-in-package <http://lintian.debian.org/tags/backup-file-in-package.html>`_
- `macos-ds-store-file-in-package <http://lintian.debian.org/tags/macos-ds-store-file-in-package.html>`_
- `macos-resource-fork-file-in-package <http://lintian.debian.org/tags/macos-resource-fork-file-in-package.html>`_
- `package-contains-vcs-control-file <http://lintian.debian.org/tags/package-contains-vcs-control-file.html>`_
- `package-installs-python-bytecode <http://lintian.debian.org/tags/package-installs-python-bytecode.html>`_
"""

ALLOW_CHOWN = coerce_boolean(os.environ.get('DPT_CHOWN_FILES', 'true'))
"""
:data:`True` to allow :func:`build_package()` to normalize file ownership by
running :man:`chown`, :data:`False` to disallow usage of :man:`chown`.

The environment variable ``$DPT_CHOWN_FILES`` can be used to control the value
of this variable (see :func:`~humanfriendly.coerce_boolean()` for acceptable
values).
"""

ALLOW_FAKEROOT_OR_SUDO = coerce_boolean(os.environ.get('DPT_ALLOW_FAKEROOT_OR_SUDO', 'true'))
"""
:data:`True` to allow :func:`build_package()` to use :man:`fakeroot` (when
available) or :man:`sudo` (when :man:`fakeroot` is not available),
:data:`False` to disallow this behavior.

The environment variable ``$DPT_ALLOW_FAKEROOT_OR_SUDO`` can be used to control
the value of this variable (see :func:`~humanfriendly.coerce_boolean()` for
acceptable values).
"""

ALLOW_HARD_LINKS = coerce_boolean(os.environ.get('DPT_HARD_LINKS', 'true'))
"""
:data:`True` to allow :func:`copy_package_files()` to use hard links to
optimize file copying, :data:`False` to disallow this behavior.

The environment variable ``$DPT_HARD_LINKS`` can be used to control the value
of this variable (see :func:`~humanfriendly.coerce_boolean()` for acceptable
values).
"""

ALLOW_RESET_SETGID = coerce_boolean(os.environ.get('DPT_RESET_SETGID', 'true'))
"""
:data:`True` to allow :func:`build_package()` to remove the sticky bit from
directories, :data:`False` to disallow this behavior.

The environment variable ``$DPT_RESET_SETGID`` can be used to control the value
of this variable (see :func:`~humanfriendly.coerce_boolean()` for acceptable
values).
"""

PARSE_STRICT = coerce_boolean(os.environ.get('DPT_PARSE_STRICT', 'true'))
"""
If :data:`PARSE_STRICT` is :data:`True` then :func:`parse_filename()` expects
filenames of ``*.deb`` archives to encode the package name, version and
architecture delimited by underscores. This is the default behavior and
backwards compatible with deb-pkg-tools 6.0 and older.

If :data:`PARSE_STRICT` is :data:`False` then :func:`parse_filename()` will
fall back to reading the package name, version and architecture from the
metadata contained in the ``*.deb`` archive.

The environment variable ``$DPT_PARSE_STRICT`` can be used to control the value
of this variable (see :func:`~humanfriendly.coerce_boolean()` for acceptable
values).
"""

ROOT_USER = os.environ.get('DPT_ROOT_USER', 'root')
"""
The name of the system user that is used by :func:`build_package() when it
normalizes file ownership using :man:`chown` (controlled by
:data:`ALLOW_CHOWN`).

The environment variable ``$DPT_ROOT_USER`` can be used to control the value
of this variable.
"""

ROOT_GROUP = os.environ.get('DPT_ROOT_GROUP', 'root')
"""
The name of the system group that is used by :func:`build_package() when it
normalizes file ownership using :man:`chown` (controlled by
:data:`ALLOW_CHOWN`).

The environment variable ``$DPT_ROOT_GROUP`` can be used to control the value
of this variable.
"""


def parse_filename(filename, cache=None):
    """
    Parse the filename of a Debian binary package archive.

    :param filename: The pathname of a Debian binary package archive (a string).
    :param cache: The :class:`.PackageCache` to use when :data:`PARSE_STRICT`
                  is :data:`False` (defaults to :data:`None`).
    :returns: A :class:`PackageFile` object.
    :raises: :exc:`~exceptions.ValueError` when the given filename cannot be parsed.

    This function parses the filename of a Debian binary package archive into
    three fields: the name of the package, its version and its architecture.
    See also :func:`determine_package_archive()`.

    Here's an example:

    >>> from deb_pkg_tools.package import parse_filename
    >>> components = parse_filename('/var/cache/apt/archives/python2.7_2.7.3-0ubuntu3.4_amd64.deb')
    >>> print(repr(components))
    PackageFile(name='python2.7',
                version='2.7.3-0ubuntu3.4',
                architecture='amd64',
                filename='/var/cache/apt/archives/python2.7_2.7.3-0ubuntu3.4_amd64.deb')

    """
    if isinstance(filename, PackageFile):<commit_msg>Properly document BINARY_PACKAGE_ARCHIVE_EXTENSIONS constant<commit_after># Debian packaging tools: Package manipulation.
#
# Author: <NAME> <<EMAIL>>
# Last Change: February 6, 2020
# URL: https://github.com/xolox/python-deb-pkg-tools

"""Functions to build and inspect Debian binary package archives (``*.deb`` files)."""

# Standard library modules.
import collections
import copy
import fnmatch
import logging
import os
import os.path
import pipes
import random
import re
import shutil
import tempfile

# External dependencies.
from debian.deb822 import Deb822
from executor import CommandNotFound, ExternalCommandFailed, execute
from humanfriendly import coerce_boolean, concatenate, format_path, pluralize, Spinner, Timer

# Modules included in our package.
from deb_pkg_tools.control import deb822_from_string, parse_control_fields, patch_control_file
from deb_pkg_tools.utils import makedirs
from deb_pkg_tools.version import Version

# Initialize a logger.
logger = logging.getLogger(__name__)

BINARY_PACKAGE_ARCHIVE_EXTENSIONS = ('.deb', '.udeb')
"""
A tuple of strings with supported filename extensions of Debian binary package
archives. Used by :func:`find_package_archives()` and :func:`parse_filename()`.
"""

# The names of control file fields that specify dependencies.
DEPENDENCY_FIELDS = ('Depends', 'Pre-Depends')

DIRECTORIES_TO_REMOVE = (
    '.bzr',         # Bazaar (version control system).
    '.git',         # Git (version control system).
    '.hg',          # Mercurial (version control system).
    '.svn',         # SVN (version control system).
    '__pycache__',  # Python 3 byte code files.
)
"""
A tuple of strings with :mod:`fnmatch` patterns of directories to remove before
building a package. Used by :func:`clean_package_tree()` which is called by
:func:`build_package()`. Avoids the following Lintian warnings:

- `package-contains-vcs-control-dir <http://lintian.debian.org/tags/package-contains-vcs-control-dir.html>`_
- `package-installs-python-pycache-dir <http://lintian.debian.org/tags/package-installs-python-pycache-dir.html>`_
"""

FILES_TO_REMOVE = (
    '*.pyc',             # Python byte code files.
    '*.pyo',             # Python optimized byte code files.
    '*~',                # Backups created by text editors (Emacs/Vim).
    '.*.s??',            # Vim named swap files.
    '.DS_Store',         # Mac OS X custom folder attributes.
    '.DS_Store.gz',      # Mac OS X custom folder attributes.
    '._*',               # Mac OS X resource forks.
    '.bzrignore',        # Bazaar ignore files.
    '.gitignore',        # Git ignore files.
    '.hg_archival.txt',  # Artefact of `hg archive'.
    '.hgignore',         # Mercurial ignore files.
    '.hgtags',           # Mercurial tags files.
    '.s??',              # Vim anonymous swap files.
)
"""
A tuple of strings with :mod:`fnmatch` patterns of files to remove before
building a package. Used by :func:`clean_package_tree()` which is called by
:func:`build_package()`. Avoids the following Lintian warnings:

- `backup-file-in-package <http://lintian.debian.org/tags/backup-file-in-package.html>`_
- `macos-ds-store-file-in-package <http://lintian.debian.org/tags/macos-ds-store-file-in-package.html>`_
- `macos-resource-fork-file-in-package <http://lintian.debian.org/tags/macos-resource-fork-file-in-package.html>`_
- `package-contains-vcs-control-file <http://lintian.debian.org/tags/package-contains-vcs-control-file.html>`_
- `package-installs-python-bytecode <http://lintian.debian.org/tags/package-installs-python-bytecode.html>`_
"""

ALLOW_CHOWN = coerce_boolean(os.environ.get('DPT_CHOWN_FILES', 'true'))
"""
:data:`True` to allow :func:`build_package()` to normalize file ownership by
running :man:`chown`, :data:`False` to disallow usage of :man:`chown`.

The environment variable ``$DPT_CHOWN_FILES`` can be used to control the value
of this variable (see :func:`~humanfriendly.coerce_boolean()` for acceptable
values).
"""

ALLOW_FAKEROOT_OR_SUDO = coerce_boolean(os.environ.get('DPT_ALLOW_FAKEROOT_OR_SUDO', 'true'))
"""
:data:`True` to allow :func:`build_package()` to use :man:`fakeroot` (when
available) or :man:`sudo` (when :man:`fakeroot` is not available),
:data:`False` to disallow this behavior.

The environment variable ``$DPT_ALLOW_FAKEROOT_OR_SUDO`` can be used to control
the value of this variable (see :func:`~humanfriendly.coerce_boolean()` for
acceptable values).
"""

ALLOW_HARD_LINKS = coerce_boolean(os.environ.get('DPT_HARD_LINKS', 'true'))
"""
:data:`True` to allow :func:`copy_package_files()` to use hard links to
optimize file copying, :data:`False` to disallow this behavior.

The environment variable ``$DPT_HARD_LINKS`` can be used to control the value
of this variable (see :func:`~humanfriendly.coerce_boolean()` for acceptable
values).
"""

ALLOW_RESET_SETGID = coerce_boolean(os.environ.get('DPT_RESET_SETGID', 'true'))
"""
:data:`True` to allow :func:`build_package()` to remove the sticky bit from
directories, :data:`False` to disallow this behavior.

The environment variable ``$DPT_RESET_SETGID`` can be used to control the value
of this variable (see :func:`~humanfriendly.coerce_boolean()` for acceptable
values).
"""

PARSE_STRICT = coerce_boolean(os.environ.get('DPT_PARSE_STRICT', 'true'))
"""
If :data:`PARSE_STRICT` is :data:`True` then :func:`parse_filename()` expects
filenames of ``*.deb`` archives to encode the package name, version and
architecture delimited by underscores. This is the default behavior and
backwards compatible with deb-pkg-tools 6.0 and older.

If :data:`PARSE_STRICT` is :data:`False` then :func:`parse_filename()` will
fall back to reading the package name, version and architecture from the
metadata contained in the ``*.deb`` archive.

The environment variable ``$DPT_PARSE_STRICT`` can be used to control the value
of this variable (see :func:`~humanfriendly.coerce_boolean()` for acceptable
values).
"""

ROOT_USER = os.environ.get('DPT_ROOT_USER', 'root')
"""
The name of the system user that is used by :func:`build_package() when it
normalizes file ownership using :man:`chown` (controlled by
:data:`ALLOW_CHOWN`).

The environment variable ``$DPT_ROOT_USER`` can be used to control the value
of this variable.
"""

ROOT_GROUP = os.environ.get('DPT_ROOT_GROUP', 'root')
"""
The name of the system group that is used by :func:`build_package() when it
normalizes file ownership using :man:`chown` (controlled by
:data:`ALLOW_CHOWN`).

The environment variable ``$DPT_ROOT_GROUP`` can be used to control the value
of this variable.
"""


def parse_filename(filename, cache=None):
    """
    Parse the filename of a Debian binary package archive.

    :param filename: The pathname of a Debian binary package archive (a string).
    :param cache: The :class:`.PackageCache` to use when :data:`PARSE_STRICT`
                  is :data:`False` (defaults to :data:`None`).
    :returns: A :class:`PackageFile` object.
    :raises: :exc:`~exceptions.ValueError` in the following circumstances:

             - The filename extension doesn't match any of the known
               :data:`BINARY_PACKAGE_ARCHIVE_EXTENSIONS`.

             - The filename doesn't have three underscore separated components
               (and :ref:`PARSE_STRICT` is :data:`True`).

    This function parses the filename of a Debian binary package archive into
    three fields: the name of the package, its version and its architecture.
    See also :func:`determine_package_archive()`.

    Here's an example:

    >>> from deb_pkg_tools.package import parse_filename
    >>> components = parse_filename('/var/cache/apt/archives/python2.7_2.7.3-0ubuntu3.4_amd64.deb')
    >>> print(repr(components))
    PackageFile(name='python2.7',
                version='2.7.3-0ubuntu3.4',
                architecture='amd64',
                filename='/var/cache/apt/archives/python2.7_2.7.3-0ubuntu3.4_amd64.deb')

    """
    if isinstance(filename, PackageFile):
{'fraction_non_alphanumeric': 0.12397916532698862, 'fraction_numerical': 0.006044627355154009, 'mean_word_length': 4.1667774086378735, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 41, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4591768', 'n_tokens_mistral': 4953, 'n_tokens_neox': 4711, 'n_words': 1424}