<commit_before># /psqtraviscontainer/package_system.py
#
# Implementations of package-system controllers for various distributions.
#
# See /LICENCE.md for Copyright information
"""Implementations of package-system controllers for various distributions."""

import abc

import fnmatch

import os

import shutil

import subprocess

import sys

import tarfile

import tempfile

import textwrap

from collections import namedtuple

from clint.textui import colored

from psqtraviscontainer import debian_package
from psqtraviscontainer import directory
from psqtraviscontainer import download

import six

import tempdir

_UBUNTU_MAIN_ARCHS = ["i386", "amd64"]
_UBUNTU_PORT_ARCHS = ["armhf", "arm64", "powerpc", "ppc64el"]
_UBUNTU_MAIN_ARCHIVE = "http://archive.ubuntu.com/ubuntu/"
_UBUNTU_PORT_ARCHIVE = "http://ports.ubuntu.com/ubuntu-ports/"


def _report_task(description):
    """Report task description."""
    sys.stdout.write(str(colored.white("-> {0}\n".format(description))))


def _run_task(executor, description, argv, env=None, detail=None):
    """Run command through executor argv and prints description."""
    detail = "[{}]".format(" ".join(argv)) if detail is None else detail
    _report_task(description + " " + detail)
    code, stdout, stderr = executor.execute(argv,
                                            requires_full_access=True,
                                            env=env)
    print(textwrap.indent(stdout, "   "))
    print(textwrap.indent(stderr, "   "))


class PackageSystem(six.with_metaclass(abc.ABCMeta, object)):
    """An abstract class representing a package manager."""

    PopenArguments = namedtuple("PopenArguments", "argv env")

    @abc.abstractmethod
    def add_repositories(self, repos):
        """Add repositories to central packaging system."""
        del repos

        raise NotImplementedError()

    @abc.abstractmethod
    def install_packages(self, package_names):
        """Install specified packages in package_names."""
        del package_names

        raise NotImplementedError()


class Dpkg(PackageSystem):
    """Debian Packaging System."""

    def __init__(self,
                 release,
                 arch,
                 executor):
        """Initialize Dpkg with release and arch."""
        super(Dpkg, self).__init__()
        self._release = release
        self._arch = arch
        self._executor = executor

    def add_repositories(self, repos):
        """Add a repository to the central packaging system."""
        _ubuntu_urls = [
            (_UBUNTU_MAIN_ARCHS, _UBUNTU_MAIN_ARCHIVE),
            (_UBUNTU_PORT_ARCHS, _UBUNTU_PORT_ARCHIVE)
        ]

        def _format_user_line(line, kwargs):
            """Format a line and turns it into a valid repo line."""
            formatted_line = line.format(**kwargs)
            return "deb {0}".format(formatted_line)

        def _value_or_error(value):
            """Return first item in value, or ERROR if value is empty."""
            return value[0] if len(value) else "ERROR"

        format_keys = {
            "ubuntu": [u[1] for u in _ubuntu_urls if self._arch in u[0]],
            "debian": ["http://ftp.debian.org/"],
            "launchpad": ["http://ppa.launchpad.net/"],
            "release": [self._release]
        }
        format_keys = {
            k: _value_or_error(v) for k, v in format_keys.items()
        }

        # We will be creating a bash script each time we need to add
        # a new source line to our sources list and executing that inside
        # the proot. This guarantees that we'll always get the right
        # permissions.
        with tempfile.NamedTemporaryFile() as bash_script:
            append_lines = [_format_user_line(l, format_keys) for l in repos]
            for count, append_line in enumerate(append_lines):
                path = "/etc/apt/sources.list.d/{0}.list".format(count)
                append_cmd = "echo \"{0}\" > {1}\n".format(append_line, path)
                bash_script.write(six.b(append_cmd))

            bash_script.flush()
            self._executor.execute_success(["bash", bash_script.name],
                                           requires_full_access=True)

    def install_packages(self, package_names):
        """Install all packages in list package_names."""
        _run_task(self._executor,
                  """Update repositories""",
                  ["apt-get", "update", "-qq", "-y", "--force-yes"])
        _run_task(self._executor,
                  """Install {0}""".format(str(package_names)),
                  ["apt-get",
                   "install",
                   "-qq",<commit_msg>package_system: Refactor out function to format apt repositories<commit_after># /psqtraviscontainer/package_system.py
#
# Implementations of package-system controllers for various distributions.
#
# See /LICENCE.md for Copyright information
"""Implementations of package-system controllers for various distributions."""

import abc

import fnmatch

import os

import shutil

import subprocess

import sys

import tarfile

import tempfile

import textwrap

from collections import namedtuple

from clint.textui import colored

from psqtraviscontainer import debian_package
from psqtraviscontainer import directory
from psqtraviscontainer import download

import six

import tempdir

_UBUNTU_MAIN_ARCHS = ["i386", "amd64"]
_UBUNTU_PORT_ARCHS = ["armhf", "arm64", "powerpc", "ppc64el"]
_UBUNTU_MAIN_ARCHIVE = "http://archive.ubuntu.com/ubuntu/"
_UBUNTU_PORT_ARCHIVE = "http://ports.ubuntu.com/ubuntu-ports/"


def _report_task(description):
    """Report task description."""
    sys.stdout.write(str(colored.white("-> {0}\n".format(description))))


def _run_task(executor, description, argv, env=None, detail=None):
    """Run command through executor argv and prints description."""
    detail = "[{}]".format(" ".join(argv)) if detail is None else detail
    _report_task(description + " " + detail)
    code, stdout, stderr = executor.execute(argv,
                                            requires_full_access=True,
                                            env=env)
    print(textwrap.indent(stdout, "   "))
    print(textwrap.indent(stderr, "   "))


class PackageSystem(six.with_metaclass(abc.ABCMeta, object)):
    """An abstract class representing a package manager."""

    PopenArguments = namedtuple("PopenArguments", "argv env")

    @abc.abstractmethod
    def add_repositories(self, repos):
        """Add repositories to central packaging system."""
        del repos

        raise NotImplementedError()

    @abc.abstractmethod
    def install_packages(self, package_names):
        """Install specified packages in package_names."""
        del package_names

        raise NotImplementedError()


class Dpkg(PackageSystem):
    """Debian Packaging System."""

    def __init__(self,
                 release,
                 arch,
                 executor):
        """Initialize Dpkg with release and arch."""
        super(Dpkg, self).__init__()
        self._release = release
        self._arch = arch
        self._executor = executor

    @staticmethod
    def format_repositories(repos, release, arch):
        """Take a list of APT lines and format them.

        There are certain shortcuts that you can use.

        {ubuntu} will be replaced by http://archive.ubuntu.com/ and
        the architecture.

        {debian} will be replaced by http://ftp.debian.org/.

        {launchpad} will be replaced by "http://ppa.launchpad.net/.

        {release} gets replaced by the release of the distribution, which
        means you don't need a repository file for every distribution.
        """
        _ubuntu_urls = [
            (_UBUNTU_MAIN_ARCHS, _UBUNTU_MAIN_ARCHIVE),
            (_UBUNTU_PORT_ARCHS, _UBUNTU_PORT_ARCHIVE)
        ]

        def _format_user_line(line, kwargs):
            """Format a line and turns it into a valid repository line."""
            formatted_line = line.format(**kwargs)
            return "deb {0}".format(formatted_line)

        def _value_or_error(value):
            """Return first item in value, or ERROR if value is empty."""
            return value[0] if len(value) else "ERROR"

        format_keys = {
            "ubuntu": [u[1] for u in _ubuntu_urls if arch in u[0]],
            "debian": ["http://ftp.debian.org/"],
            "launchpad": ["http://ppa.launchpad.net/"],
            "release": [release]
        }
        format_keys = {
            k: _value_or_error(v) for k, v in format_keys.items()
        }

        return [_format_user_line(l, format_keys) for l in repos]

    def add_repositories(self, repos):
        """Add a repository to the central packaging system."""
        # We will be creating a bash script each time we need to add
        # a new source line to our sources list and executing that inside
        # the proot. This guarantees that we'll always get the right
        # permissions.
        with tempfile.NamedTemporaryFile() as bash_script:
            append_lines = Dpkg.format_repositories(repos,
                                                    self._release,
                                                    self._arch)
            for count, append_line in enumerate(append_lines):
                path = "/etc/apt/sources.list.d/{0}.list".format(count)
                append_cmd = "echo \"{0}\" > {1}\n".format(append_line, path)
                bash_script.write(six.b(append_cmd))

            bash_script.flush()
            self._executor.execute_success(["bash", bash_script.name],
                                           requires_full_access=True)

    def install_packages(self, package_names):
        """Install all packages in list package_names."""
        _run_task(self._executor,
                  """Update repositories""",
                  ["apt-get", "update", "-qq", "-y", "--force-yes"])
        _run_task(self._executor,
                  """Install {0}""".format(str(package_names)),
                  ["apt-get",
                   "install",
                   "-qq",
{'fraction_non_alphanumeric': 0.10712516103458528, 'fraction_numerical': 0.00356753542760876, 'mean_word_length': 2.3617588274483676, 'pattern_counts': {'":': 8, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2215802', 'n_tokens_mistral': 2876, 'n_tokens_neox': 2701, 'n_words': 765}