<commit_before>#!/usr/bin/env python
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import argparse
import logging
import multiprocessing
import subprocess
import sys
import yaml
import yum

log = logging.getLogger()
log.setLevel(logging.DEBUG)
ch = logging.StreamHandler(sys.stdout)
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s %(levelname)s: %(message)s')
ch.setFormatter(formatter)
log.addHandler(ch)


def parse_opts(argv):
    parser = argparse.ArgumentParser("Verify and possibly update rpms in "
                                     "containers")
    parser.add_argument('-c', '--containers', required=True,
                        help="YAML File containing a list of containers to "
                             "inspect.")
    parser.add_argument('-p', '--process-count',
                        help="Number of processes to use in the pool when "
                             "running docker containers.",
                        default=multiprocessing.cpu_count())
    parser.add_argument('-u', '--update',
                        action='store_true',
                        help="Run yum update in any containers that need "
                             "updating.",
                        default=False)
    parser.add_argument('-k', '--packages', nargs='*',
                        help="""List of packages to update, by default all""",
                        default=None)
    opts = parser.parse_args(argv[1:])
    return opts


def rm_container(name):
    log.info('Removing container: %s' % name)
    subproc = subprocess.Popen(['/usr/bin/docker', 'rm', name],
                               stdout=subprocess.PIPE,
                               stderr=subprocess.PIPE)
    cmd_stdout, cmd_stderr = subproc.communicate()
    if cmd_stdout:
        log.debug(cmd_stdout)
    if cmd_stderr and \
            cmd_stderr != 'Error response from daemon: ' \
            'No such container: {}\n'.format(name):
        log.debug(cmd_stderr)


def populate_container_rpms_list((container)):

    dcmd = ['/usr/bin/docker', 'run',
            '--user', 'root',
            '--rm',
            container]

    dcmd.extend(['rpm', '-qa'])

    log.info('Running docker command: %s' % ' '.join(dcmd))

    subproc = subprocess.Popen(dcmd, stdout=subprocess.PIPE,
                               stderr=subprocess.PIPE)
    cmd_stdout, cmd_stderr = subproc.communicate()
    if subproc.returncode != 0:
        log.error('Failed running rpm -qa for %s' % container)
        log.error(cmd_stderr)

    rpms = cmd_stdout.split("\n")

    return (subproc.returncode, container, rpms)


def yum_update_container((container, name, packages)):

    container_name = 'yum-update-%s' % name

    rm_container(container_name)

    # Get the command from the original so that when we commit
    # we can ensure the command doesn't get updated.
    dcmd = ['/usr/bin/docker', 'inspect',
            '--format', '{{json .Config.Cmd}}',
            container]
    subproc = subprocess.Popen(dcmd, stdout=subprocess.PIPE,
                               stderr=subprocess.PIPE)
    cmd_stdout, cmd_stderr = subproc.communicate()
    container_cmd = cmd_stdout
    log.info('Original container command: %s' % container_cmd)<commit_msg>Remove extra parenthesis from container-update.py<commit_after>#!/usr/bin/env python
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import argparse
import logging
import multiprocessing
import subprocess
import sys
import yaml
import yum

log = logging.getLogger()
log.setLevel(logging.DEBUG)
ch = logging.StreamHandler(sys.stdout)
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s %(levelname)s: %(message)s')
ch.setFormatter(formatter)
log.addHandler(ch)


def parse_opts(argv):
    parser = argparse.ArgumentParser("Verify and possibly update rpms in "
                                     "containers")
    parser.add_argument('-c', '--containers', required=True,
                        help="YAML File containing a list of containers to "
                             "inspect.")
    parser.add_argument('-p', '--process-count',
                        help="Number of processes to use in the pool when "
                             "running docker containers.",
                        default=multiprocessing.cpu_count())
    parser.add_argument('-u', '--update',
                        action='store_true',
                        help="Run yum update in any containers that need "
                             "updating.",
                        default=False)
    parser.add_argument('-k', '--packages', nargs='*',
                        help="""List of packages to update, by default all""",
                        default=None)
    opts = parser.parse_args(argv[1:])
    return opts


def rm_container(name):
    log.info('Removing container: %s' % name)
    subproc = subprocess.Popen(['/usr/bin/docker', 'rm', name],
                               stdout=subprocess.PIPE,
                               stderr=subprocess.PIPE)
    cmd_stdout, cmd_stderr = subproc.communicate()
    if cmd_stdout:
        log.debug(cmd_stdout)
    if cmd_stderr and \
            cmd_stderr != 'Error response from daemon: ' \
            'No such container: {}\n'.format(name):
        log.debug(cmd_stderr)


def populate_container_rpms_list(container):

    dcmd = ['/usr/bin/docker', 'run',
            '--user', 'root',
            '--rm',
            container]

    dcmd.extend(['rpm', '-qa'])

    log.info('Running docker command: %s' % ' '.join(dcmd))

    subproc = subprocess.Popen(dcmd, stdout=subprocess.PIPE,
                               stderr=subprocess.PIPE)
    cmd_stdout, cmd_stderr = subproc.communicate()
    if subproc.returncode != 0:
        log.error('Failed running rpm -qa for %s' % container)
        log.error(cmd_stderr)

    rpms = cmd_stdout.split("\n")

    return (subproc.returncode, container, rpms)


def yum_update_container(container, name, packages):

    container_name = 'yum-update-%s' % name

    rm_container(container_name)

    # Get the command from the original so that when we commit
    # we can ensure the command doesn't get updated.
    dcmd = ['/usr/bin/docker', 'inspect',
            '--format', '{{json .Config.Cmd}}',
            container]
    subproc = subprocess.Popen(dcmd, stdout=subprocess.PIPE,
                               stderr=subprocess.PIPE)
    cmd_stdout, cmd_stderr = subproc.communicate()
    container_cmd = cmd_stdout
    log.info('Original container command: %s' % container_cmd)
{'fraction_non_alphanumeric': 0.11087700816434026, 'fraction_numerical': 0.0015801948907031868, 'mean_word_length': 2.27511858559724, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2459864', 'n_tokens_mistral': 2170, 'n_tokens_neox': 2088, 'n_words': 585}