<commit_before>#!/usr/bin/env python3

import logging
import re
import urllib
from collections import defaultdict

import click
import requests
from bs4 import BeautifulSoup
from cachecontrol import CacheControl
from prettytable import PrettyTable
from requests.packages.urllib3.exceptions import InsecureRequestWarning
from toolchest.rpm import rpmvercmp, utils

requests.packages.urllib3.disable_warnings(InsecureRequestWarning)


class DiffBuilds(object):

    def __fetch_container_directory_list(self, base_url, node):
        containers_url = ("{}/{}/var/log/extra/podman/"
                          "containers/".format(base_url, node))
        page = requests.get(containers_url).content
        return [page, containers_url]

    def __parse_container_directory_list(self, html_page):
        page = html_page[0]
        containers_url = html_page[1]
        soup = BeautifulSoup(page, 'html.parser')
        table = soup.find('table')
        try:
            rows = table.findAll('tr')
        except Exception:
            this_error = ("(A list of containers was not found "
                          "at: {})".format(containers_url))
            logging.error(this_error)
            logging.warning("Try using the -a option for a repoquery")
            raise Exception(this_error)
        containers = []
        for tr in rows:
            logging.debug(tr)
            if tr.find('a'):
                full_link = tr.find('a').get('href')
                dirty_link = re.match(r'^\.', full_link)
                index_link = re.match(r'.*\/index.html', full_link)
                if index_link:
                    # container_name/index.html
                    container_name = full_link[:-11]
                elif dirty_link:
                    # ./container_name/
                    container_name = full_link[2:-1]
                else:
                    # container_name/
                    container_name = full_link[:-1]
                is_alpha = re.match('[a-zA-Z]', container_name[0:4])
                if is_alpha:
                    containers.append(container_name)
        return containers

    def get_directory_list(self, base_url, node):
        """
        Gets the directory listing of the directory

        Get's all the href a links from beautifulsoup
        to create a list of containers. Remove any
        href a links that are not a real container dir.

        Parameters:
        url (string): base url passed
        node (string): the node type

        Returns:
        list: a list of ALL the directories in the containers dir
        """
        page = self.__fetch_container_directory_list(base_url, node)
        containers = self.__parse_container_directory_list(page)
        return containers

    def process_containers(self, cache, base_url, node, containers_list):
        """
        break this into two parts for better unit testing
        """
        containers_strings = self.process_containers_step1(cache,
                                                           base_url,
                                                           node,
                                                           containers_list)
        dict_of_containers = self.process_containers_step2(containers_strings)
        return dict_of_containers

    def process_containers_step1(self, cache, base_url, node, containers_list):
        """
        Pull the podman_info file and scrape out rpm info

        Get each podman_info file tries .txt and .txt.gz
        Process the string to return the rpms.

        Parameters:
        cache: (object) requests caching object
        base_url (string): base url passed
        node (string): the node type
        containers_list (list) list of container names

        Returns:
        dict_of_containers: (dict) keyname package, value = rpm version
        """
        dict_of_containers = {}
        if containers_list == []:
            logging.warning(
                "containers were not logged for: {}".format(base_url))
            return dict_of_containers
        containers_list.pop(0)
        for c in containers_list:
            logging.info("processing {} containers: {}".format(node, c))
            # for podman_info_log.txt, .gz, .txt.gz
            url = ("{}/{}/var/log/extra/podman/containers/{}/podman_info.log"
                   .format(base_url, node, c))
            req = cache.get(url, verify=False)
            if req.status_code == 200:
                container_info_temp = str(req.content.decode('UTF-8'))
            else:
                url = ("{}/{}/var/log/extra/podman/containers/{}"
                       "/podman_info.log.txt.gz".format(base_url, node, c))
                req = cache.get(url, verify=False)
                if req.status_code == 200:
                    container_info_temp = str(req.content.decode('UTF-8'))
                else:
                    url = ("{}/{}/var/log/extra/podman/containers/{}"
                           "/podman_info.log.gz".format(base_url, node, c))
                    req = cache.get(url, verify=False)
                    if req.status_code == 200:
                        container_info_temp = str(req.content.decode('UTF-8'))
                    else:
                        this_error = "request failed to fetch: {}".format(url)
                        raise Exception(this_error)

            if "Installed Packages" in container_info_temp:
                container_name = c[:-1]
                container_info_temp = container_info_temp.partition(
                    "Installed Packages")[2].split("\n")
                container_info_temp.pop(0)
                dict_of_containers[container_name] = container_info_temp
        return dict_of_containers

    def process_containers_step2(self, dict_of_containers):
        for container_name, value in dict_of_containers.items():
            container_info = []
            for counter, i in enumerate(value):
                # remove .x86_64
                try:
                    parts = i.split(" ")
                    # rpms can be logged to multiple lines
                    # the length of a full line should be three
                    is_alpha = re.match('[a-zA-Z]', parts[0])
                    if len(parts) > 1 and is_alpha:
                        nvr = "{}-{}".format(i.strip().split()[0][:-7],
                                             i.strip().split()[1])
                        container_info.append(nvr)
                    else:
                        # rpm info on more than one line
                        if len(parts) == 1 and is_alpha:
                            name = i[:-7]
                            version = value[counter + 1].strip().split(" ")[0]
                            nvr = "{}-{}".format(name, version)
                            container_info.append(nvr)
                except IndexError:
                    if str(i) != "":
                        logging.warning("Index Error found on\
                                        (ignore empty): " + str(i))
            dict_of_containers[container_name] = container_info
        return dict_of_containers

    def get_repoquery_logs(self, cache, base_url, node):
        """
        In cases where the container logs are not available, use repoquery

        Some CI jobs do not have the container logs.  A file in
        /var/log/extra/all_available_packages.txt should be present that
        is a repoquery of all enabled yum repos on the node.

        Parameters:
        cache (object): requests cache object
        base_url (string): base url passed
        node (string): the node type

        Returns:
        list: a list of rpms
        """
        url = "{}/{}/var/log/extra/all_available_packages.txt".format(
            base_url, node)
        list = cache.get(url, verify=False)
        if list.status_code != 200:
            url = "{}/{}/var/log/extra/all_available_packages.txt.gz".format(
                base_url, node)
            list = cache.get(url, verify=False)
            if list.status_code != 200:
                repoquery_not_avail = ("logs for base node {} were NOT "
                                       "available: {}".format(node, url))
                raise Exception(repoquery_not_avail)
        nice_list = list.content.decode('UTF-8').splitlines()
        return nice_list

    def json_extract(self, obj, key):
        """
        extract values from nested JSON.

        Parameters:
            obj (object): json object
            key (string): key from json converted to a dict.
        """
        arr = []

        def extract(obj, arr, key):
            """Recursively search for values of key in JSON tree."""
            if isinstance(obj, dict):
                for k, v in obj.items():
                    # print("key: {} value: {}".format(k, v))
                    if isinstance(v, (dict, list)):
                        extract(v, arr, key)
                    elif k == key:
                        arr.append(v)
            elif isinstance(obj, list):
                for item in obj:
                    extract(item, arr, key)
            return arr

        values = extract(obj, arr, key)
        return values

    def get_compose_logs(self, cache, base_url):
        """
        Go to the source and diff two compose builds

        Using the path to the compose as the baseurl
        e.g...  16.x/RHOS_foo/compose
        then tack on the build info
        metadata/rpms.json

        Parameters:
        cache (object): requests cache object
        base_url (string): base url passed

        Returns:
        list: a list of rpms
        """
        url = "{}/metadata/rpms.json".format(base_url)
        result = cache.get(url, verify=False)
        if result.status_code != 200:
            compose_not_avail = ("logs for the compose were NOT "
                                 "available: {}".format(url))
            raise Exception(compose_not_avail)
        return result.json()

    def parse_compose(self, json_result):
        result = json_result
        data = result['payload']['rpms']

        rpms_list_in_path = self.json_extract(data, "path")
        nice_list = []
        for i in rpms_list_in_path:
            nice_list.append(i.split("/")[-1])

        return nice_list

    def get_logs(self, cache, base_url, node):
        """
        Gets the rpm logs from the base_url

        base_url should be passed here
        the base node and container rpm lists will be parsed after
        e.g. https://storage.gra.cloud.ovh.net/v1/\
        AUTH_dcaab5e32b234d56b626f72581e3644c/zuul_opendev_logs_a7d/756888/2/\
        gate/tripleo-ci-centos-8-containers-multinode-train/a7d91e1/logs

        Parameters:
        cache (object): requests cachine object
        base_url (string): base url passed
        node (string): the node type

        Returns:
        list: a list of ALL the directories in the containers dir
        """

        # first test if we're using http(s) or local file
        test_url = urllib.parse.urlparse(base_url)
        if test_url.scheme == 'file':
            file_error = "file format is not supported, yet"
            raise Exception(file_error)
        else:
            url = "{}/{}/var/log/extra/rpm-list.txt".format(base_url, node)
            logging.info("processing host node {}".format(node))
            list = cache.get(url, verify=False)
            if list.status_code != 200:
                url = "{}/{}/var/log/extra/rpm-list.txt.gz".format(
                    base_url, node)
                list = cache.get(url, verify=False)
                if list.status_code != 200:
                    logs_not_avail = ("logs for base node {} were NOT "
                                      "available: {}".format(node, url))
                    raise Exception(logs_not_avail)
            nice_list = list.content.decode('UTF-8').splitlines()
        if nice_list:
            # check containers
            containers_list = self.get_directory_list(base_url, node)
            rpms_from_containers = self.process_containers(cache,<commit_msg>diff_builds overrides built-in name in local scope<commit_after>#!/usr/bin/env python3

import logging
import re
import urllib
from collections import defaultdict

import click
import requests
from bs4 import BeautifulSoup
from cachecontrol import CacheControl
from prettytable import PrettyTable
from requests.packages.urllib3.exceptions import InsecureRequestWarning
from toolchest.rpm import rpmvercmp, utils

requests.packages.urllib3.disable_warnings(InsecureRequestWarning)


class DiffBuilds(object):

    def __fetch_container_directory_list(self, base_url, node):
        containers_url = ("{}/{}/var/log/extra/podman/"
                          "containers/".format(base_url, node))
        page = requests.get(containers_url).content
        return [page, containers_url]

    def __parse_container_directory_list(self, html_page):
        page = html_page[0]
        containers_url = html_page[1]
        soup = BeautifulSoup(page, 'html.parser')
        table = soup.find('table')
        try:
            rows = table.findAll('tr')
        except Exception:
            this_error = ("(A list of containers was not found "
                          "at: {})".format(containers_url))
            logging.error(this_error)
            logging.warning("Try using the -a option for a repoquery")
            raise Exception(this_error)
        containers = []
        for tr in rows:
            logging.debug(tr)
            if tr.find('a'):
                full_link = tr.find('a').get('href')
                dirty_link = re.match(r'^\.', full_link)
                index_link = re.match(r'.*\/index.html', full_link)
                if index_link:
                    # container_name/index.html
                    container_name = full_link[:-11]
                elif dirty_link:
                    # ./container_name/
                    container_name = full_link[2:-1]
                else:
                    # container_name/
                    container_name = full_link[:-1]
                is_alpha = re.match('[a-zA-Z]', container_name[0:4])
                if is_alpha:
                    containers.append(container_name)
        return containers

    def get_directory_list(self, base_url, node):
        """
        Gets the directory listing of the directory

        Get's all the href a links from beautifulsoup
        to create a list of containers. Remove any
        href a links that are not a real container dir.

        Parameters:
        url (string): base url passed
        node (string): the node type

        Returns:
        list: a list of ALL the directories in the containers dir
        """
        page = self.__fetch_container_directory_list(base_url, node)
        containers = self.__parse_container_directory_list(page)
        return containers

    def process_containers(self, cache, base_url, node, containers_list):
        """
        break this into two parts for better unit testing
        """
        containers_strings = self.process_containers_step1(cache,
                                                           base_url,
                                                           node,
                                                           containers_list)
        dict_of_containers = self.process_containers_step2(containers_strings)
        return dict_of_containers

    def process_containers_step1(self, cache, base_url, node, containers_list):
        """
        Pull the podman_info file and scrape out rpm info

        Get each podman_info file tries .txt and .txt.gz
        Process the string to return the rpms.

        Parameters:
        cache: (object) requests caching object
        base_url (string): base url passed
        node (string): the node type
        containers_list (list) list of container names

        Returns:
        dict_of_containers: (dict) keyname package, value = rpm version
        """
        dict_of_containers = {}
        if containers_list == []:
            logging.warning(
                "containers were not logged for: {}".format(base_url))
            return dict_of_containers
        containers_list.pop(0)
        for c in containers_list:
            logging.info("processing {} containers: {}".format(node, c))
            # for podman_info_log.txt, .gz, .txt.gz
            url = ("{}/{}/var/log/extra/podman/containers/{}/podman_info.log"
                   .format(base_url, node, c))
            req = cache.get(url, verify=False)
            if req.status_code == 200:
                container_info_temp = str(req.content.decode('UTF-8'))
            else:
                url = ("{}/{}/var/log/extra/podman/containers/{}"
                       "/podman_info.log.txt.gz".format(base_url, node, c))
                req = cache.get(url, verify=False)
                if req.status_code == 200:
                    container_info_temp = str(req.content.decode('UTF-8'))
                else:
                    url = ("{}/{}/var/log/extra/podman/containers/{}"
                           "/podman_info.log.gz".format(base_url, node, c))
                    req = cache.get(url, verify=False)
                    if req.status_code == 200:
                        container_info_temp = str(req.content.decode('UTF-8'))
                    else:
                        this_error = "request failed to fetch: {}".format(url)
                        raise Exception(this_error)

            if "Installed Packages" in container_info_temp:
                container_name = c[:-1]
                container_info_temp = container_info_temp.partition(
                    "Installed Packages")[2].split("\n")
                container_info_temp.pop(0)
                dict_of_containers[container_name] = container_info_temp
        return dict_of_containers

    def process_containers_step2(self, dict_of_containers):
        for container_name, value in dict_of_containers.items():
            container_info = []
            for counter, i in enumerate(value):
                # remove .x86_64
                try:
                    parts = i.split(" ")
                    # rpms can be logged to multiple lines
                    # the length of a full line should be three
                    is_alpha = re.match('[a-zA-Z]', parts[0])
                    if len(parts) > 1 and is_alpha:
                        nvr = "{}-{}".format(i.strip().split()[0][:-7],
                                             i.strip().split()[1])
                        container_info.append(nvr)
                    else:
                        # rpm info on more than one line
                        if len(parts) == 1 and is_alpha:
                            name = i[:-7]
                            version = value[counter + 1].strip().split(" ")[0]
                            nvr = "{}-{}".format(name, version)
                            container_info.append(nvr)
                except IndexError:
                    if str(i) != "":
                        logging.warning("Index Error found on\
                                        (ignore empty): " + str(i))
            dict_of_containers[container_name] = container_info
        return dict_of_containers

    def get_repoquery_logs(self, cache, base_url, node):
        """
        In cases where the container logs are not available, use repoquery

        Some CI jobs do not have the container logs.  A file in
        /var/log/extra/all_available_packages.txt should be present that
        is a repoquery of all enabled yum repos on the node.

        Parameters:
        cache (object): requests cache object
        base_url (string): base url passed
        node (string): the node type

        Returns:
        list: a list of rpms
        """
        url = "{}/{}/var/log/extra/all_available_packages.txt".format(
            base_url, node)
        rpmlist = cache.get(url, verify=False)
        if rpmlist.status_code != 200:
            url = "{}/{}/var/log/extra/all_available_packages.txt.gz".format(
                base_url, node)
            rpmlist = cache.get(url, verify=False)
            if rpmlist.status_code != 200:
                repoquery_not_avail = ("logs for base node {} were NOT "
                                       "available: {}".format(node, url))
                raise Exception(repoquery_not_avail)
        nice_list = rpmlist.content.decode('UTF-8').splitlines()
        return nice_list

    def json_extract(self, obj, key):
        """
        extract values from nested JSON.

        Parameters:
            obj (object): json object
            key (string): key from json converted to a dict.
        """
        arr = []

        def extract(obj, arr, key):
            """Recursively search for values of key in JSON tree."""
            if isinstance(obj, dict):
                for k, v in obj.items():
                    # print("key: {} value: {}".format(k, v))
                    if isinstance(v, (dict, list)):
                        extract(v, arr, key)
                    elif k == key:
                        arr.append(v)
            elif isinstance(obj, list):
                for item in obj:
                    extract(item, arr, key)
            return arr

        values = extract(obj, arr, key)
        return values

    def get_compose_logs(self, cache, base_url):
        """
        Go to the source and diff two compose builds

        Using the path to the compose as the baseurl
        e.g...  16.x/RHOS_foo/compose
        then tack on the build info
        metadata/rpms.json

        Parameters:
        cache (object): requests cache object
        base_url (string): base url passed

        Returns:
        list: a list of rpms
        """
        url = "{}/metadata/rpms.json".format(base_url)
        result = cache.get(url, verify=False)
        if result.status_code != 200:
            compose_not_avail = ("logs for the compose were NOT "
                                 "available: {}".format(url))
            raise Exception(compose_not_avail)
        return result.json()

    def parse_compose(self, json_result):
        result = json_result
        data = result['payload']['rpms']

        rpms_list_in_path = self.json_extract(data, "path")
        nice_list = []
        for i in rpms_list_in_path:
            nice_list.append(i.split("/")[-1])

        return nice_list

    def get_logs(self, cache, base_url, node):
        """
        Gets the rpm logs from the base_url

        base_url should be passed here
        the base node and container rpm lists will be parsed after
        e.g. https://storage.gra.cloud.ovh.net/v1/\
        AUTH_dcaab5e32b234d56b626f72581e3644c/zuul_opendev_logs_a7d/756888/2/\
        gate/tripleo-ci-centos-8-containers-multinode-train/a7d91e1/logs

        Parameters:
        cache (object): requests cachine object
        base_url (string): base url passed
        node (string): the node type

        Returns:
        list: a list of ALL the directories in the containers dir
        """

        # first test if we're using http(s) or local file
        test_url = urllib.parse.urlparse(base_url)
        if test_url.scheme == 'file':
            file_error = "file format is not supported, yet"
            raise Exception(file_error)
        else:
            url = "{}/{}/var/log/extra/rpm-list.txt".format(base_url, node)
            logging.info("processing host node {}".format(node))
            rpmlist = cache.get(url, verify=False)
            if rpmlist.status_code != 200:
                url = "{}/{}/var/log/extra/rpm-list.txt.gz".format(
                    base_url, node)
                rpmlist = cache.get(url, verify=False)
                if rpmlist.status_code != 200:
                    logs_not_avail = ("logs for base node {} were NOT "
                                      "available: {}".format(node, url))
                    raise Exception(logs_not_avail)
            nice_list = rpmlist.content.decode('UTF-8').splitlines()
        if nice_list:
            # check containers
            containers_list = self.get_directory_list(base_url, node)
            rpms_from_containers = self.process_containers(cache,
{'fraction_non_alphanumeric': 0.09498583220401627, 'fraction_numerical': 0.008213215063036426, 'mean_word_length': 1.6360684130764234, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3025683', 'n_tokens_mistral': 6751, 'n_tokens_neox': 6410, 'n_words': 1711}