<commit_before>#!/usr/bin/env python2
# IBM_PROLOG_BEGIN_TAG
# This is an automatically generated prolog.
#
# $Source: op-test-framework/common/OpTestSystem.py $
#
# OpenPOWER Automated Test Project
#
# Contributors Listed Below - COPYRIGHT 2015,2017
# [+] International Business Machines Corp.
#
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied. See the License for the specific language governing
# permissions and limitations under the License.
#
# IBM_PROLOG_END_TAG

## @package OpTestSystem
#  System package for OpenPower testing.
#
#  This class encapsulates all interfaces and classes required to do end to end
#  automated flashing and testing of OpenPower systems.

import time
import subprocess
import pexpect
import socket
import commands
import unittest

import OpTestIPMI # circular dependencies, use package
import OpTestQemu
from OpTestFSP import OpTestFSP
from OpTestConstants import OpTestConstants as BMC_CONST
from OpTestError import OpTestError
from OpTestHost import OpTestHost
from OpTestUtil import OpTestUtil
from OpTestSSH import ConsoleState as SSHConnectionState
from Exceptions import HostbootShutdown, WaitForIt, RecoverFailed, UnknownStateTransition, ConsoleSettings, UnexpectedCase, StoppingSystem
from OpTestSSH import OpTestSSH

import logging
import OpTestLogger
log = OpTestLogger.optest_logger_glob.get_logger(__name__)

class OpSystemState():
    '''
    This class is used as an enum as to what state op-test *thinks* the host is in.
    These states are used to drive a state machine in OpTestSystem.
    '''
    UNKNOWN = 0
    OFF = 1
    IPLing = 2
    PETITBOOT = 3
    PETITBOOT_SHELL = 4
    BOOTING = 5
    OS = 6
    POWERING_OFF = 7
    UNKNOWN_BAD = 8 # special case, use set_state to place system in hold for later goto

class OpTestSystem(object):

    ## Initialize this object
    #  @param i_bmcIP The IP address of the BMC
    #  @param i_bmcUser The userid to log into the BMC with
    #  @param i_bmcPasswd The password of the userid to log into the BMC with
    #  @param i_bmcUserIpmi The userid to issue the BMC IPMI commands with
    #  @param i_bmcPasswdIpmi The password of BMC IPMI userid
    #
    # "Only required for inband tests" else Default = None
    # @param i_hostIP The IP address of the Host
    # @param i_hostuser The userid to log into the Host
    # @param i_hostPasswd The password of the userid to log into the host with
    #

    def __init__(self,
                 bmc=None, host=None,
                 prompt=None,
                 state=OpSystemState.UNKNOWN):
        self.bmc = self.cv_BMC = bmc
        self.cv_HOST = host
        self.cv_IPMI = bmc.get_ipmi()
        self.rest = self.bmc.get_rest_api()
        self.console = self.bmc.get_host_console()
        self.util = OpTestUtil()
        self.prompt = prompt # build_prompt located in OpTestUtil
        # system console state tracking, reset on boot and state changes, set when valid
        self.PS1_set = -1
        self.SUDO_set = -1
        self.LOGIN_set = -1
        self.expect_prompt = self.util.build_prompt(prompt) + "$"
        self.previous_state = None # used for PS1, LOGIN, SUDO state tracking
        self.target_state = None # used in WaitForIt
        self.detect_counter = 0 # outside scope of detection to prevent loops
        self.never_rebooted = True # outside scope to prevent loops
        self.block_setup_term = 0
        self.stop = 0
        self.ignore = 0

        self.openpower = 'openpower' # string to define petitboot kernel cat /proc/version column 3, change if using debug petitboot kernel

        # dictionary used in sorted order
        # column 1 is the string, column 2 is the action
        # normally None is the action, otherwise a handler mostly used for exceptions
        self.petitboot_expect_table = {
          'Petitboot'                              : None,
          '/ #'                                    : None,
          'shutdown requested'                     : self.hostboot_callback,
          'x=exit'                                 : None,
          'login: '                                : self.login_callback,
          'mon> '                                  : self.xmon_callback,
          'System shutting down with error status' : self.guard_callback,
        }

        self.login_expect_table = {
          'login: '                           : None,
          '/ #'                               : self.petitboot_callback,
          'mon> '                             : self.xmon_callback,
        }

        # tunables for customizations, put them here all together

        # ipmi versus ssh settings, sometimes tuning is needed based on type, so keeping split for tuning
        # to basically turn off reconnect based on stale buffers set threshold equal to watermark, e.g. 100
        if isinstance(self.console, OpTestIPMI.IPMIConsole):
          self.threshold_petitboot = 12 # stale buffer check
          self.threshold_login = 12 # long enough to skip the refresh until kexec, stale buffers need to be jumped over
          self.petitboot_kicker = 0
          self.petitboot_refresh = 0 # petitboot menu cannot tolerate, cancels default boot
          self.petitboot_reconnect = 1
          self.login_refresh = 0
          self.login_reconnect = 1 # less reliable connections, ipmi act/deact does not trigger default boot cancel
          self.login_fresh_start = 0
        else:
          self.threshold_petitboot = 12 # stale buffer check
          self.threshold_login = 12 # long enough to skip the refresh until kexec, stale buffers need to be jumped over
          self.petitboot_kicker = 0
          self.petitboot_refresh = 0 # petitboot menu cannot tolerate, cancels default boot
          self.petitboot_reconnect = 1 # NEW ssh triggers default boot cancel, just saying
          self.login_refresh = 0
          self.login_reconnect = 1 # NEW ssh triggers default boot cancel, just saying
          self.login_fresh_start = 0

        # watermark is the loop counter (loop_max) used in conjunction with timeout
        # timeout is the expect timeout for each iteration
        # watermark will automatically increase in case the loop is too short
        self.ipl_watermark = 100
        self.ipl_timeout = 4 # needs consideration with petitboot timeout
        self.booting_watermark = 100
        self.booting_timeout = 5
        self.kill_cord = 150 # just a ceiling on giving up

        # We have a state machine for going in between states of the system
        # initially, everything in UNKNOWN, so we reset things.
        # UNKNOWN is used to flag the system to auto-detect the state if
        # possible to efficiently achieve state transitions.
        # But, we allow setting an initial state if you, say, need to
        # run against an already IPLed system
        self.state = state
        self.stateHandlers = {}
        self.stateHandlers[OpSystemState.UNKNOWN] = self.run_UNKNOWN
        self.stateHandlers[OpSystemState.OFF] = self.run_OFF
        self.stateHandlers[OpSystemState.IPLing] = self.run_IPLing
        self.stateHandlers[OpSystemState.PETITBOOT] = self.run_PETITBOOT
        self.stateHandlers[OpSystemState.PETITBOOT_SHELL] = self.run_PETITBOOT_SHELL
        self.stateHandlers[OpSystemState.BOOTING] = self.run_BOOTING
        self.stateHandlers[OpSystemState.OS] = self.run_OS
        self.stateHandlers[OpSystemState.POWERING_OFF] = self.run_POWERING_OFF
        self.stateHandlers[OpSystemState.UNKNOWN_BAD] = self.run_UNKNOWN

        # We track the state of loaded IPMI modules here, that way
        # we only need to try the modprobe once per IPL.
        # We reset as soon as we transition away from OpSystemState.OS
        # a TODO is to support doing this in petitboot shell as well.
        self.ipmiDriversLoaded = False

    def hostboot_callback(self, **kwargs):
        default_vals = {'my_r': None, 'value': None}
        for key in default_vals:
          if key not in kwargs.keys():
            kwargs[key] = default_vals[key]
        self.state = OpSystemState.UNKNOWN_BAD
        self.stop = 1
        raise HostbootShutdown()

    def login_callback(self, **kwargs):
        default_vals = {'my_r': None, 'value': None}
        for key in default_vals:
          if key not in kwargs.keys():
            kwargs[key] = default_vals[key]
        log.warning("\n\n *** OpTestSystem found the login prompt \"{}\" but this is unexpected, we will retry\n\n".format(kwargs['value']))
        # raise the WaitForIt exception to be bubbled back to recycle early rather than having to wait the full loop_max
        raise WaitForIt(expect_dict=self.petitboot_expect_table, reconnect_count=-1)

    def petitboot_callback(self, **kwargs):
        default_vals = {'my_r': None, 'value': None}
        for key in default_vals:
          if key not in kwargs.keys():
            kwargs[key] = default_vals[key]
        log.warning("\n\n *** OpTestSystem found the petitboot prompt \"{}\" but this is unexpected, we will retry\n\n".format(kwargs['value']))
        # raise the WaitForIt exception to be bubbled back to recycle early rather than having to wait the full loop_max
        raise WaitForIt(expect_dict=self.login_expect_table, reconnect_count=-1)

    def guard_callback(self, **kwargs):
        default_vals = {'my_r': None, 'value': None}
        for key in default_vals:
          if key not in kwargs.keys():
            kwargs[key] = default_vals[key]
        guard_exception = UnexpectedCase(state=self.state, msg="We hit the guard_callback value={}, manually restart the system".format(kwargs['value']))
        self.state = OpSystemState.UNKNOWN_BAD
        self.stop = 1
        raise guard_exception

    def xmon_callback(self, **kwargs):
        default_vals = {'my_r': None, 'value': None}
        for key in default_vals:
          if key not in kwargs.keys():
            kwargs[key] = default_vals[key]
        xmon_check_r = kwargs['my_r']
        xmon_value = kwargs['value']
        time.sleep(2)
        sys_console = self.console.get_console()
        time.sleep(2)
        sys_console.sendline("t")
        time.sleep(2)
        rc = sys_console.expect([".*mon> ", pexpect.TIMEOUT, pexpect.EOF], timeout=10)
        xmon_backtrace = sys_console.after
        sys_console.sendline("r")
        time.sleep(2)
        rc = sys_console.expect([".*mon> ", pexpect.TIMEOUT, pexpect.EOF], timeout=10)
        xmon_registers = sys_console.after
        sys_console.sendline("S")
        time.sleep(2)
        rc = sys_console.expect([".*mon> ", pexpect.TIMEOUT, pexpect.EOF], timeout=10)
        xmon_special_registers = sys_console.after
        self.stop = 1
        my_msg = ('We hit the xmon_callback with \"{}\" backtrace=\n{}\n registers=\n{}\n special_registers=\n{}\n'
                  .format(xmon_value, xmon_backtrace, xmon_registers, xmon_special_registers))
        xmon_exception = UnexpectedCase(state=self.state, msg=my_msg)
        self.state = OpSystemState.UNKNOWN_BAD
        raise xmon_exception

    def skiboot_log_on_console(self):
        return True

    def has_host_accessible_eeprom(self):
        return True

    def has_host_led_support(self):
        return False

    def has_centaurs_in_dt(self):
        proc_gen = self.host().host_get_proc_gen()
        if proc_gen in ["POWER9"]:
            return False
        return True

    def has_mtd_pnor_access(self):
        return True

    def host(self):
        return self.cv_HOST

    def bmc(self):
        return self.cv_BMC

    def rest(self):
        return self.rest

    def ipmi(self):
        return self.cv_IPMI<commit_msg>OpTestSystem: Stop IPL when skiboot abort detected<commit_after>#!/usr/bin/env python2
# IBM_PROLOG_BEGIN_TAG
# This is an automatically generated prolog.
#
# $Source: op-test-framework/common/OpTestSystem.py $
#
# OpenPOWER Automated Test Project
#
# Contributors Listed Below - COPYRIGHT 2015,2017
# [+] International Business Machines Corp.
#
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied. See the License for the specific language governing
# permissions and limitations under the License.
#
# IBM_PROLOG_END_TAG

## @package OpTestSystem
#  System package for OpenPower testing.
#
#  This class encapsulates all interfaces and classes required to do end to end
#  automated flashing and testing of OpenPower systems.

import time
import subprocess
import pexpect
import socket
import commands
import unittest

import OpTestIPMI # circular dependencies, use package
import OpTestQemu
from OpTestFSP import OpTestFSP
from OpTestConstants import OpTestConstants as BMC_CONST
from OpTestError import OpTestError
from OpTestHost import OpTestHost
from OpTestUtil import OpTestUtil
from OpTestSSH import ConsoleState as SSHConnectionState
from Exceptions import HostbootShutdown, WaitForIt, RecoverFailed, UnknownStateTransition, ConsoleSettings, UnexpectedCase, StoppingSystem
from OpTestSSH import OpTestSSH

import logging
import OpTestLogger
log = OpTestLogger.optest_logger_glob.get_logger(__name__)

class OpSystemState():
    '''
    This class is used as an enum as to what state op-test *thinks* the host is in.
    These states are used to drive a state machine in OpTestSystem.
    '''
    UNKNOWN = 0
    OFF = 1
    IPLing = 2
    PETITBOOT = 3
    PETITBOOT_SHELL = 4
    BOOTING = 5
    OS = 6
    POWERING_OFF = 7
    UNKNOWN_BAD = 8 # special case, use set_state to place system in hold for later goto

class OpTestSystem(object):

    ## Initialize this object
    #  @param i_bmcIP The IP address of the BMC
    #  @param i_bmcUser The userid to log into the BMC with
    #  @param i_bmcPasswd The password of the userid to log into the BMC with
    #  @param i_bmcUserIpmi The userid to issue the BMC IPMI commands with
    #  @param i_bmcPasswdIpmi The password of BMC IPMI userid
    #
    # "Only required for inband tests" else Default = None
    # @param i_hostIP The IP address of the Host
    # @param i_hostuser The userid to log into the Host
    # @param i_hostPasswd The password of the userid to log into the host with
    #

    def __init__(self,
                 bmc=None, host=None,
                 prompt=None,
                 state=OpSystemState.UNKNOWN):
        self.bmc = self.cv_BMC = bmc
        self.cv_HOST = host
        self.cv_IPMI = bmc.get_ipmi()
        self.rest = self.bmc.get_rest_api()
        self.console = self.bmc.get_host_console()
        self.util = OpTestUtil()
        self.prompt = prompt # build_prompt located in OpTestUtil
        # system console state tracking, reset on boot and state changes, set when valid
        self.PS1_set = -1
        self.SUDO_set = -1
        self.LOGIN_set = -1
        self.expect_prompt = self.util.build_prompt(prompt) + "$"
        self.previous_state = None # used for PS1, LOGIN, SUDO state tracking
        self.target_state = None # used in WaitForIt
        self.detect_counter = 0 # outside scope of detection to prevent loops
        self.never_rebooted = True # outside scope to prevent loops
        self.block_setup_term = 0
        self.stop = 0
        self.ignore = 0

        self.openpower = 'openpower' # string to define petitboot kernel cat /proc/version column 3, change if using debug petitboot kernel

        # dictionary used in sorted order
        # column 1 is the string, column 2 is the action
        # normally None is the action, otherwise a handler mostly used for exceptions
        self.petitboot_expect_table = {
          'Petitboot'                              : None,
          '/ #'                                    : None,
          'shutdown requested'                     : self.hostboot_callback,
          'x=exit'                                 : None,
          'login: '                                : self.login_callback,
          'mon> '                                  : self.xmon_callback,
          'System shutting down with error status' : self.guard_callback,
          'Aborting!'                              : self.skiboot_callback,
        }

        self.login_expect_table = {
          'login: '                           : None,
          '/ #'                               : self.petitboot_callback,
          'mon> '                             : self.xmon_callback,
        }

        # tunables for customizations, put them here all together

        # ipmi versus ssh settings, sometimes tuning is needed based on type, so keeping split for tuning
        # to basically turn off reconnect based on stale buffers set threshold equal to watermark, e.g. 100
        if isinstance(self.console, OpTestIPMI.IPMIConsole):
          self.threshold_petitboot = 12 # stale buffer check
          self.threshold_login = 12 # long enough to skip the refresh until kexec, stale buffers need to be jumped over
          self.petitboot_kicker = 0
          self.petitboot_refresh = 0 # petitboot menu cannot tolerate, cancels default boot
          self.petitboot_reconnect = 1
          self.login_refresh = 0
          self.login_reconnect = 1 # less reliable connections, ipmi act/deact does not trigger default boot cancel
          self.login_fresh_start = 0
        else:
          self.threshold_petitboot = 12 # stale buffer check
          self.threshold_login = 12 # long enough to skip the refresh until kexec, stale buffers need to be jumped over
          self.petitboot_kicker = 0
          self.petitboot_refresh = 0 # petitboot menu cannot tolerate, cancels default boot
          self.petitboot_reconnect = 1 # NEW ssh triggers default boot cancel, just saying
          self.login_refresh = 0
          self.login_reconnect = 1 # NEW ssh triggers default boot cancel, just saying
          self.login_fresh_start = 0

        # watermark is the loop counter (loop_max) used in conjunction with timeout
        # timeout is the expect timeout for each iteration
        # watermark will automatically increase in case the loop is too short
        self.ipl_watermark = 100
        self.ipl_timeout = 4 # needs consideration with petitboot timeout
        self.booting_watermark = 100
        self.booting_timeout = 5
        self.kill_cord = 150 # just a ceiling on giving up

        # We have a state machine for going in between states of the system
        # initially, everything in UNKNOWN, so we reset things.
        # UNKNOWN is used to flag the system to auto-detect the state if
        # possible to efficiently achieve state transitions.
        # But, we allow setting an initial state if you, say, need to
        # run against an already IPLed system
        self.state = state
        self.stateHandlers = {}
        self.stateHandlers[OpSystemState.UNKNOWN] = self.run_UNKNOWN
        self.stateHandlers[OpSystemState.OFF] = self.run_OFF
        self.stateHandlers[OpSystemState.IPLing] = self.run_IPLing
        self.stateHandlers[OpSystemState.PETITBOOT] = self.run_PETITBOOT
        self.stateHandlers[OpSystemState.PETITBOOT_SHELL] = self.run_PETITBOOT_SHELL
        self.stateHandlers[OpSystemState.BOOTING] = self.run_BOOTING
        self.stateHandlers[OpSystemState.OS] = self.run_OS
        self.stateHandlers[OpSystemState.POWERING_OFF] = self.run_POWERING_OFF
        self.stateHandlers[OpSystemState.UNKNOWN_BAD] = self.run_UNKNOWN

        # We track the state of loaded IPMI modules here, that way
        # we only need to try the modprobe once per IPL.
        # We reset as soon as we transition away from OpSystemState.OS
        # a TODO is to support doing this in petitboot shell as well.
        self.ipmiDriversLoaded = False

    def hostboot_callback(self, **kwargs):
        default_vals = {'my_r': None, 'value': None}
        for key in default_vals:
          if key not in kwargs.keys():
            kwargs[key] = default_vals[key]
        self.state = OpSystemState.UNKNOWN_BAD
        self.stop = 1
        raise HostbootShutdown()

    def login_callback(self, **kwargs):
        default_vals = {'my_r': None, 'value': None}
        for key in default_vals:
          if key not in kwargs.keys():
            kwargs[key] = default_vals[key]
        log.warning("\n\n *** OpTestSystem found the login prompt \"{}\" but this is unexpected, we will retry\n\n".format(kwargs['value']))
        # raise the WaitForIt exception to be bubbled back to recycle early rather than having to wait the full loop_max
        raise WaitForIt(expect_dict=self.petitboot_expect_table, reconnect_count=-1)

    def petitboot_callback(self, **kwargs):
        default_vals = {'my_r': None, 'value': None}
        for key in default_vals:
          if key not in kwargs.keys():
            kwargs[key] = default_vals[key]
        log.warning("\n\n *** OpTestSystem found the petitboot prompt \"{}\" but this is unexpected, we will retry\n\n".format(kwargs['value']))
        # raise the WaitForIt exception to be bubbled back to recycle early rather than having to wait the full loop_max
        raise WaitForIt(expect_dict=self.login_expect_table, reconnect_count=-1)

    def guard_callback(self, **kwargs):
        default_vals = {'my_r': None, 'value': None}
        for key in default_vals:
          if key not in kwargs.keys():
            kwargs[key] = default_vals[key]
        guard_exception = UnexpectedCase(state=self.state, msg="We hit the guard_callback value={}, manually restart the system".format(kwargs['value']))
        self.state = OpSystemState.UNKNOWN_BAD
        self.stop = 1
        raise guard_exception

    def xmon_callback(self, **kwargs):
        default_vals = {'my_r': None, 'value': None}
        for key in default_vals:
          if key not in kwargs.keys():
            kwargs[key] = default_vals[key]
        xmon_check_r = kwargs['my_r']
        xmon_value = kwargs['value']
        time.sleep(2)
        sys_console = self.console.get_console()
        time.sleep(2)
        sys_console.sendline("t")
        time.sleep(2)
        rc = sys_console.expect([".*mon> ", pexpect.TIMEOUT, pexpect.EOF], timeout=10)
        xmon_backtrace = sys_console.after
        sys_console.sendline("r")
        time.sleep(2)
        rc = sys_console.expect([".*mon> ", pexpect.TIMEOUT, pexpect.EOF], timeout=10)
        xmon_registers = sys_console.after
        sys_console.sendline("S")
        time.sleep(2)
        rc = sys_console.expect([".*mon> ", pexpect.TIMEOUT, pexpect.EOF], timeout=10)
        xmon_special_registers = sys_console.after
        self.stop = 1
        my_msg = ('We hit the xmon_callback with \"{}\" backtrace=\n{}\n registers=\n{}\n special_registers=\n{}\n'
                  .format(xmon_value, xmon_backtrace, xmon_registers, xmon_special_registers))
        xmon_exception = UnexpectedCase(state=self.state, msg=my_msg)
        self.state = OpSystemState.UNKNOWN_BAD
        raise xmon_exception

    def skiboot_callback(self, **kwargs):
        default_vals = {'my_r': None, 'value': None}
        for key in default_vals:
          if key not in kwargs.keys():
            kwargs[key] = default_vals[key]
        skiboot_exception = UnexpectedCase(state=self.state, msg="We hit the skiboot_callback value={}, manually restart the system".format(kwargs['value']))
        self.state = OpSystemState.UNKNOWN_BAD
        self.stop = 1
        raise skiboot_exception

    def skiboot_log_on_console(self):
        return True

    def has_host_accessible_eeprom(self):
        return True

    def has_host_led_support(self):
        return False

    def has_centaurs_in_dt(self):
        proc_gen = self.host().host_get_proc_gen()
        if proc_gen in ["POWER9"]:
            return False
        return True

    def has_mtd_pnor_access(self):
        return True

    def host(self):
        return self.cv_HOST

    def bmc(self):
        return self.cv_BMC

    def rest(self):
        return self.rest

    def ipmi(self):
        return self.cv_IPMI
{'fraction_non_alphanumeric': 0.07448532987518236, 'fraction_numerical': 0.006929810342032745, 'mean_word_length': 2.53336197021764, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '199078', 'n_tokens_mistral': 7175, 'n_tokens_neox': 6965, 'n_words': 2285}