<commit_before>*** Settings ***
Documentation     This is a library for simple improvements over SSHLibrary for other robot libraries to use.
Library           OperatingSystem
Library           String
Library           SSHLibrary

*** Keywords ***
Open_Ssh_Connection
    [Arguments]    ${name}    ${ip}    ${user}    ${pswd}
    [Documentation]    Create SSH connection to \{ip} aliased as \${name} and log in using \${user} and \${pswd} (or rsa).
    ...    Log to output file. The new connection is left active.
    BuiltIn.Log_Many    ${name}    ${ip}    ${user}    ${pswd}
    SSHLibrary.Open_Connection    ${ip}    alias=${name}    timeout=${SSH_TIMEOUT}
    ${out} =    BuiltIn.Run_Keyword_If    """${pswd}""" != "rsa_id"    SSHLibrary.Login    ${user}    ${pswd}
    ${out2} =    BuiltIn.Run_Keyword_If    """${pswd}""" == "rsa_id"    SSHLibrary.Login_With_Public_Key    ${user}    %{HOME}/.ssh/id_rsa    any
    BuiltIn.Run_Keyword_If    """${out}""" != "None"    OperatingSystem.Append_To_File    ${RESULTS_FOLDER}/output_${name}.log    *** Command: Login${\n}${out}${\n}
    BuiltIn.Run_Keyword_If    """${out2}""" != "None"    OperatingSystem.Append_To_File    ${RESULTS_FOLDER}/output_${name}.log    *** Command: Login${\n}${out2}${\n}

Switch_And_Execute_With_Copied_File
    [Arguments]    ${ssh_session}    ${file_path}    ${command_prefix}    ${expected_rc}=0    ${ignore_stderr}=${False}
    [Documentation]    Switch to \${ssh_session} and continue with Execute_Command_With_Copied_File.
    BuiltIn.Log_Many    ${ssh_session}    ${file_path}    ${command_prefix}    ${expected_rc}    ${ignore_stderr}
    SSHLibrary.Switch_Connection    ${ssh_session}
    BuiltIn.Run_Keyword_And_Return    Execute_Command_With_Copied_File    ${file_path}    ${command_prefix}    expected_rc=${expected_rc}    ignore_stderr=${ignore_stderr}

Execute_Command_With_Copied_File
    [Arguments]    ${file_path}    ${command_prefix}    ${expected_rc}=0    ${ignore_stderr}=${False}
    [Documentation]    Put file to current remote directory and execute command which takes computed file name as argument.
    BuiltIn.Log_Many    ${file_path}    ${command_prefix}    ${expected_rc}    ${ignore_stderr}
    Builtin.Comment    TODO: Do not pollute current remote directory. See https://github.com/contiv/vpp/issues/195
    SSHLibrary.Put_File    ${file_path}    .
    ${splitted_path} =    String.Split_String    ${file_path}    separator=${/}
    BuiltIn.Run_Keyword_And_Return    Execute_Command_And_Log    ${command_prefix} @{splitted_path}[-1]    expected_rc=${expected_rc}    ignore_stderr=${ignore_stderr}

Switch_Execute_And_Log_To_File
    [Arguments]    ${ssh_session}    ${command}    ${expected_rc}=0    ${ignore_stderr}=${False}    ${ignore_rc}=${False}    ${compress}=${False}
    [Documentation]    Call Switch_And_Execute_Command redirecting stdout to a remote file, download the file.
    ...    To distinguish separate invocations, suite name, test name, session alias
    ...    and full command are used to construct file name.
    BuiltIn.Log_Many    ${ssh_session}    ${command}    ${expected_rc}    ${ignore_stderr}    ${ignore_rc}    ${compress}
    SSHLibrary.Switch_Connection    ${ssh_session}
    ${connection} =    SSHLibrary.Get_Connection
    # In teardown, ${TEST_NAME} does not exist.
    ${testname} =    BuiltIn.Get_Variable_Value    ${TEST_NAME}    ${EMPTY}
    ${filename_with_spaces} =    BuiltIn.Set_Variable    ${testname}__${SUITE_NAME}__${connection.alias}__${command}.log
    ${filename} =    String.Replace_String    ${filename_with_spaces}    ${SPACE}    _
    BuiltIn.Log    ${filename}
    Execute_Command_And_Log    ${command} > ${filename}    expected_rc=${expected_rc}    ignore_stderr=${ignore_stderr}    ignore_rc=${ignore_rc}
    BuiltIn.Run_Keyword_If    ${compress}    Execute_Command_And_Log    xz -9e ${filename}
    ${filename} =    Builtin.Set_Variable_If    ${compress}    ${filename}.xz    ${filename}
    SSHLibrary.Get_File    ${filename}    ${RESULTS_FOLDER}/${filename}
    [Teardown]    Execute_Command_And_Log    rm ${filename}

Switch_And_Execute_Command
    [Arguments]    ${ssh_session}    ${command}    ${expected_rc}=0    ${ignore_stderr}=${False}    ${ignore_rc}=${False}
    [Documentation]    Switch to \${ssh_session}, and continue with Execute_Command_And_Log.
    BuiltIn.Log_Many    ${ssh_session}    ${command}    ${expected_rc}    ${ignore_stderr}    ${ignore_rc}
    SSHLibrary.Switch_Connection    ${ssh_session}
    BuiltIn.Run_Keyword_And_Return    Execute_Command_And_Log    ${command}    expected_rc=${expected_rc}    ignore_stderr=${ignore_stderr}    ignore_rc=${ignore_rc}

Execute_Command_And_Log
    [Arguments]    ${command}    ${expected_rc}=0    ${ignore_stderr}=${False}    ${ignore_rc}=${False}
    [Documentation]    Execute \${command} on current SSH session, log results, maybe fail on nonempty stderr, check \${expected_rc}, return stdout.
    BuiltIn.Log_Many    ${command}    ${expected_rc}    ${ignore_stderr}    ${ignore_rc}
    ${stdout}    ${stderr}    ${rc} =    SSHLibrary.Execute_Command    ${command}    return_stderr=True    return_rc=True
    Append_Command_Log    ${command}    ${stdout}    ${stderr}    ${rc}
    BuiltIn.Run_Keyword_Unless    ${ignore_stderr}    BuiltIn.Should_Be_Empty    ${stderr}
    BuiltIn.Run_Keyword_Unless    ${ignore_rc}    BuiltIn.Should_Be_Equal_As_Numbers    ${rc}    ${expected_rc}
    [Return]    ${stdout}

Switch_And_Write_Command
    [Arguments]    ${ssh_session}    ${command}    ${prompt}=vpp#
    [Documentation]    Switch to \${ssh_session}, and continue with Write_Command_And_Log
    BuiltIn.Log_Many    ${ssh_session}    ${command}    ${prompt}
    SSHLibrary.Switch_Connection    ${ssh_session}
    BuiltIn.Run_Keyword_And_Return    Write_Command_And_Log    ${command}    ${prompt}

Write_Command_And_Log
    [Arguments]    ${command}    ${prompt}=vpp#
    [Documentation]    Write \${command} on current SSH session, wait for prompt, log output, return output.
    BuiltIn.Log_Many    ${command}    ${prompt}
    SSHLibrary.Write    ${command}
    ${output} =    SSHLibrary.Read_Until    ${prompt}
    Append_Command_Log    ${command}    ${output}
    [Return]    ${output}

Append_Command_Log
    [Arguments]    ${command}    ${output}=${EMPTY}    ${stderr}=${EMPTY}    ${rc}=${EMPTY}
    [Documentation]    Detect connection alias and time, append line with command and output to appropriate log file.
    Builtin.Log_Many    ${command}    ${output}    ${stderr}    ${rc}
    ${connection} =    SSHLibrary.Get_Connection
    ${time} =    DateTime.Get_Current_Date
    ${if_output} =    BuiltIn.Set_Variable_If    """${output}"""    ${output}${\n}    ${EMPTY}
    ${if_stderr} =    BuiltIn.Set_Variable_If    """${stderr}"""    *** Stderr: ${stderr}${\n}    ${EMPTY}
    ${if_rc} =    BuiltIn.Set_Variable_If    """${rc}"""    *** Return code: ${rc}${\n}    ${EMPTY}
    OperatingSystem.Append_To_File    ${RESULTS_FOLDER}/output_${connection.alias}.log    ${time}${\n}*** Command: ${command}${\n}${if_stderr}${if_rc}${if_output}<commit_msg>Do not python-parse output strings<commit_after>*** Settings ***
Documentation     This is a library for simple improvements over SSHLibrary for other robot libraries to use.
Library           OperatingSystem
Library           String
Library           SSHLibrary

*** Keywords ***
Open_Ssh_Connection
    [Arguments]    ${name}    ${ip}    ${user}    ${pswd}
    [Documentation]    Create SSH connection to \{ip} aliased as \${name} and log in using \${user} and \${pswd} (or rsa).
    ...    Log to output file. The new connection is left active.
    BuiltIn.Log_Many    ${name}    ${ip}    ${user}    ${pswd}
    SSHLibrary.Open_Connection    ${ip}    alias=${name}    timeout=${SSH_TIMEOUT}
    ${out} =    BuiltIn.Run_Keyword_If    """${pswd}""" != "rsa_id"    SSHLibrary.Login    ${user}    ${pswd}
    ${out2} =    BuiltIn.Run_Keyword_If    """${pswd}""" == "rsa_id"    SSHLibrary.Login_With_Public_Key    ${user}    %{HOME}/.ssh/id_rsa    any
    BuiltIn.Run_Keyword_If    """${out}""" != "None"    OperatingSystem.Append_To_File    ${RESULTS_FOLDER}/output_${name}.log    *** Command: Login${\n}${out}${\n}
    BuiltIn.Run_Keyword_If    """${out2}""" != "None"    OperatingSystem.Append_To_File    ${RESULTS_FOLDER}/output_${name}.log    *** Command: Login${\n}${out2}${\n}

Switch_And_Execute_With_Copied_File
    [Arguments]    ${ssh_session}    ${file_path}    ${command_prefix}    ${expected_rc}=0    ${ignore_stderr}=${False}
    [Documentation]    Switch to \${ssh_session} and continue with Execute_Command_With_Copied_File.
    BuiltIn.Log_Many    ${ssh_session}    ${file_path}    ${command_prefix}    ${expected_rc}    ${ignore_stderr}
    SSHLibrary.Switch_Connection    ${ssh_session}
    BuiltIn.Run_Keyword_And_Return    Execute_Command_With_Copied_File    ${file_path}    ${command_prefix}    expected_rc=${expected_rc}    ignore_stderr=${ignore_stderr}

Execute_Command_With_Copied_File
    [Arguments]    ${file_path}    ${command_prefix}    ${expected_rc}=0    ${ignore_stderr}=${False}
    [Documentation]    Put file to current remote directory and execute command which takes computed file name as argument.
    BuiltIn.Log_Many    ${file_path}    ${command_prefix}    ${expected_rc}    ${ignore_stderr}
    Builtin.Comment    TODO: Do not pollute current remote directory. See https://github.com/contiv/vpp/issues/195
    SSHLibrary.Put_File    ${file_path}    .
    ${splitted_path} =    String.Split_String    ${file_path}    separator=${/}
    BuiltIn.Run_Keyword_And_Return    Execute_Command_And_Log    ${command_prefix} @{splitted_path}[-1]    expected_rc=${expected_rc}    ignore_stderr=${ignore_stderr}

Switch_Execute_And_Log_To_File
    [Arguments]    ${ssh_session}    ${command}    ${expected_rc}=0    ${ignore_stderr}=${False}    ${ignore_rc}=${False}    ${compress}=${False}
    [Documentation]    Call Switch_And_Execute_Command redirecting stdout to a remote file, download the file.
    ...    To distinguish separate invocations, suite name, test name, session alias
    ...    and full command are used to construct file name.
    BuiltIn.Log_Many    ${ssh_session}    ${command}    ${expected_rc}    ${ignore_stderr}    ${ignore_rc}    ${compress}
    SSHLibrary.Switch_Connection    ${ssh_session}
    ${connection} =    SSHLibrary.Get_Connection
    # In teardown, ${TEST_NAME} does not exist.
    ${testname} =    BuiltIn.Get_Variable_Value    ${TEST_NAME}    ${EMPTY}
    ${filename_with_spaces} =    BuiltIn.Set_Variable    ${testname}__${SUITE_NAME}__${connection.alias}__${command}.log
    ${filename} =    String.Replace_String    ${filename_with_spaces}    ${SPACE}    _
    BuiltIn.Log    ${filename}
    Execute_Command_And_Log    ${command} > ${filename}    expected_rc=${expected_rc}    ignore_stderr=${ignore_stderr}    ignore_rc=${ignore_rc}
    BuiltIn.Run_Keyword_If    ${compress}    Execute_Command_And_Log    xz -9e ${filename}
    ${filename} =    Builtin.Set_Variable_If    ${compress}    ${filename}.xz    ${filename}
    SSHLibrary.Get_File    ${filename}    ${RESULTS_FOLDER}/${filename}
    [Teardown]    Execute_Command_And_Log    rm ${filename}

Switch_And_Execute_Command
    [Arguments]    ${ssh_session}    ${command}    ${expected_rc}=0    ${ignore_stderr}=${False}    ${ignore_rc}=${False}
    [Documentation]    Switch to \${ssh_session}, and continue with Execute_Command_And_Log.
    BuiltIn.Log_Many    ${ssh_session}    ${command}    ${expected_rc}    ${ignore_stderr}    ${ignore_rc}
    SSHLibrary.Switch_Connection    ${ssh_session}
    BuiltIn.Run_Keyword_And_Return    Execute_Command_And_Log    ${command}    expected_rc=${expected_rc}    ignore_stderr=${ignore_stderr}    ignore_rc=${ignore_rc}

Execute_Command_And_Log
    [Arguments]    ${command}    ${expected_rc}=0    ${ignore_stderr}=${False}    ${ignore_rc}=${False}
    [Documentation]    Execute \${command} on current SSH session, log results, maybe fail on nonempty stderr, check \${expected_rc}, return stdout.
    BuiltIn.Log_Many    ${command}    ${expected_rc}    ${ignore_stderr}    ${ignore_rc}
    ${stdout}    ${stderr}    ${rc} =    SSHLibrary.Execute_Command    ${command}    return_stderr=True    return_rc=True
    Append_Command_Log    ${command}    ${stdout}    ${stderr}    ${rc}
    BuiltIn.Run_Keyword_Unless    ${ignore_stderr}    BuiltIn.Should_Be_Empty    ${stderr}
    BuiltIn.Run_Keyword_Unless    ${ignore_rc}    BuiltIn.Should_Be_Equal_As_Numbers    ${rc}    ${expected_rc}
    [Return]    ${stdout}

Switch_And_Write_Command
    [Arguments]    ${ssh_session}    ${command}    ${prompt}=vpp#
    [Documentation]    Switch to \${ssh_session}, and continue with Write_Command_And_Log
    BuiltIn.Log_Many    ${ssh_session}    ${command}    ${prompt}
    SSHLibrary.Switch_Connection    ${ssh_session}
    BuiltIn.Run_Keyword_And_Return    Write_Command_And_Log    ${command}    ${prompt}

Write_Command_And_Log
    [Arguments]    ${command}    ${prompt}=vpp#
    [Documentation]    Write \${command} on current SSH session, wait for prompt, log output, return output.
    BuiltIn.Log_Many    ${command}    ${prompt}
    SSHLibrary.Write    ${command}
    ${output} =    SSHLibrary.Read_Until    ${prompt}
    Append_Command_Log    ${command}    ${output}
    [Return]    ${output}

Append_Command_Log
    [Arguments]    ${command}    ${output}=${EMPTY}    ${stderr}=${EMPTY}    ${rc}=${EMPTY}
    [Documentation]    Detect connection alias and time, append line with command and output to appropriate log file.
    Builtin.Log_Many    ${command}    ${output}    ${stderr}    ${rc}
    ${connection} =    SSHLibrary.Get_Connection
    ${time} =    DateTime.Get_Current_Date
    ${output_length} =    BuiltIn.Get_Length    ${output}
    ${if_output} =    BuiltIn.Set_Variable_If    ${output_length}    ${output}${\n}    ${EMPTY}
    ${stderr_length} =    Builtin.Get_Length    ${stderr}
    ${if_stderr} =    BuiltIn.Set_Variable_If    ${stderr_length}    *** Stderr: ${stderr}${\n}    ${EMPTY}
    ${if_rc} =    BuiltIn.Set_Variable_If    """${rc}"""    *** Return code: ${rc}${\n}    ${EMPTY}
    OperatingSystem.Append_To_File    ${RESULTS_FOLDER}/output_${connection.alias}.log    ${time}${\n}*** Command: ${command}${\n}${if_stderr}${if_rc}${if_output}
{'fraction_non_alphanumeric': 0.13617645005932855, 'fraction_numerical': 0.0018147553570182174, 'mean_word_length': 3.595253367543297, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5093386', 'n_tokens_mistral': 4621, 'n_tokens_neox': 4559, 'n_words': 925}