<commit_before>// Copyright (c) The Diem Core Contributors
// SPDX-License-Identifier: Apache-2.0

use anyhow::{anyhow, Result};
use std::{collections::BTreeSet, fs, path::Path};
use structopt::StructOpt;
use walkdir::WalkDir;

use bytecode_interpreter::{
    concrete::{
        runtime::{convert_move_struct_tag, convert_move_value},
        settings::InterpreterSettings,
        ty::BaseType,
        value::GlobalState,
    },
    interpret_with_default_pipeline_and_bcs_arguments,
};
use diem_framework::diem_stdlib_files;
use diem_types::{
    access_path::Path as AP,
    account_address::AccountAddress,
    account_config::{
        from_currency_code_string, reserved_vm_address, type_tag_for_currency_code, ACCOUNT_MODULE,
    },
    block_metadata::BlockMetadata,
    transaction::{
        Script, ScriptFunction, Transaction, TransactionArgument, TransactionOutput,
        TransactionPayload, TransactionStatus, WriteSetPayload,
    },
};
use diem_vm::{
    convert_changeset_and_events,
    script_to_script_function::remapping,
    system_module_names::{
        BLOCK_PROLOGUE, DIEM_BLOCK_MODULE, SCRIPT_PROLOGUE_NAME, USER_EPILOGUE_NAME,
        WRITESET_EPILOGUE_NAME, WRITESET_PROLOGUE_NAME,
    },
    transaction_metadata::TransactionMetadata,
};
use language_e2e_tests::{
    data_store::FakeDataStore,
    executor::{
        TraceSeqMapping, TRACE_DIR_DATA, TRACE_DIR_INPUT, TRACE_DIR_META, TRACE_DIR_OUTPUT,
        TRACE_FILE_ERROR, TRACE_FILE_NAME,
    },
};
use move_binary_format::{errors::VMResult, CompiledModule};
use move_core_types::{
    effects::{ChangeSet, Event},
    gas_schedule::GasAlgebra,
    identifier::IdentStr,
    language_storage::{ModuleId, TypeTag},
    value::MoveValue,
    vm_status::{KeptVMStatus, VMStatus},
};
use move_model::{model::GlobalEnv, run_model_builder};
use move_vm_runtime::{logging::NoContextLog, move_vm::MoveVM, session::Session};
use move_vm_types::gas_schedule::GasStatus;

//**************************************************************************************************
// Utilities
//**************************************************************************************************

fn script_to_script_function(script: &Script) -> Option<ScriptFunction> {
    remapping(script.code()).map(|(module_id, func_name)| {
        ScriptFunction::new(
            module_id.clone(),
            func_name.to_owned(),
            script.ty_args().to_vec(),
            script
                .args()
                .iter()
                .map(|arg| {
                    match arg {
                        TransactionArgument::U8(i) => MoveValue::U8(*i),
                        TransactionArgument::U64(i) => MoveValue::U64(*i),
                        TransactionArgument::U128(i) => MoveValue::U128(*i),
                        TransactionArgument::Address(a) => MoveValue::Address(*a),
                        TransactionArgument::Bool(b) => MoveValue::Bool(*b),
                        TransactionArgument::U8Vector(v) => MoveValue::vector_u8(v.clone()),
                    }
                    .simple_serialize()
                    .unwrap()
                })
                .collect(),
        )
    })
}

fn compare_output(
    expect_output: &TransactionOutput,
    actual_output: VMResult<(ChangeSet, Vec<Event>)>,
) {
    match actual_output {
        Ok((change_set, events)) => {
            let (actual_write_set, actual_events) =
                convert_changeset_and_events(change_set, events).unwrap();
            assert_eq!(expect_output.write_set(), &actual_write_set);
            assert_eq!(expect_output.events(), &actual_events);
        }
        Err(err) => {
            assert!(expect_output.write_set().is_empty());
            assert!(expect_output.events().is_empty());
            let expect_status = expect_output.status();
            let actual_status = &err.into_vm_status();
            match (expect_status, actual_status) {
                (
                    TransactionStatus::Keep(KeptVMStatus::MoveAbort(expect_loc, expect_code)),
                    VMStatus::MoveAbort(actual_loc, actual_code),
                ) => {
                    assert_eq!(expect_loc, actual_loc);
                    assert_eq!(expect_code, actual_code);
                }
                (
                    TransactionStatus::Keep(KeptVMStatus::ExecutionFailure {
                        location: expect_loc,
                        function: expect_func,
                        code_offset: expect_offset,
                    }),
                    VMStatus::ExecutionFailure {
                        status_code: _,
                        location: actual_loc,
                        function: actual_func,
                        code_offset: actual_offset,
                    },
                ) => {
                    assert_eq!(expect_loc, actual_loc);
                    assert_eq!(expect_func, actual_func);
                    assert_eq!(expect_offset, actual_offset);
                }
                _ => panic!(
                    "Execution error does not match\nExpect: {:?}\nActual: {}",
                    expect_status, actual_status
                ),
            }
        }
    }
}

//**************************************************************************************************
// Cross-VM comparison
//**************************************************************************************************

struct CrossRunner<'env> {
    env: &'env GlobalEnv,
    move_vm_state: FakeDataStore,
    stackless_vm_state: GlobalState,
    stackless_vm_settings: InterpreterSettings,
    flags: &'env ReplayFlags,
}

impl<'env> CrossRunner<'env> {
    pub fn new(env: &'env GlobalEnv, data_store: &FakeDataStore, flags: &'env ReplayFlags) -> Self {
        fn serialize_module(module: &CompiledModule) -> Vec<u8> {
            let mut code = vec![];
            module.serialize(&mut code).unwrap();
            code
        }

        let mut move_vm_state = data_store.clone();
        let mut stackless_vm_state = GlobalState::default();
        let mut included_modules = BTreeSet::new();
        for (ap, blob) in data_store.inner() {
            match ap.get_path() {
                AP::Code(module_id) => {
                    let module_env = env.find_module_by_language_storage_id(&module_id).unwrap();
                    // update the module code to the same version as the one in the GlobalEnv
                    move_vm_state.add_module(
                        &module_id,
                        serialize_module(module_env.get_verified_module()),
                    );
                    // mark this module as included
                    included_modules.insert(module_id);
                }
                AP::Resource(struct_tag) => {
                    let inst = convert_move_struct_tag(env, &struct_tag).unwrap();
                    let struct_ty = BaseType::mk_struct(inst);
                    let struct_val =
                        MoveValue::simple_deserialize(blob, &struct_ty.to_move_type_layout())
                            .unwrap();
                    let resource = convert_move_value(env, &struct_val, &struct_ty).unwrap();
                    let inst = struct_ty.into_struct_inst();
                    stackless_vm_state.put_resource(ap.address, inst, resource);
                }
            }
        }
        for module_env in env.get_modules() {
            let verified_module = module_env.get_verified_module();
            let module_id = verified_module.self_id();
            if included_modules.contains(&module_id) {
                continue;
            }
            // add new modules in the environment
            move_vm_state.add_module(&module_id, serialize_module(verified_module));
        }

        let settings = if flags.verbose_stackless_vm {
            InterpreterSettings::verbose_default()
        } else {
            InterpreterSettings::default()
        };

        Self {
            env,
            move_vm_state,
            stackless_vm_state,
            stackless_vm_settings: settings,
            flags,
        }
    }

    pub fn step_function_and_compare(
        &mut self,
        module_id: &ModuleId,
        function_name: &IdentStr,
        ty_args: &[TypeTag],
        args: &[Vec<u8>],
    ) {
        if self.flags.verbose_trace_xrun {
            eprintln!(
                "[-] xrun: 0x{}::{}::{}<{}>",
                module_id.address().short_str_lossless(),
                module_id.name(),
                function_name,
                ty_args
                    .iter()
                    .map(|tag| tag.to_string())
                    .collect::<Vec<_>>()
                    .join(", ")
            )
        }

        // execute via move VM
        let move_vm = MoveVM::new();
        let mut session = move_vm.new_session(&self.move_vm_state);
        let move_vm_return_values = execute_function_via_session(
            &mut session,
            module_id,
            function_name,
            ty_args.to_vec(),
            args.to_vec(),
        );
        let (move_vm_change_set, move_events) = session.finish().unwrap();

        // execute via stackless VM
        let (stackless_vm_return_values, stackless_vm_change_set, new_stackless_vm_state) =
            interpret_with_default_pipeline_and_bcs_arguments(
                self.env,
                module_id,
                function_name,
                ty_args,
                args,
                None,
                &self.stackless_vm_state,
                self.stackless_vm_settings.clone(),
            );

        // compare
        assert_eq!(move_vm_return_values, stackless_vm_return_values);
        assert_eq!(move_vm_change_set, stackless_vm_change_set);

        // update the states
        let (move_write_set, _) =
            convert_changeset_and_events(move_vm_change_set, move_events).unwrap();
        self.move_vm_state.add_write_set(&move_write_set);
        self.stackless_vm_state = new_stackless_vm_state;
    }

    pub fn step_script_function_and_compare(
        &mut self,
        module_id: &ModuleId,
        function_name: &IdentStr,
        ty_args: &[TypeTag],
        args: &[Vec<u8>],
        senders: &[AccountAddress],
    ) {
        if self.flags.verbose_trace_xrun {
            eprintln!(
                "[-] xrun: 0x{}::{}::{}<{}>",
                module_id.address().short_str_lossless(),
                module_id.name(),
                function_name,
                ty_args
                    .iter()
                    .map(|tag| tag.to_string())
                    .collect::<Vec<_>>()
                    .join(", ")
            )
        }

        // execute via move VM
        let move_vm = MoveVM::new();
        let mut session = move_vm.new_session(&self.move_vm_state);
        let move_vm_return_values = execute_script_function_via_session(
            &mut session,
            module_id,
            function_name,
            ty_args.to_vec(),
            args.to_vec(),
            senders.to_vec(),
        );
        let (move_vm_change_set, move_events) = session.finish().unwrap();

        // execute via stackless VM
        let (stackless_vm_return_values, stackless_vm_change_set, new_stackless_vm_state) =
            interpret_with_default_pipeline_and_bcs_arguments(
                self.env,
                module_id,
                function_name,
                ty_args,
                args,
                Some(senders),
                &self.stackless_vm_state,
                self.stackless_vm_settings.clone(),
            );
        let stackless_vm_return_values =
            stackless_vm_return_values.map(|rets| assert!(rets.is_empty()));

        // compare
        assert_eq!(move_vm_return_values, stackless_vm_return_values);
        assert_eq!(move_vm_change_set, stackless_vm_change_set);

        // update the states
        let (move_write_set, _) =
            convert_changeset_and_events(move_vm_change_set, move_events).unwrap();
        self.move_vm_state.add_write_set(&move_write_set);
        self.stackless_vm_state = new_stackless_vm_state;
    }
}

//**************************************************************************************************
// Executors
//**************************************************************************************************

fn execute_function_via_session(
    session: &mut Session<FakeDataStore>,
    module_id: &ModuleId,
    function_name: &IdentStr,
    ty_args: Vec<TypeTag>,
    args: Vec<Vec<u8>>,
) -> VMResult<Vec<Vec<u8>>> {
    let mut gas_status = GasStatus::new_unmetered();
    let log_context = NoContextLog::new();
    session.execute_function(
        module_id,
        function_name,<commit_msg>[test-infra][e2e] adapt Move VM results before cross-comparison<commit_after>// Copyright (c) The Diem Core Contributors
// SPDX-License-Identifier: Apache-2.0

use anyhow::{anyhow, Result};
use std::{collections::BTreeSet, fs, path::Path};
use structopt::StructOpt;
use walkdir::WalkDir;

use bytecode_interpreter::{
    concrete::{
        runtime::{convert_move_struct_tag, convert_move_value},
        settings::InterpreterSettings,
        ty::BaseType,
        value::GlobalState,
    },
    interpret_with_default_pipeline_and_bcs_arguments,
};
use diem_framework::diem_stdlib_files;
use diem_types::{
    access_path::Path as AP,
    account_address::AccountAddress,
    account_config::{
        from_currency_code_string, reserved_vm_address, type_tag_for_currency_code, ACCOUNT_MODULE,
    },
    block_metadata::BlockMetadata,
    transaction::{
        Script, ScriptFunction, Transaction, TransactionArgument, TransactionOutput,
        TransactionPayload, TransactionStatus, WriteSetPayload,
    },
};
use diem_vm::{
    convert_changeset_and_events,
    script_to_script_function::remapping,
    system_module_names::{
        BLOCK_PROLOGUE, DIEM_BLOCK_MODULE, SCRIPT_PROLOGUE_NAME, USER_EPILOGUE_NAME,
        WRITESET_EPILOGUE_NAME, WRITESET_PROLOGUE_NAME,
    },
    transaction_metadata::TransactionMetadata,
};
use language_e2e_tests::{
    data_store::FakeDataStore,
    executor::{
        TraceSeqMapping, TRACE_DIR_DATA, TRACE_DIR_INPUT, TRACE_DIR_META, TRACE_DIR_OUTPUT,
        TRACE_FILE_ERROR, TRACE_FILE_NAME,
    },
};
use move_binary_format::{
    errors::{PartialVMError, VMResult},
    CompiledModule,
};
use move_core_types::{
    effects::{ChangeSet, Event},
    gas_schedule::GasAlgebra,
    identifier::IdentStr,
    language_storage::{ModuleId, TypeTag},
    value::MoveValue,
    vm_status::{KeptVMStatus, VMStatus},
};
use move_model::{model::GlobalEnv, run_model_builder};
use move_vm_runtime::{logging::NoContextLog, move_vm::MoveVM, session::Session};
use move_vm_types::gas_schedule::GasStatus;

//**************************************************************************************************
// Utilities
//**************************************************************************************************

fn script_to_script_function(script: &Script) -> Option<ScriptFunction> {
    remapping(script.code()).map(|(module_id, func_name)| {
        ScriptFunction::new(
            module_id.clone(),
            func_name.to_owned(),
            script.ty_args().to_vec(),
            script
                .args()
                .iter()
                .map(|arg| {
                    match arg {
                        TransactionArgument::U8(i) => MoveValue::U8(*i),
                        TransactionArgument::U64(i) => MoveValue::U64(*i),
                        TransactionArgument::U128(i) => MoveValue::U128(*i),
                        TransactionArgument::Address(a) => MoveValue::Address(*a),
                        TransactionArgument::Bool(b) => MoveValue::Bool(*b),
                        TransactionArgument::U8Vector(v) => MoveValue::vector_u8(v.clone()),
                    }
                    .simple_serialize()
                    .unwrap()
                })
                .collect(),
        )
    })
}

fn compare_output(
    expect_output: &TransactionOutput,
    actual_output: VMResult<(ChangeSet, Vec<Event>)>,
) {
    match actual_output {
        Ok((change_set, events)) => {
            let (actual_write_set, actual_events) =
                convert_changeset_and_events(change_set, events).unwrap();
            assert_eq!(expect_output.write_set(), &actual_write_set);
            assert_eq!(expect_output.events(), &actual_events);
        }
        Err(err) => {
            assert!(expect_output.write_set().is_empty());
            assert!(expect_output.events().is_empty());
            let expect_status = expect_output.status();
            let actual_status = &err.into_vm_status();
            match (expect_status, actual_status) {
                (
                    TransactionStatus::Keep(KeptVMStatus::MoveAbort(expect_loc, expect_code)),
                    VMStatus::MoveAbort(actual_loc, actual_code),
                ) => {
                    assert_eq!(expect_loc, actual_loc);
                    assert_eq!(expect_code, actual_code);
                }
                (
                    TransactionStatus::Keep(KeptVMStatus::ExecutionFailure {
                        location: expect_loc,
                        function: expect_func,
                        code_offset: expect_offset,
                    }),
                    VMStatus::ExecutionFailure {
                        status_code: _,
                        location: actual_loc,
                        function: actual_func,
                        code_offset: actual_offset,
                    },
                ) => {
                    assert_eq!(expect_loc, actual_loc);
                    assert_eq!(expect_func, actual_func);
                    assert_eq!(expect_offset, actual_offset);
                }
                _ => panic!(
                    "Execution error does not match\nExpect: {:?}\nActual: {}",
                    expect_status, actual_status
                ),
            }
        }
    }
}

fn adapt_move_vm_result<T>(result: VMResult<T>) -> VMResult<T> {
    result.map_err(|err| {
        let (status_code, sub_status, _, location, _, _) = err.all_data();
        let adapted = PartialVMError::new(status_code);
        let adapted = match sub_status {
            None => adapted,
            Some(status_code) => adapted.with_sub_status(status_code),
        };
        adapted.finish(location)
    })
}

//**************************************************************************************************
// Cross-VM comparison
//**************************************************************************************************

struct CrossRunner<'env> {
    env: &'env GlobalEnv,
    move_vm_state: FakeDataStore,
    stackless_vm_state: GlobalState,
    stackless_vm_settings: InterpreterSettings,
    flags: &'env ReplayFlags,
}

impl<'env> CrossRunner<'env> {
    pub fn new(env: &'env GlobalEnv, data_store: &FakeDataStore, flags: &'env ReplayFlags) -> Self {
        fn serialize_module(module: &CompiledModule) -> Vec<u8> {
            let mut code = vec![];
            module.serialize(&mut code).unwrap();
            code
        }

        let mut move_vm_state = data_store.clone();
        let mut stackless_vm_state = GlobalState::default();
        let mut included_modules = BTreeSet::new();
        for (ap, blob) in data_store.inner() {
            match ap.get_path() {
                AP::Code(module_id) => {
                    let module_env = env.find_module_by_language_storage_id(&module_id).unwrap();
                    // update the module code to the same version as the one in the GlobalEnv
                    move_vm_state.add_module(
                        &module_id,
                        serialize_module(module_env.get_verified_module()),
                    );
                    // mark this module as included
                    included_modules.insert(module_id);
                }
                AP::Resource(struct_tag) => {
                    let inst = convert_move_struct_tag(env, &struct_tag).unwrap();
                    let struct_ty = BaseType::mk_struct(inst);
                    let struct_val =
                        MoveValue::simple_deserialize(blob, &struct_ty.to_move_type_layout())
                            .unwrap();
                    let resource = convert_move_value(env, &struct_val, &struct_ty).unwrap();
                    let inst = struct_ty.into_struct_inst();
                    stackless_vm_state.put_resource(ap.address, inst, resource);
                }
            }
        }
        for module_env in env.get_modules() {
            let verified_module = module_env.get_verified_module();
            let module_id = verified_module.self_id();
            if included_modules.contains(&module_id) {
                continue;
            }
            // add new modules in the environment
            move_vm_state.add_module(&module_id, serialize_module(verified_module));
        }

        let settings = if flags.verbose_stackless_vm {
            InterpreterSettings::verbose_default()
        } else {
            InterpreterSettings::default()
        };

        Self {
            env,
            move_vm_state,
            stackless_vm_state,
            stackless_vm_settings: settings,
            flags,
        }
    }

    pub fn step_function_and_compare(
        &mut self,
        module_id: &ModuleId,
        function_name: &IdentStr,
        ty_args: &[TypeTag],
        args: &[Vec<u8>],
    ) {
        if self.flags.verbose_trace_xrun {
            eprintln!(
                "[-] xrun: 0x{}::{}::{}<{}>",
                module_id.address().short_str_lossless(),
                module_id.name(),
                function_name,
                ty_args
                    .iter()
                    .map(|tag| tag.to_string())
                    .collect::<Vec<_>>()
                    .join(", ")
            )
        }

        // execute via move VM
        let move_vm = MoveVM::new();
        let mut session = move_vm.new_session(&self.move_vm_state);
        let move_vm_return_values = execute_function_via_session(
            &mut session,
            module_id,
            function_name,
            ty_args.to_vec(),
            args.to_vec(),
        );
        let (move_vm_change_set, move_events) = session.finish().unwrap();

        // execute via stackless VM
        let (stackless_vm_return_values, stackless_vm_change_set, new_stackless_vm_state) =
            interpret_with_default_pipeline_and_bcs_arguments(
                self.env,
                module_id,
                function_name,
                ty_args,
                args,
                None,
                &self.stackless_vm_state,
                self.stackless_vm_settings.clone(),
            );

        // compare
        let move_vm_return_values = adapt_move_vm_result(move_vm_return_values);
        assert_eq!(move_vm_return_values, stackless_vm_return_values);
        assert_eq!(move_vm_change_set, stackless_vm_change_set);

        // update the states
        let (move_write_set, _) =
            convert_changeset_and_events(move_vm_change_set, move_events).unwrap();
        self.move_vm_state.add_write_set(&move_write_set);
        self.stackless_vm_state = new_stackless_vm_state;
    }

    pub fn step_script_function_and_compare(
        &mut self,
        module_id: &ModuleId,
        function_name: &IdentStr,
        ty_args: &[TypeTag],
        args: &[Vec<u8>],
        senders: &[AccountAddress],
    ) {
        if self.flags.verbose_trace_xrun {
            eprintln!(
                "[-] xrun: 0x{}::{}::{}<{}>",
                module_id.address().short_str_lossless(),
                module_id.name(),
                function_name,
                ty_args
                    .iter()
                    .map(|tag| tag.to_string())
                    .collect::<Vec<_>>()
                    .join(", ")
            )
        }

        // execute via move VM
        let move_vm = MoveVM::new();
        let mut session = move_vm.new_session(&self.move_vm_state);
        let move_vm_return_values = execute_script_function_via_session(
            &mut session,
            module_id,
            function_name,
            ty_args.to_vec(),
            args.to_vec(),
            senders.to_vec(),
        );
        let (move_vm_change_set, move_events) = session.finish().unwrap();

        // execute via stackless VM
        let (stackless_vm_return_values, stackless_vm_change_set, new_stackless_vm_state) =
            interpret_with_default_pipeline_and_bcs_arguments(
                self.env,
                module_id,
                function_name,
                ty_args,
                args,
                Some(senders),
                &self.stackless_vm_state,
                self.stackless_vm_settings.clone(),
            );
        let stackless_vm_return_values =
            stackless_vm_return_values.map(|rets| assert!(rets.is_empty()));

        // compare
        let move_vm_return_values = adapt_move_vm_result(move_vm_return_values);
        assert_eq!(move_vm_return_values, stackless_vm_return_values);
        assert_eq!(move_vm_change_set, stackless_vm_change_set);

        // update the states
        let (move_write_set, _) =
            convert_changeset_and_events(move_vm_change_set, move_events).unwrap();
        self.move_vm_state.add_write_set(&move_write_set);
        self.stackless_vm_state = new_stackless_vm_state;
    }
}

//**************************************************************************************************
// Executors
//**************************************************************************************************

fn execute_function_via_session(
    session: &mut Session<FakeDataStore>,
    module_id: &ModuleId,
    function_name: &IdentStr,
    ty_args: Vec<TypeTag>,
    args: Vec<Vec<u8>>,
) -> VMResult<Vec<Vec<u8>>> {
    let mut gas_status = GasStatus::new_unmetered();
    let log_context = NoContextLog::new();
    session.execute_function(
        module_id,
        function_name,
{'fraction_non_alphanumeric': 0.14447107155083874, 'fraction_numerical': 0.0018638974476016584, 'mean_word_length': 2.049530216912191, 'pattern_counts': {'":': 0, '<': 50, '<?xml version=': 0, '>': 87, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4219721', 'n_tokens_mistral': 7320, 'n_tokens_neox': 7114, 'n_words': 1121}