<commit_before>#!/usr/bin/env python3

from core_symbol import CORE_SYMBOL
from Cluster import Cluster
from WalletMgr import WalletMgr
from Node import Node
from TestHelper import TestHelper
from testUtils import Utils
import testUtils
import time

import decimal
import math
import re

###############################################################
# nodeos_under_min_avail_ram
#
# Sets up 4 producing nodes using --chain-state-db-guard-size-mb and --chain-state-db-size-mb to verify that nodeos will
# shutdown safely when --chain-state-db-guard-size-mb is reached and restarts the shutdown nodes, with a higher
# --chain-state-db-size-mb size, to verify that the node can restart and continue till the guard is reached again. The
# test both verifies all nodes going down and 1 node at a time.
#
###############################################################

Print=Utils.Print
errorExit=Utils.errorExit

class NamedAccounts:

    def __init__(self, cluster, numAccounts):
        Print("NamedAccounts %d" % (numAccounts))
        self.numAccounts=numAccounts
        self.accounts=cluster.createAccountKeys(numAccounts)
        if self.accounts is None:
            errorExit("FAILURE - create keys")
        accountNum = 0
        for account in self.accounts:
            Print("NamedAccounts Name for %d" % (accountNum))
            account.name=self.setName(accountNum)
            accountNum+=1

    def setName(self, num):
        retStr="test"
        digits=[]
        maxDigitVal=5
        maxDigits=8
        temp=num
        while len(digits) < maxDigits:
            digit=(num % maxDigitVal)+1
            num=int(num/maxDigitVal)
            digits.append(digit)

        digits.reverse()
        for digit in digits:
            retStr=retStr+str(digit)

        Print("NamedAccounts Name for %d is %s" % (temp, retStr))
        return retStr


args = TestHelper.parse_args({"--dump-error-details","--keep-logs","-v","--leave-running","--clean-run","--wallet-port"})
Utils.Debug=args.v
totalNodes=4
cluster=Cluster(walletd=True)
dumpErrorDetails=args.dump_error_details
keepLogs=args.keep_logs
dontKill=args.leave_running
killAll=args.clean_run
walletPort=args.wallet_port

walletMgr=WalletMgr(True, port=walletPort)
testSuccessful=False
killEosInstances=not dontKill
killWallet=not dontKill

WalletdName=Utils.EosWalletName
ClientName="cleos"

try:
    TestHelper.printSystemInfo("BEGIN")
    cluster.setWalletMgr(walletMgr)

    cluster.killall(allInstances=killAll)
    cluster.cleanup()
    Print("Stand up cluster")
    minRAMFlag="--chain-state-db-guard-size-mb"
    minRAMValue=1002
    maxRAMFlag="--chain-state-db-size-mb"
    maxRAMValue=1010
    extraNodeosArgs=" %s %d %s %d " % (minRAMFlag, minRAMValue, maxRAMFlag, maxRAMValue)
    if cluster.launch(onlyBios=False, pnodes=totalNodes, totalNodes=totalNodes, totalProducers=totalNodes, extraNodeosArgs=extraNodeosArgs, useBiosBootFile=False) is False:
        Utils.cmdError("launcher")
        errorExit("Failed to stand up eos cluster.")

    Print("Validating system accounts after bootstrap")
    cluster.validateAccounts(None)

    Print("creating accounts")
    namedAccounts=NamedAccounts(cluster,10)
    accounts=namedAccounts.accounts

    testWalletName="test"

    Print("Creating wallet \"%s\"." % (testWalletName))
    testWallet=walletMgr.create(testWalletName, [cluster.eosioAccount])

    for _, account in cluster.defProducerAccounts.items():
        walletMgr.importKey(account, testWallet, ignoreDupKeyWarning=True)

    Print("Wallet \"%s\" password=%s." % (testWalletName, testWallet.password.encode("utf-8")))

    nodes=[]
    nodes.append(cluster.getNode(0))
    nodes.append(cluster.getNode(1))
    nodes.append(cluster.getNode(2))
    nodes.append(cluster.getNode(3))
    numNodes=len(nodes)


    for account in accounts:
        walletMgr.importKey(account, testWallet)

    # create accounts via eosio as otherwise a bid is needed
    for account in accounts:
        Print("Create new account %s via %s" % (account.name, cluster.eosioAccount.name))
        trans=nodes[0].createInitializeAccount(account, cluster.eosioAccount, stakedDeposit=500000, waitForTransBlock=False, stakeNet=50000, stakeCPU=50000, buyRAM=50000, exitOnError=True)
        transferAmount="70000000.0000 {0}".format(CORE_SYMBOL)
        Print("Transfer funds %s from account %s to %s" % (transferAmount, cluster.eosioAccount.name, account.name))
        nodes[0].transferFunds(cluster.eosioAccount, account, transferAmount, "test transfer")
        trans=nodes[0].delegatebw(account, 1000000.0000, 68000000.0000, waitForTransBlock=True, exitOnError=True)

    contractAccount=cluster.createAccountKeys(1)[0]
    contractAccount.name="contracttest"
    walletMgr.importKey(contractAccount, testWallet)
    Print("Create new account %s via %s" % (contractAccount.name, cluster.eosioAccount.name))
    trans=nodes[0].createInitializeAccount(contractAccount, cluster.eosioAccount, stakedDeposit=500000, waitForTransBlock=False, stakeNet=50000, stakeCPU=50000, buyRAM=50000, exitOnError=True)
    transferAmount="90000000.0000 {0}".format(CORE_SYMBOL)
    Print("Transfer funds %s from account %s to %s" % (transferAmount, cluster.eosioAccount.name, contractAccount.name))
    nodes[0].transferFunds(cluster.eosioAccount, contractAccount, transferAmount, "test transfer")
    trans=nodes[0].delegatebw(contractAccount, 1000000.0000, 88000000.0000, waitForTransBlock=True, exitOnError=True)

    contractDir="unittests/test-contracts/integration_test"
    wasmFile="integration_test.wasm"
    abiFile="integration_test.abi"
    Print("Publish contract")
    trans=nodes[0].publishContract(contractAccount.name, contractDir, wasmFile, abiFile, waitForTransBlock=True)
    if trans is None:
        Utils.cmdError("%s set contract %s" % (ClientName, contractAccount.name))
        errorExit("Failed to publish contract.")

    contract=contractAccount.name
    Print("push create action to %s contract" % (contract))
    action="store"
    numAmount=5000
    keepProcessing=True
    count=0
    while keepProcessing:
        numAmount+=1
        timeOutCount=0
        for fromIndex in range(namedAccounts.numAccounts):
            count+=1
            toIndex=fromIndex+1
            if toIndex==namedAccounts.numAccounts:
                toIndex=0
            fromAccount=accounts[fromIndex]
            toAccount=accounts[toIndex]
            data="{\"from\":\"%s\",\"to\":\"%s\",\"num\":%d}" % (fromAccount.name, toAccount.name, numAmount)
            opts="--permission %s@active --permission %s@active --expiration 90" % (contract, fromAccount.name)
            try:
                trans=nodes[count % numNodes].pushMessage(contract, action, data, opts)
                if trans is None or not trans[0]:
                    timeOutCount+=1
                    if timeOutCount>=3:
                        Print("Failed to push create action to eosio contract for %d consecutive times, looks like nodeos already exited." % (timeOutCount))
                        keepProcessing=False
                        break

                    Print("Failed to push create action to eosio contract. sleep for 5 seconds")
                    time.sleep(5)
                else:
                    timeOutCount=0
                time.sleep(1)
            except TypeError as ex:
                keepProcessing=False
                break

    #spread the actions to all accounts, to use each accounts tps bandwidth
    fromIndexStart=fromIndex+1 if fromIndex+1<namedAccounts.numAccounts else 0

    if count < 5 or count > 15:
        strMsg="little" if count < 15 else "much"
        Utils.cmdError("Was able to send %d store actions which was too %s" % (count, strMsg))
        errorExit("Incorrect number of store actions sent")

    # Make sure all the nodes are shutdown (may take a little while for this to happen, so making multiple passes)
    allDone=False
    count=0
    while not allDone:
        allDone=True
        for node in nodes:
            if node.verifyAlive():
                allDone=False
        if not allDone:
            time.sleep(5)<commit_msg>Fixed test to not count failed attempts as actual sends and added some leeway in validation.<commit_after>#!/usr/bin/env python3

from core_symbol import CORE_SYMBOL
from Cluster import Cluster
from WalletMgr import WalletMgr
from Node import Node
from TestHelper import TestHelper
from testUtils import Utils
import testUtils
import time

import decimal
import math
import re

###############################################################
# nodeos_under_min_avail_ram
#
# Sets up 4 producing nodes using --chain-state-db-guard-size-mb and --chain-state-db-size-mb to verify that nodeos will
# shutdown safely when --chain-state-db-guard-size-mb is reached and restarts the shutdown nodes, with a higher
# --chain-state-db-size-mb size, to verify that the node can restart and continue till the guard is reached again. The
# test both verifies all nodes going down and 1 node at a time.
#
###############################################################

Print=Utils.Print
errorExit=Utils.errorExit

class NamedAccounts:

    def __init__(self, cluster, numAccounts):
        Print("NamedAccounts %d" % (numAccounts))
        self.numAccounts=numAccounts
        self.accounts=cluster.createAccountKeys(numAccounts)
        if self.accounts is None:
            errorExit("FAILURE - create keys")
        accountNum = 0
        for account in self.accounts:
            Print("NamedAccounts Name for %d" % (accountNum))
            account.name=self.setName(accountNum)
            accountNum+=1

    def setName(self, num):
        retStr="test"
        digits=[]
        maxDigitVal=5
        maxDigits=8
        temp=num
        while len(digits) < maxDigits:
            digit=(num % maxDigitVal)+1
            num=int(num/maxDigitVal)
            digits.append(digit)

        digits.reverse()
        for digit in digits:
            retStr=retStr+str(digit)

        Print("NamedAccounts Name for %d is %s" % (temp, retStr))
        return retStr


args = TestHelper.parse_args({"--dump-error-details","--keep-logs","-v","--leave-running","--clean-run","--wallet-port"})
Utils.Debug=args.v
totalNodes=4
cluster=Cluster(walletd=True)
dumpErrorDetails=args.dump_error_details
keepLogs=args.keep_logs
dontKill=args.leave_running
killAll=args.clean_run
walletPort=args.wallet_port

walletMgr=WalletMgr(True, port=walletPort)
testSuccessful=False
killEosInstances=not dontKill
killWallet=not dontKill

WalletdName=Utils.EosWalletName
ClientName="cleos"

try:
    TestHelper.printSystemInfo("BEGIN")
    cluster.setWalletMgr(walletMgr)

    cluster.killall(allInstances=killAll)
    cluster.cleanup()
    Print("Stand up cluster")
    minRAMFlag="--chain-state-db-guard-size-mb"
    minRAMValue=1002
    maxRAMFlag="--chain-state-db-size-mb"
    maxRAMValue=1010
    extraNodeosArgs=" %s %d %s %d " % (minRAMFlag, minRAMValue, maxRAMFlag, maxRAMValue)
    if cluster.launch(onlyBios=False, pnodes=totalNodes, totalNodes=totalNodes, totalProducers=totalNodes, extraNodeosArgs=extraNodeosArgs, useBiosBootFile=False) is False:
        Utils.cmdError("launcher")
        errorExit("Failed to stand up eos cluster.")

    Print("Validating system accounts after bootstrap")
    cluster.validateAccounts(None)

    Print("creating accounts")
    namedAccounts=NamedAccounts(cluster,10)
    accounts=namedAccounts.accounts

    testWalletName="test"

    Print("Creating wallet \"%s\"." % (testWalletName))
    testWallet=walletMgr.create(testWalletName, [cluster.eosioAccount])

    for _, account in cluster.defProducerAccounts.items():
        walletMgr.importKey(account, testWallet, ignoreDupKeyWarning=True)

    Print("Wallet \"%s\" password=%s." % (testWalletName, testWallet.password.encode("utf-8")))

    nodes=[]
    nodes.append(cluster.getNode(0))
    nodes.append(cluster.getNode(1))
    nodes.append(cluster.getNode(2))
    nodes.append(cluster.getNode(3))
    numNodes=len(nodes)


    for account in accounts:
        walletMgr.importKey(account, testWallet)

    # create accounts via eosio as otherwise a bid is needed
    for account in accounts:
        Print("Create new account %s via %s" % (account.name, cluster.eosioAccount.name))
        trans=nodes[0].createInitializeAccount(account, cluster.eosioAccount, stakedDeposit=500000, waitForTransBlock=False, stakeNet=50000, stakeCPU=50000, buyRAM=50000, exitOnError=True)
        transferAmount="70000000.0000 {0}".format(CORE_SYMBOL)
        Print("Transfer funds %s from account %s to %s" % (transferAmount, cluster.eosioAccount.name, account.name))
        nodes[0].transferFunds(cluster.eosioAccount, account, transferAmount, "test transfer")
        trans=nodes[0].delegatebw(account, 1000000.0000, 68000000.0000, waitForTransBlock=True, exitOnError=True)

    contractAccount=cluster.createAccountKeys(1)[0]
    contractAccount.name="contracttest"
    walletMgr.importKey(contractAccount, testWallet)
    Print("Create new account %s via %s" % (contractAccount.name, cluster.eosioAccount.name))
    trans=nodes[0].createInitializeAccount(contractAccount, cluster.eosioAccount, stakedDeposit=500000, waitForTransBlock=False, stakeNet=50000, stakeCPU=50000, buyRAM=50000, exitOnError=True)
    transferAmount="90000000.0000 {0}".format(CORE_SYMBOL)
    Print("Transfer funds %s from account %s to %s" % (transferAmount, cluster.eosioAccount.name, contractAccount.name))
    nodes[0].transferFunds(cluster.eosioAccount, contractAccount, transferAmount, "test transfer")
    trans=nodes[0].delegatebw(contractAccount, 1000000.0000, 88000000.0000, waitForTransBlock=True, exitOnError=True)

    contractDir="unittests/test-contracts/integration_test"
    wasmFile="integration_test.wasm"
    abiFile="integration_test.abi"
    Print("Publish contract")
    trans=nodes[0].publishContract(contractAccount.name, contractDir, wasmFile, abiFile, waitForTransBlock=True)
    if trans is None:
        Utils.cmdError("%s set contract %s" % (ClientName, contractAccount.name))
        errorExit("Failed to publish contract.")

    contract=contractAccount.name
    Print("push create action to %s contract" % (contract))
    action="store"
    numAmount=5000
    keepProcessing=True
    count=0
    while keepProcessing:
        numAmount+=1
        timeOutCount=0
        for fromIndex in range(namedAccounts.numAccounts):
            count+=1
            toIndex=fromIndex+1
            if toIndex==namedAccounts.numAccounts:
                toIndex=0
            fromAccount=accounts[fromIndex]
            toAccount=accounts[toIndex]
            data="{\"from\":\"%s\",\"to\":\"%s\",\"num\":%d}" % (fromAccount.name, toAccount.name, numAmount)
            opts="--permission %s@active --permission %s@active --expiration 90" % (contract, fromAccount.name)
            try:
                trans=nodes[count % numNodes].pushMessage(contract, action, data, opts)
                if trans is None or not trans[0]:
                    timeOutCount+=1
                    if timeOutCount>=3:
                        Print("Failed to push create action to eosio contract for %d consecutive times, looks like nodeos already exited." % (timeOutCount))
                        keepProcessing=False
                        break

                    Print("Failed to push create action to eosio contract. sleep for 5 seconds")
                    count-=1 # failed attempt shouldn't be counted
                    time.sleep(5)
                else:
                    timeOutCount=0
                time.sleep(1)
            except TypeError as ex:
                keepProcessing=False
                break

    #spread the actions to all accounts, to use each accounts tps bandwidth
    fromIndexStart=fromIndex+1 if fromIndex+1<namedAccounts.numAccounts else 0

    # min and max are subjective, just assigned to make sure that many small changes in nodeos don't 
    # result in the test not correctly validating behavior
    if count < 5 or count > 20:
        strMsg="little" if count < 20 else "much"
        Utils.cmdError("Was able to send %d store actions which was too %s" % (count, strMsg))
        errorExit("Incorrect number of store actions sent")

    # Make sure all the nodes are shutdown (may take a little while for this to happen, so making multiple passes)
    allDone=False
    count=0
    while not allDone:
        allDone=True
        for node in nodes:
            if node.verifyAlive():
                allDone=False
        if not allDone:
            time.sleep(5)
{'fraction_non_alphanumeric': 0.1147383003071734, 'fraction_numerical': 0.021261217852195386, 'mean_word_length': 3.3116073747078683, 'pattern_counts': {'":': 6, '<': 11, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 4, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '912305', 'n_tokens_mistral': 5081, 'n_tokens_neox': 4778, 'n_words': 1163}