<commit_before>#!/usr/bin/env python3

from testUtils import Utils
from Cluster import Cluster
from TestHelper import TestHelper

import random
import subprocess
import signal
import os
import shutil
import re
from datetime import datetime
from datetime import timedelta

Print=Utils.Print
errorExit=Utils.errorExit

stagingDir="rsmStaging"
dataDir=stagingDir+"/data"
configDir=stagingDir+"/etc"
traceDir=dataDir+"/traceDir"

loggingFile=configDir+"/logging.json"
stderrFile=dataDir + "/stderr.txt"

testNum=0

# We need debug level to get more information about nodeos process
logging="""{
  "includes": [],
  "appenders": [{
      "name": "stderr",
      "type": "console",
      "args": {
        "stream": "std_error",
        "level_colors": [{
            "level": "debug",
            "color": "green"
          },{
            "level": "warn",
            "color": "brown"
          },{
            "level": "error",
            "color": "red"
          }
        ]
      },
      "enabled": true
    }
  ],
  "loggers": [{
      "name": "default",
      "level": "debug",
      "enabled": true,
      "additivity": false,
      "appenders": [
        "stderr"
      ]
    }
  ]
}"""

def cleanDirectories():
    os.path.exists(stagingDir) and shutil.rmtree(stagingDir)

def prepareDirectories():
    # Prepare own directories so we don't depend on others to make sure
    # tests are repeatable
    cleanDirectories()
    os.makedirs(stagingDir)
    os.makedirs(dataDir)
    os.makedirs(configDir)

    with open(loggingFile, "w") as textFile:
        print(logging,file=textFile)

def runNodeos(extraNodeosArgs, myTimeout):
    """Startup nodeos, wait for timeout (before forced shutdown) and collect output."""
    if debug: Print("Launching nodeos process.")
    cmd="programs/nodeos/nodeos --config-dir rsmStaging/etc -e -p eosio --plugin eosio::chain_api_plugin --plugin eosio::history_api_plugin --data-dir " + dataDir + " "

    cmd=cmd + extraNodeosArgs;
    if debug: Print("cmd: %s" % (cmd))
    with open(stderrFile, 'w') as serr:
        proc=subprocess.Popen(cmd.split(), stdout=subprocess.PIPE, stderr=serr)

        try:
            proc.communicate(timeout=myTimeout)
        except (subprocess.TimeoutExpired) as _:
            if debug: Print("Timed out\n")
            proc.send_signal(signal.SIGKILL)

def isMsgInStderrFile(msg):
    msgFound=False
    with open(stderrFile) as errFile:
        for line in errFile:
            if msg in line:
                msgFound=True
                break
    return msgFound

def testCommon(title, extraNodeosArgs, expectedMsgs):
    global testNum
    testNum+=1
    Print("Test %d: %s" % (testNum, title))

    prepareDirectories()

    timeout=120  # Leave sufficient time such nodeos can start up fully in any platforms
    runNodeos(extraNodeosArgs, timeout)

    for msg in expectedMsgs:
        if not isMsgInStderrFile(msg):
            errorExit ("Log should have contained \"%s\"" % (expectedMsgs))

def extractTimestamp(msg):
    matches = re.compile("\s+([0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{3})\s").search(msg)
    return datetime.strptime(matches.group(0).strip(), '%Y-%m-%dT%H:%M:%S.%f')

maxFirstIntervalTolerance = 5
maxIntervalTolerance = 0.5

def isMsgIntervalValid(msg, interval):
    pre = ""
    hasMsg = validInterval = False
    firstInterval = True
    with open(stderrFile) as errFile:
        for line in errFile:
            if msg in line:
                hasMsg = True
                if pre:
                    curInterval = extractTimestamp(line) - extractTimestamp(pre)
                    if firstInterval:
                        intervalTolerance = maxFirstIntervalTolerance
                        firstInterval = False
                    else:
                        intervalTolerance = maxIntervalTolerance
                    if abs((curInterval - timedelta(seconds=interval)).total_seconds()) <= intervalTolerance:
                        validInterval = True
                    else:
                        validInterval = False
                        break
                pre = line
    return hasMsg, validInterval
<commit_msg>add debug print<commit_after>#!/usr/bin/env python3

from testUtils import Utils
from Cluster import Cluster
from TestHelper import TestHelper

import random
import subprocess
import signal
import os
import shutil
import re
from datetime import datetime
from datetime import timedelta

Print=Utils.Print
errorExit=Utils.errorExit

stagingDir="rsmStaging"
dataDir=stagingDir+"/data"
configDir=stagingDir+"/etc"
traceDir=dataDir+"/traceDir"

loggingFile=configDir+"/logging.json"
stderrFile=dataDir + "/stderr.txt"

testNum=0

# We need debug level to get more information about nodeos process
logging="""{
  "includes": [],
  "appenders": [{
      "name": "stderr",
      "type": "console",
      "args": {
        "stream": "std_error",
        "level_colors": [{
            "level": "debug",
            "color": "green"
          },{
            "level": "warn",
            "color": "brown"
          },{
            "level": "error",
            "color": "red"
          }
        ]
      },
      "enabled": true
    }
  ],
  "loggers": [{
      "name": "default",
      "level": "debug",
      "enabled": true,
      "additivity": false,
      "appenders": [
        "stderr"
      ]
    }
  ]
}"""

def cleanDirectories():
    os.path.exists(stagingDir) and shutil.rmtree(stagingDir)

def prepareDirectories():
    # Prepare own directories so we don't depend on others to make sure
    # tests are repeatable
    cleanDirectories()
    os.makedirs(stagingDir)
    os.makedirs(dataDir)
    os.makedirs(configDir)

    with open(loggingFile, "w") as textFile:
        print(logging,file=textFile)

def runNodeos(extraNodeosArgs, myTimeout):
    """Startup nodeos, wait for timeout (before forced shutdown) and collect output."""
    if debug: Print("Launching nodeos process.")
    cmd="programs/nodeos/nodeos --config-dir rsmStaging/etc -e -p eosio --plugin eosio::chain_api_plugin --plugin eosio::history_api_plugin --data-dir " + dataDir + " "

    cmd=cmd + extraNodeosArgs;
    if debug: Print("cmd: %s" % (cmd))
    with open(stderrFile, 'w') as serr:
        proc=subprocess.Popen(cmd.split(), stdout=subprocess.PIPE, stderr=serr)

        try:
            proc.communicate(timeout=myTimeout)
        except (subprocess.TimeoutExpired) as _:
            if debug: Print("Timed out\n")
            proc.send_signal(signal.SIGKILL)

def isMsgInStderrFile(msg):
    msgFound=False
    with open(stderrFile) as errFile:
        for line in errFile:
            if msg in line:
                msgFound=True
                break
    return msgFound

def testCommon(title, extraNodeosArgs, expectedMsgs):
    global testNum
    testNum+=1
    Print("Test %d: %s" % (testNum, title))

    prepareDirectories()

    timeout=120  # Leave sufficient time such nodeos can start up fully in any platforms
    runNodeos(extraNodeosArgs, timeout)

    for msg in expectedMsgs:
        if not isMsgInStderrFile(msg):
            errorExit ("Log should have contained \"%s\"" % (expectedMsgs))

def extractTimestamp(msg):
    matches = re.compile("\s+([0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{3})\s").search(msg)
    return datetime.strptime(matches.group(0).strip(), '%Y-%m-%dT%H:%M:%S.%f')

maxFirstIntervalTolerance = 5
maxIntervalTolerance = 0.5

def isMsgIntervalValid(msg, interval):
    pre = ""
    hasMsg = validInterval = False
    firstInterval = True
    with open(stderrFile) as errFile:
        for line in errFile:
            if msg in line:
                Print(line)
                hasMsg = True
                if pre:
                    curInterval = extractTimestamp(line) - extractTimestamp(pre)
                    Print(curInterval)
                    if firstInterval:
                        intervalTolerance = maxFirstIntervalTolerance
                        firstInterval = False
                    else:
                        intervalTolerance = maxIntervalTolerance
                    if abs((curInterval - timedelta(seconds=interval)).total_seconds()) <= intervalTolerance:
                        validInterval = True
                    else:
                        validInterval = False
                        break
                pre = line
    return hasMsg, validInterval

{'fraction_non_alphanumeric': 0.116751872547854, 'fraction_numerical': 0.007371299488764713, 'mean_word_length': 2.3394204049225884, 'pattern_counts': {'":': 40, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '915552', 'n_tokens_mistral': 2558, 'n_tokens_neox': 2465, 'n_words': 610}