<commit_before>import os
import os.path
import sys
import stat
import glob
import ConfigParser
import subprocess
import collections
import difflib
import filecmp
import shlex
import time
from tarantool_silverbox_server import TarantoolSilverboxServer
from tarantool_connection import AdminConnection, DataConnection
import tarantool_preprocessor
import re
import cStringIO
import string

class TestRunException(RuntimeError):
  """A common exception to use across the program."""
  def __init__(self, message):
    self.message = message
  def __str__(self):
    return self.message

class FilteredStream:
  """Helper class to filter .result file output"""
  def __init__(self, filename):
    self.stream = open(filename, "w+")
    self.filters = []
  def write(self, fragment):
    """Apply all filters, then write result to the undelrying stream.
    Do line-oriented filtering: the fragment doesn't have to represent
    just one line."""
    fragment_stream = cStringIO.StringIO(fragment)
    for line in fragment_stream:
      original_len = len(line.strip())
      for pattern, replacement in self.filters:
        line = re.sub(pattern, replacement, line)
# don't write lines that are completely filtered out:
        if original_len and len(line.strip()) == 0:
          return
      self.stream.write(line)
  def push_filter(self, pattern, replacement):
    self.filters.append([pattern, replacement])
  def pop_filter(self):
    self.filters.pop()
  def clear_all_filters(self):
    filters = []
  def close(self):
    self.clear_all_filters()
    self.stream.close()


class Test:
  """An individual test file. A test can run itself, and remembers
  its completion state."""
  def __init__(self, name, args, suite_ini):
    """Initialize test properties: path to test file, path to
    temporary result file, path to the client program, test status."""
    self.name = name
    self.result = name.replace(".test", ".result")
    self.reject = name.replace(".test", ".reject")
    self.valgrind_log = args.vardir + "/valgrind.log"
    self.args = args
    self.suite_ini = suite_ini
    self.is_executed = False
    self.is_executed_ok = None
    self.is_equal_result = None
    self.is_valgrind_clean = True

  def passed(self):
    """Return true if this test was run successfully."""
    return self.is_executed and self.is_executed_ok and self.is_equal_result

  def run(self):
    """Execute the test assuming it's a python program.
    If the test aborts, print its output to stdout, and raise
    an exception. Else, comprare result and reject files.
    If there is a difference, print it to stdout and raise an
    exception. The exception is raised only if is_force flag is
    not set."""

    sys.stdout.write(string.ljust(self.name, 31))
# for better diagnostics in case of a long-running test
    sys.stdout.flush()

    diagnostics = "unknown"
    save_stdout = sys.stdout
    admin = AdminConnection(self.suite_ini["host"],
                            self.suite_ini["admin_port"])
    sql = DataConnection(self.suite_ini["host"],
                         self.suite_ini["port"])
    server = self.suite_ini["server"]
    try:
      admin.connect()
      sql.connect()
      sys.stdout = FilteredStream(self.reject)
      server = self.suite_ini["server"]
      execfile(self.name, globals(), locals())
      self.is_executed_ok = True
    except Exception as e:
      print e
      diagnostics = str(e)
    finally:
      if sys.stdout and sys.stdout != save_stdout:
        sys.stdout.close()
      sys.stdout = save_stdout;
      admin.disconnect()
      sql.disconnect()

    self.is_executed = True

    if self.is_executed_ok and os.path.isfile(self.result):
        self.is_equal_result = filecmp.cmp(self.result, self.reject)

    if self.args.valgrind:
      self.is_valgrind_clean = os.path.getsize(self.valgrind_log) == 0

    if self.is_executed_ok and self.is_equal_result and self.is_valgrind_clean:
      print "[ pass ]"
      os.remove(self.reject)
    elif (self.is_executed_ok and not self.is_equal_result and not
        os.path.isfile(self.result)):
      os.rename(self.reject, self.result)
      print "[ NEW ]"
    else:
      print "[ fail ]"
      where = ""
      if not self.is_executed_ok:
        self.print_diagnostics()
        where = ": test execution aborted, reason '{0}'".format(diagnostics)
      elif not self.is_valgrind_clean:
        print "Test failed! Valgrind reports errors" \
	      " (see {0}/valgrind.log)".format(self.args.vardir)
      else:
        self.print_unidiff()
        where = ": wrong test output"
      if not self.suite_ini["is_force"]:
        raise TestRunException("Failed to run test " + self.name + where)


  def print_diagnostics(self):
    """Print 10 lines of client program output leading to test
    failure. Used to diagnose a failure of the client program"""

    print "Test failed! Last 10 lines of the result file:"
    with open(self.reject, "r+") as reject:
      tail_10 = collections.deque(reject, 10)
      for line in tail_10:
        sys.stdout.write(line)<commit_msg>[test-runner] Always check for valgrind errors.<commit_after>import os
import os.path
import sys
import stat
import glob
import ConfigParser
import subprocess
import collections
import difflib
import filecmp
import shlex
import time
from tarantool_silverbox_server import TarantoolSilverboxServer
from tarantool_connection import AdminConnection, DataConnection
import tarantool_preprocessor
import re
import cStringIO
import string

class TestRunException(RuntimeError):
  """A common exception to use across the program."""
  def __init__(self, message):
    self.message = message
  def __str__(self):
    return self.message

class FilteredStream:
  """Helper class to filter .result file output"""
  def __init__(self, filename):
    self.stream = open(filename, "w+")
    self.filters = []
  def write(self, fragment):
    """Apply all filters, then write result to the undelrying stream.
    Do line-oriented filtering: the fragment doesn't have to represent
    just one line."""
    fragment_stream = cStringIO.StringIO(fragment)
    for line in fragment_stream:
      original_len = len(line.strip())
      for pattern, replacement in self.filters:
        line = re.sub(pattern, replacement, line)
# don't write lines that are completely filtered out:
        if original_len and len(line.strip()) == 0:
          return
      self.stream.write(line)
  def push_filter(self, pattern, replacement):
    self.filters.append([pattern, replacement])
  def pop_filter(self):
    self.filters.pop()
  def clear_all_filters(self):
    filters = []
  def close(self):
    self.clear_all_filters()
    self.stream.close()


class Test:
  """An individual test file. A test can run itself, and remembers
  its completion state."""
  def __init__(self, name, args, suite_ini):
    """Initialize test properties: path to test file, path to
    temporary result file, path to the client program, test status."""
    self.name = name
    self.result = name.replace(".test", ".result")
    self.reject = name.replace(".test", ".reject")
    self.valgrind_log = args.vardir + "/valgrind.log"
    self.args = args
    self.suite_ini = suite_ini
    self.is_executed = False
    self.is_executed_ok = None
    self.is_equal_result = None
    self.is_valgrind_clean = True

  def passed(self):
    """Return true if this test was run successfully."""
    return self.is_executed and self.is_executed_ok and self.is_equal_result

  def run(self):
    """Execute the test assuming it's a python program.
    If the test aborts, print its output to stdout, and raise
    an exception. Else, comprare result and reject files.
    If there is a difference, print it to stdout and raise an
    exception. The exception is raised only if is_force flag is
    not set."""

    sys.stdout.write(string.ljust(self.name, 31))
# for better diagnostics in case of a long-running test
    sys.stdout.flush()

    diagnostics = "unknown"
    save_stdout = sys.stdout
    admin = AdminConnection(self.suite_ini["host"],
                            self.suite_ini["admin_port"])
    sql = DataConnection(self.suite_ini["host"],
                         self.suite_ini["port"])
    server = self.suite_ini["server"]
    try:
      admin.connect()
      sql.connect()
      sys.stdout = FilteredStream(self.reject)
      server = self.suite_ini["server"]
      execfile(self.name, globals(), locals())
      self.is_executed_ok = True
    except Exception as e:
      print e
      diagnostics = str(e)
    finally:
      if sys.stdout and sys.stdout != save_stdout:
        sys.stdout.close()
      sys.stdout = save_stdout;
      admin.disconnect()
      sql.disconnect()

    self.is_executed = True

    if self.is_executed_ok and os.path.isfile(self.result):
        self.is_equal_result = filecmp.cmp(self.result, self.reject)

    if self.args.valgrind:
      self.is_valgrind_clean = os.path.getsize(self.valgrind_log) == 0

    if self.is_executed_ok and self.is_equal_result and self.is_valgrind_clean:
      print "[ pass ]"
      os.remove(self.reject)
    elif (self.is_executed_ok and not self.is_equal_result and not
        os.path.isfile(self.result)):
      os.rename(self.reject, self.result)
      print "[ NEW ]"
    else:
      print "[ fail ]"
      where = ""
      if not self.is_executed_ok:
        self.print_diagnostics()
        where = ": test execution aborted, reason '{0}'".format(diagnostics)
      elif not self.is_equal_result:
        self.print_unidiff()
        where = ": wrong test output"
      if not self.is_valgrind_clean:
        print "Test failed! Valgrind reports errors" \
	      " (see {0}/valgrind.log)".format(self.args.vardir)
      if not self.suite_ini["is_force"]:
        raise TestRunException("Failed to run test " + self.name + where)


  def print_diagnostics(self):
    """Print 10 lines of client program output leading to test
    failure. Used to diagnose a failure of the client program"""

    print "Test failed! Last 10 lines of the result file:"
    with open(self.reject, "r+") as reject:
      tail_10 = collections.deque(reject, 10)
      for line in tail_10:
        sys.stdout.write(line)
{'fraction_non_alphanumeric': 0.09912907329484294, 'fraction_numerical': 0.003131421861238869, 'mean_word_length': 3.112676056338028, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1840880', 'n_tokens_mistral': 3076, 'n_tokens_neox': 3007, 'n_words': 920}