<commit_before>#!/usr/bin/python3

from fnmatch import fnmatch
from collections import namedtuple
import os

from utils import ERROR_COLOR, NO_COLOR


class RequiredFieldError(Exception):
    pass


class UnknownFieldError(Exception):
    pass


TestStruct = namedtuple('TestStruct', 'name run expect timeout before after suite kernel requirement env')


class TestParser(object):
    @staticmethod
    def read_all(test_filter):
        try:
            for root, subdirs, files in os.walk('./runtime'):
                for ignore_dir in ["scripts", "outputs"]:
                    if ignore_dir in subdirs:
                        subdirs.remove(ignore_dir)
                for filename in files:
                    if filename.startswith("."):
                        continue
                    parser = TestParser.read(root + '/' + filename, test_filter)
                    if parser[1]:
                        yield parser
        except RequiredFieldError as error:
            print(ERROR_COLOR + str(error) + NO_COLOR)

    @staticmethod
    def read(file_name, test_filter):
        tests = []
        test_lines = []
        test_suite = file_name.split('/')[-1]
        with open (file_name, 'r') as file:
            for line in file.readlines():
                if line.startswith("#"):
                    continue
                if line != '\n':
                    test_lines.append(line)
                else:
                    test_struct = TestParser.__read_test_struct(test_lines, test_suite)
                    if fnmatch("%s.%s" % (test_suite, test_struct.name), test_filter):
                        tests.append(test_struct)
                    test_lines = []
            test_struct = TestParser.__read_test_struct(test_lines, test_suite)
            if fnmatch("%s.%s" % (test_suite, test_struct.name), test_filter):
                tests.append(test_struct)

        return (test_suite, tests)

    @staticmethod
    def __read_test_struct(test, test_suite):
        name = ''
        run = ''
        expect = ''
        timeout = ''
        before = ''
        after = ''
        kernel = ''
        requirement = ''
        env = {}

        for item in test:
            item_split = item.split()
            item_name = item_split[0]
            line = ' '.join(item_split[1:])
<commit_msg>Fix runtime test parser to not break with commented out tests (#824)<commit_after>#!/usr/bin/python3

from fnmatch import fnmatch
from collections import namedtuple
import os

from utils import ERROR_COLOR, NO_COLOR


class RequiredFieldError(Exception):
    pass


class UnknownFieldError(Exception):
    pass


TestStruct = namedtuple('TestStruct', 'name run expect timeout before after suite kernel requirement env')


class TestParser(object):
    @staticmethod
    def read_all(test_filter):
        try:
            for root, subdirs, files in os.walk('./runtime'):
                for ignore_dir in ["scripts", "outputs"]:
                    if ignore_dir in subdirs:
                        subdirs.remove(ignore_dir)
                for filename in files:
                    if filename.startswith("."):
                        continue
                    parser = TestParser.read(root + '/' + filename, test_filter)
                    if parser[1]:
                        yield parser
        except RequiredFieldError as error:
            print(ERROR_COLOR + str(error) + NO_COLOR)

    @staticmethod
    def read(file_name, test_filter):
        tests = []
        test_lines = []
        test_suite = file_name.split('/')[-1]
        with open (file_name, 'r') as file:
            lines = file.readlines()
            line_num = 0
            for line in lines:
                line_num += 1
                if line.startswith("#"):
                    continue
                if line != '\n':
                    test_lines.append(line)
                if line == '\n' or line_num == len(lines):
                    if test_lines:
                        test_struct = TestParser.__read_test_struct(test_lines, test_suite)
                        if fnmatch("%s.%s" % (test_suite, test_struct.name), test_filter):
                            tests.append(test_struct)
                        test_lines = []

        return (test_suite, tests)

    @staticmethod
    def __read_test_struct(test, test_suite):
        name = ''
        run = ''
        expect = ''
        timeout = ''
        before = ''
        after = ''
        kernel = ''
        requirement = ''
        env = {}

        for item in test:
            item_split = item.split()
            item_name = item_split[0]
            line = ' '.join(item_split[1:])

{'fraction_non_alphanumeric': 0.09622359718369959, 'fraction_numerical': 0.0032003413697461063, 'mean_word_length': 1.621923937360179, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6388356', 'n_tokens_mistral': 1270, 'n_tokens_neox': 1209, 'n_words': 313}