<commit_before>from datetime import datetime
import unittest

from apel.parsers import SlurmParser
from apel.parsers.slurm import parse_local_timestamp


class ParserSlurmTest(unittest.TestCase):
    '''
    Test case for SLURM parser
    '''

    def setUp(self):
        self.parser = SlurmParser('testSite', 'testHost', True)

    def test_parse_local_timestamp(self):
        '''
        The output of this function depends on the timezone of the testing
        computer.
        '''
        # Not yet implemented
        pass

    def test_parse_line(self):

        line1 = ('1000|cream_176801680|dteam005|dteam|2013-03-27T17:13:24|2013-03-27T17:13:26|00:00:02|2|prod|1|1|cert-40|||COMPLETED')

        line1_values = {"JobName": "1000",
                        "LocalUserID": "dteam005",
                        "LocalUserGroup": "dteam",
                        "WallDuration": 2,
                        "CpuDuration": 2,
                        "StartTime": datetime(2013, 3, 27, 17, 13, 24),
                        "StopTime": datetime(2013, 3, 27, 17, 13, 26),
                        "MemoryReal": None,
                        "MemoryVirtual": None,
                        "NodeCount": 1,
                        "Processors": 1
                        }

        cases = {line1: line1_values}

        for line in cases.keys():
            record = self.parser.parse(line)
            cont = record._record_content

            self.assertTrue("Site" in cont)
            self.assertTrue("JobName" in cont)
            self.assertTrue("LocalUserID" in cont)
            self.assertTrue("LocalUserGroup" in cont)
            self.assertTrue("WallDuration" in cont)
            self.assertTrue("CpuDuration" in cont)
            self.assertTrue("StartTime" in cont)
            self.assertTrue("StopTime" in cont)
            self.assertTrue("MemoryReal" in cont)
            self.assertTrue("MemoryReal" in cont)

            for key in cases[line].keys():
                self.assertEqual(cont[key], cases[line][key], "%s != %s for key %s" % (cont[key], cases[line][key], key))

if __name__ == '__main__':
    unittest.main()
<commit_msg>Add more test strings to test_slurm.<commit_after>from datetime import datetime
import unittest

from apel.parsers import SlurmParser
from apel.parsers.slurm import parse_local_timestamp


class ParserSlurmTest(unittest.TestCase):
    '''
    Test case for SLURM parser
    '''

    def setUp(self):
        self.parser = SlurmParser('testSite', 'testHost', True)

    def test_parse_local_timestamp(self):
        '''
        The output of this function depends on the timezone of the testing
        computer.
        '''
        # Not yet implemented
        pass

    def test_parse_line(self):

        line1 = ('1000|cream_176801680|dteam005|dteam|2013-03-27T17:13:24|2013-03-27T17:13:26|00:00:02|2|prod|1|1|cert-40|||COMPLETED')
        line2 = ('278952.batch|batch|||2013-10-23T21:37:24|2013-10-25T00:01:37|1-02:24:13|95053||1|1|wn36|438.50M|1567524K|COMPLETED')
        line3 = ('297720.batch|batch|||2013-10-25T12:11:20|2013-10-25T12:11:36|00:00:16|16||1|1|wn37|3228K|23820K|COMPLETED')

        line1_values = {"JobName": "1000",
                        "LocalUserID": "dteam005",
                        "LocalUserGroup": "dteam",
                        "WallDuration": 2,
                        "CpuDuration": 2,
                        "StartTime": datetime(2013, 3, 27, 17, 13, 24),
                        "StopTime": datetime(2013, 3, 27, 17, 13, 26),
                        "MemoryReal": None,
                        "MemoryVirtual": None,
                        "NodeCount": 1,
                        "Processors": 1
                        }

        line2_values = {"JobName": "278952.batch",
                        "LocalUserID": None,
                        "LocalUserGroup": None,
                        "WallDuration": 95053,
                        "CpuDuration": 95053,
                        "StartTime": datetime(2013, 10, 23, 21, 37, 24),
                        "StopTime": datetime(2013, 10, 25, 00, 01, 37),
                        "MemoryReal": 438500,
                        "MemoryVirtual": 1567524,
                        "NodeCount": 1,
                        "Processors": 1
                        }

        line3_values = {"JobName": "297720.batch",
                        "LocalUserID": None,
                        "LocalUserGroup": None,
                        "WallDuration": 16,
                        "CpuDuration": 16,
                        "StartTime": datetime(2013, 10, 25, 12, 11, 20),
                        "StopTime": datetime(2013, 10, 25, 12, 11, 36),
                        "MemoryReal": 3228,
                        "MemoryVirtual": 23820,
                        "NodeCount": 1,
                        "Processors": 1
                        }

        cases = {line1: line1_values,
                 line2: line2_values,
                 line3: line3_values}

        for line in cases.keys():
            record = self.parser.parse(line)
            cont = record._record_content

            self.assertTrue("Site" in cont)
            self.assertTrue("JobName" in cont)
            self.assertTrue("LocalUserID" in cont)
            self.assertTrue("LocalUserGroup" in cont)
            self.assertTrue("WallDuration" in cont)
            self.assertTrue("CpuDuration" in cont)
            self.assertTrue("StartTime" in cont)
            self.assertTrue("StopTime" in cont)
            self.assertTrue("MemoryReal" in cont)
            self.assertTrue("MemoryReal" in cont)

            for key in cases[line].keys():
                self.assertEqual(cont[key], cases[line][key], "%s != %s for key %s" % (cont[key], cases[line][key], key))

if __name__ == '__main__':
    unittest.main()

{'fraction_non_alphanumeric': 0.11779835390946503, 'fraction_numerical': 0.07287379972565158, 'mean_word_length': 1.6549840691852526, 'pattern_counts': {'":': 44, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1311956', 'n_tokens_mistral': 1958, 'n_tokens_neox': 1641, 'n_words': 363}