<commit_before>#!/usr/bin/env python

import unittest, os
from subprocess import Popen, PIPE
from src.redisconnector import RedisConnectionPool, RedisConnector
from src.scheduler import Scheduler, Link
from src.job import Job
from unittest_constants import RedisConst
from unittest_job import JobSentinel

class SchedulerTestSuite(unittest.TestCase):
    def setUp(self):
        test_pool = RedisConnectionPool({
            "host": RedisConst.redisHost(),
            "port": RedisConst.redisPort(),
            "db": RedisConst.redisTestDb()
        })
        self.connector = RedisConnector(test_pool)
        self.connector.flushdb()
        self.settings = {
            "REDIS_HOST": RedisConst.redisHost(),
            "REDIS_PORT": RedisConst.redisPort(),
            "REDIS_DB": RedisConst.redisTestDb(),
            "SPARK_UI_ADDRESS": "http://localhost:8080",
            "SPARK_UI_RUN_ADDRESS": "http://localhost:4040",
            "SPARK_MASTER_ADDRESS": "spark://localhost:7077"
        }

    def tearDown(self):
        self.connector = None

    def test_init(self):
        with self.assertRaises(StandardError):
            scheduler = Scheduler({})
        scheduler = Scheduler(self.settings)
        self.assertEqual(scheduler.poolSize, 5)
        self.assertEqual(scheduler.isRunning, False)
        self.assertEqual(scheduler.forceSparkMasterAddress, False)
        # update settings to include "force-master-address" option
        self.settings["FORCE_SPARK_MASTER_ADDRESS"] = True
        scheduler = Scheduler(self.settings)
        self.assertEqual(scheduler.forceSparkMasterAddress, True)

    def test_fetchStatus(self):
        job1 = JobSentinel.job()
        job2 = JobSentinel.job()
        # set entry point
        now = job1.createtime
        status = job1.status
        job1.submittime = now + 100
        job2.submittime = now + 30
        job1.createtime = now
        job2.createtime = now + 10
        scheduler = Scheduler(self.settings)
        scheduler.updateJob(job1, status, job1.priority)
        scheduler.updateJob(job2, status, job2.priority)
        jobs = scheduler.jobsForStatus(status, -1)
        self.assertEqual(len(jobs), 2)
        self.assertEqual([x.uid for x in jobs], [job2.uid, job1.uid])

    def test_add(self):
        job1 = JobSentinel.job()
        job2 = JobSentinel.job()<commit_msg>test fix<commit_after>#!/usr/bin/env python

import unittest, os
from subprocess import Popen, PIPE
from src.redisconnector import RedisConnectionPool, RedisConnector
from src.scheduler import Scheduler, Link
from src.job import Job
from unittest_constants import RedisConst
from unittest_job import JobSentinel

class SchedulerTestSuite(unittest.TestCase):
    def setUp(self):
        test_pool = RedisConnectionPool({
            "host": RedisConst.redisHost(),
            "port": RedisConst.redisPort(),
            "db": RedisConst.redisTestDb()
        })
        self.connector = RedisConnector(test_pool)
        self.connector.flushdb()
        self.settings = {
            "REDIS_HOST": RedisConst.redisHost(),
            "REDIS_PORT": RedisConst.redisPort(),
            "REDIS_DB": RedisConst.redisTestDb(),
            "SPARK_UI_ADDRESS": "http://localhost:8080",
            "SPARK_UI_RUN_ADDRESS": "http://localhost:4040",
            "SPARK_MASTER_ADDRESS": "spark://localhost:7077"
        }

    def tearDown(self):
        self.connector = None

    def test_init(self):
        with self.assertRaises(StandardError):
            scheduler = Scheduler({})
        scheduler = Scheduler(self.settings)
        # check default pool size
        self.assertEqual(scheduler.poolSize, 3)
        self.assertEqual(scheduler.isRunning, False)
        self.assertEqual(scheduler.forceSparkMasterAddress, False)
        # update settings to include "force-master-address" option
        self.settings["FORCE_SPARK_MASTER_ADDRESS"] = True
        scheduler = Scheduler(self.settings, 5)
        self.assertEqual(scheduler.poolSize, 5)
        self.assertEqual(scheduler.forceSparkMasterAddress, True)

    def test_fetchStatus(self):
        job1 = JobSentinel.job()
        job2 = JobSentinel.job()
        # set entry point
        now = job1.createtime
        status = job1.status
        job1.submittime = now + 100
        job2.submittime = now + 30
        job1.createtime = now
        job2.createtime = now + 10
        scheduler = Scheduler(self.settings)
        scheduler.updateJob(job1, status, job1.priority)
        scheduler.updateJob(job2, status, job2.priority)
        jobs = scheduler.jobsForStatus(status, -1)
        self.assertEqual(len(jobs), 2)
        self.assertEqual([x.uid for x in jobs], [job2.uid, job1.uid])

    def test_add(self):
        job1 = JobSentinel.job()
        job2 = JobSentinel.job()
{'fraction_non_alphanumeric': 0.10262281432139883, 'fraction_numerical': 0.01623646960865945, 'mean_word_length': 2.6904761904761907, 'pattern_counts': {'":': 18, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '329339', 'n_tokens_mistral': 1477, 'n_tokens_neox': 1422, 'n_words': 287}