<commit_before>from pyiso import client_factory, BALANCING_AUTHORITIES
from pyiso.base import BaseClient
from unittest import TestCase
import pytz
from datetime import datetime, timedelta
from nose_parameterized import parameterized


class TestBaseLMP(TestCase):
    def setUp(self):
        # set up expected values from base client
        bc = BaseClient()
        self.MARKET_CHOICES = bc.MARKET_CHOICES
        self.FREQUENCY_CHOICES = bc.FREQUENCY_CHOICES

        # set up other expected values
        self.BA_CHOICES = BALANCING_AUTHORITIES.keys()

    def _run_test(self, ba_name, expect_data=True, tol_min=8, **kwargs):
        # set up
        c = client_factory(ba_name)

        # get data
        data = c.get_lmp(**kwargs)

        # test number
        if expect_data:
            self.assertGreaterEqual(len(data), 1)
        else:
            self.assertEqual(data, [])
            return data

        # test contents
        for dp in data:
            # test key names
            self.assertEqual(set(['lmp', 'ba_name',
                                  'timestamp', 'market', 'node_id', 'lmp_type', 'freq']),
                             set(dp.keys()))

            # test values
            self.assertEqual(dp['timestamp'].tzinfo, pytz.utc)
            self.assertIn(dp['ba_name'], self.BA_CHOICES)
            self.assertIn(dp['lmp_type'],
                          ['LMP', 'prc', 'energy', 'TotalLMP'])

            # test for numeric price
            self.assertGreaterEqual(dp['lmp']+1, dp['lmp'])

        # test correct temporal relationship to now
        timestamps = [t['timestamp'] for t in data]
        now = pytz.utc.localize(datetime.utcnow())
        if c.options['forecast']:
            self.assertGreaterEqual(max(timestamps), now - timedelta(hours=1))
        elif c.options['latest']:
            tset = list(set(timestamps))
            self.assertEqual(len(tset), 1)
            # within 8 min
            delta = now - tset[0]
            self.assertLess(abs(delta.total_seconds()), tol_min*60)
        else:
            if 'end_at' in kwargs and kwargs['end_at']:
                self.assertLess(max(timestamps), kwargs['end_at'] + timedelta(hours=1))
            else:
                self.assertLess(max(timestamps), now)

        # return
        return data

    def _run_notimplemented_test(self, ba_name, **kwargs):
        # set up
        c = client_factory(ba_name)

        # method not implemented yet
        self.assertRaises(NotImplementedError, c.get_lmp)


class TestCAISOLMP(TestBaseLMP):
    def test_latest(self):
        # basic test
        data = self._run_test('CAISO', node_id='SLAP_PGP2-APND',
                              latest=True)

        # test all timestamps are equal
        timestamps = [d['timestamp'] for d in data]
        self.assertEqual(len(set(timestamps)), 1)

        # test flags
        for dp in data:
            self.assertEqual(dp['market'], self.MARKET_CHOICES.fivemin)
            self.assertEqual(dp['freq'], self.FREQUENCY_CHOICES.fivemin)

    def date_range(self, market):
        # basic test
        today = datetime.today().replace(tzinfo=pytz.utc)
        data = self._run_test('CAISO', node_id='SLAP_PGP2-APND',
                              start_at=today-timedelta(days=2),
                              end_at=today-timedelta(days=1),
                              market=market)

        # test timestamps are not equal
        timestamps = [d['timestamp'] for d in data]
        self.assertGreater(len(set(timestamps)), 1)
        self.assertEqual(list(set([d['market'] for d in data])),
                         [market])
        return data

    def test_date_range_rtm(self):
        data = self.date_range(self.MARKET_CHOICES.fivemin)
        self.assertGreaterEqual(len(data), 12*23)
        self.assertLessEqual(len(data), 12*24)

    def test_date_range_dam(self):
        data = self.date_range(self.MARKET_CHOICES.dam)
        self.assertEqual(len(data), 24)

    def test_date_range_hourly(self):
        data = self.date_range(self.MARKET_CHOICES.hourly)
        self.assertGreaterEqual(len(data), 92)
        self.assertLessEqual(len(data), 96)

    def test_date_range_rtpd(self):
        data = self.date_range(self.MARKET_CHOICES.fifteenmin)
        self.assertEqual(len(data), 24*4)

    def test_forecast(self):
        # basic test
        now = pytz.utc.localize(datetime.utcnow())
        data = self._run_test('CAISO', node_id='SLAP_PGP2-APND',
                              start_at=now,
                              end_at=now+timedelta(days=1),
                              market=self.MARKET_CHOICES.dam)

        # test timestamps are not equal
        timestamps = [d['timestamp'] for d in data]
        self.assertGreater(len(set(timestamps)), 1)

    def test_bad_node(self):
        self._run_test('CAISO', node_id='badnode', expect_data=False, latest=True)
<commit_msg>fix caiso lmp test<commit_after>from pyiso import client_factory, BALANCING_AUTHORITIES
from pyiso.base import BaseClient
from unittest import TestCase
import pytz
from datetime import datetime, timedelta
from nose_parameterized import parameterized


class TestBaseLMP(TestCase):
    def setUp(self):
        # set up expected values from base client
        bc = BaseClient()
        self.MARKET_CHOICES = bc.MARKET_CHOICES
        self.FREQUENCY_CHOICES = bc.FREQUENCY_CHOICES

        # set up other expected values
        self.BA_CHOICES = BALANCING_AUTHORITIES.keys()

    def _run_test(self, ba_name, expect_data=True, tol_min=8, **kwargs):
        # set up
        c = client_factory(ba_name)

        # get data
        data = c.get_lmp(**kwargs)

        # test number
        if expect_data:
            self.assertGreaterEqual(len(data), 1)
        else:
            self.assertEqual(data, [])
            return data

        # test contents
        for dp in data:
            # test key names
            self.assertEqual(set(['lmp', 'ba_name',
                                  'timestamp', 'market', 'node_id', 'lmp_type', 'freq']),
                             set(dp.keys()))

            # test values
            self.assertEqual(dp['timestamp'].tzinfo, pytz.utc)
            self.assertIn(dp['ba_name'], self.BA_CHOICES)
            self.assertIn(dp['lmp_type'],
                          ['LMP', 'prc', 'energy', 'TotalLMP'])

            # test for numeric price
            self.assertGreaterEqual(dp['lmp']+1, dp['lmp'])

        # test correct temporal relationship to now
        timestamps = [t['timestamp'] for t in data]
        now = pytz.utc.localize(datetime.utcnow())
        if c.options['forecast']:
            self.assertGreaterEqual(max(timestamps), now - timedelta(hours=1))
        elif c.options['latest']:
            tset = list(set(timestamps))
            self.assertEqual(len(tset), 1)
            # within 8 min
            delta = now - tset[0]
            self.assertLess(abs(delta.total_seconds()), tol_min*60)
        else:
            if 'end_at' in kwargs and kwargs['end_at']:
                self.assertLess(max(timestamps), kwargs['end_at'] + timedelta(hours=1))
            else:
                self.assertLess(max(timestamps), now)

        # return
        return data

    def _run_notimplemented_test(self, ba_name, **kwargs):
        # set up
        c = client_factory(ba_name)

        # method not implemented yet
        self.assertRaises(NotImplementedError, c.get_lmp)


class TestCAISOLMP(TestBaseLMP):
    def test_latest(self):
        # basic test
        data = self._run_test('CAISO', node_id='SLAP_PGP2-APND',
                              latest=True)

        # test all timestamps are equal
        timestamps = [d['timestamp'] for d in data]
        self.assertEqual(len(set(timestamps)), 1)

        # test flags
        for dp in data:
            self.assertEqual(dp['market'], self.MARKET_CHOICES.fivemin)
            self.assertEqual(dp['freq'], self.FREQUENCY_CHOICES.fivemin)

    def date_range(self, market):
        # basic test
        today = datetime.today().replace(tzinfo=pytz.utc)
        data = self._run_test('CAISO', node_id='SLAP_PGP2-APND',
                              start_at=today-timedelta(days=2),
                              end_at=today-timedelta(days=1),
                              market=market)

        # test timestamps are not equal
        timestamps = [d['timestamp'] for d in data]
        self.assertGreater(len(set(timestamps)), 1)
        self.assertEqual(list(set([d['market'] for d in data])),
                         [market])
        return data

    def test_date_range_rtm(self):
        data = self.date_range(self.MARKET_CHOICES.fivemin)
        self.assertGreaterEqual(len(data), 12*23)
        self.assertLessEqual(len(data), 12*24)

    def test_date_range_dam(self):
        data = self.date_range(self.MARKET_CHOICES.dam)
        self.assertEqual(len(data), 24)

    def test_date_range_hourly(self):
        data = self.date_range(self.MARKET_CHOICES.hourly)
        self.assertGreaterEqual(len(data), 92)
        self.assertLessEqual(len(data), 96)

    def test_date_range_rtpd(self):
        data = self.date_range(self.MARKET_CHOICES.fifteenmin)
        self.assertIn(len(data), [24*4, 24*4-1])

    def test_forecast(self):
        # basic test
        now = pytz.utc.localize(datetime.utcnow())
        data = self._run_test('CAISO', node_id='SLAP_PGP2-APND',
                              start_at=now,
                              end_at=now+timedelta(days=1),
                              market=self.MARKET_CHOICES.dam)

        # test timestamps are not equal
        timestamps = [d['timestamp'] for d in data]
        self.assertGreater(len(set(timestamps)), 1)

    def test_bad_node(self):
        self._run_test('CAISO', node_id='badnode', expect_data=False, latest=True)

{'fraction_non_alphanumeric': 0.10965713126394806, 'fraction_numerical': 0.007709474538445932, 'mean_word_length': 2.1298412698412696, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7345978', 'n_tokens_mistral': 3160, 'n_tokens_neox': 3016, 'n_words': 594}