<commit_before>import os
import numpy as np
import pandas as pd
from desc.twinkles import OpSimOrdering
from lsst.utils import getPackageDir
import unittest

class TestOpSimOrdering(unittest.TestCase):

    def setUp(self):
        self.opSimDBPath = '/Users/rbiswas/data/LSST/OpSimData/minion_1016_sqlite.db'
        self.ops = OpSimOrdering(self.opSimDBPath, timeMax=0.8)
        self.numRecords = len(self.ops.filteredOpSim)
        self.numUniqueRecords = len(self.ops.uniqueOpSimRecords)

    def test_numRecords(self):
        """
        Given the setup where no records are dropped due to predictedPhoSimTime
        being too long, show that the filtered OpSim records (in favor of lower
        propID belonging to the OpSim WFD proposals has the same number of records
        as dropping duplicates.
        """
        pts = self.ops.fullOpSimDF(self.opSimDBPath)
        pts.drop_duplicates(subset='obsHistID', inplace=True)
        self.assertEqual(len(pts), self.numUniqueRecords)
        self.assertGreaterEqual(self.numUniqueRecords, self.numRecords)

    def test_conservedNumRecordsInSplit(self):<commit_msg>Modified tests for new `OpSimOrdering` API 	modified:   tests/test_opSimOrdering.py<commit_after>import os
import numpy as np
import pandas as pd
from desc.twinkles import OpSimOrdering
from lsst.utils import getPackageDir
import unittest

class TestOpSimOrdering(unittest.TestCase):

    def setUp(self):
        self.opSimDBPath = '/Users/rbiswas/data/LSST/OpSimData/minion_1016_sqlite.db'
        self.ops = OpSimOrdering(self.opSimDBPath, timeMax=100., randomForestPickle=None)
        self.numRecords = len(self.ops.filteredOpSim)
        self.numUniqueRecords = len(self.ops.uniqueOpSimRecords)

    def test_numRecords(self):
        """
        Given the setup where no records are dropped due to predictedPhoSimTime
        being too long, show that the filtered OpSim records (in favor of lower
        propID belonging to the OpSim WFD proposals has the same number of records
        as dropping duplicates.
        """
        pts = self.ops.fullOpSimDF(self.opSimDBPath)
        pts.drop_duplicates(subset='obsHistID', inplace=True)
        self.assertEqual(len(pts), self.numUniqueRecords)
        self.assertGreaterEqual(self.numUniqueRecords, self.numRecords)

    def test_conservedNumRecordsInSplit(self):
{'fraction_non_alphanumeric': 0.07525510204081633, 'fraction_numerical': 0.0055272108843537416, 'mean_word_length': 3.595703125, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4525237', 'n_tokens_mistral': 723, 'n_tokens_neox': 689, 'n_words': 177}