<commit_before>import unittest
import os
import sys
sys.path.append('../lib/')
sys.path.append(os.path.dirname(os.path.realpath(__file__)))
import alchemy
from alchemy.schema import *


class TestAlchemy(unittest.TestCase):

    def setUp(self):
        # this basically resets our testing database
        os.system("cp {0}/alchemy.raw {0}/test.db".format(config.get("sqlite").get("path")))
        session = alchemy.fetch_session()
        pass

    def tearDown(self):
        # we keep this to tidy up our database if it fails
        session.close()
        pass

    def test_raw_clean(self):
        # add a Clean record to mark something against
        asg0 = session.query(RawAssignee).limit(10).all()
        asg1 = session.query(RawAssignee).limit(10).offset(10).all()

        alchemy.match(asg0)
        alchemy.match(asg1)
        alchemy.match([asg0[0], asg1[0].assignee])

    def test_match_all(self):
        alchemy.match(session.query(RawAssignee))

    def test_set_default(self):
        # create two items
        loc = session.query(RawLocation).all()
        alchemy.match(loc)

        alchemy.match(loc[0], {"city": u"Frisco", "state": u"Cali", "country": u"US", "longitude": 10.0, "latitude": 10.0})
        self.assertEqual("Frisco, Cali, US", loc[0].location.address)

        alchemy.match(loc[0], keepdefault=True)
        self.assertEqual("Frisco, Cali, US", loc[0].location.address)
        self.assertEqual(10.0, loc[0].location.latitude)
        self.assertEqual(10.0, loc[0].location.longitude)

        alchemy.match(loc[0])
        self.assertEqual("Hong Kong, OH, US", loc[0].location.address)
        self.assertEqual(10.0, loc[0].location.latitude)
        self.assertEqual(10.0, loc[0].location.longitude)

        alchemy.match(loc[0], {"city": u"Frisco"}, keepdefault=True)
        self.assertEqual("Frisco, OH, US", loc[0].location.address)
        self.assertEqual(10.0, loc[0].location.latitude)
        self.assertEqual(10.0, loc[0].location.longitude)

    def test_unmatch_asgloc(self):
        loc = session.query(RawLocation).limit(20).all()
        asg = session.query(RawAssignee).limit(20).all()
        alchemy.match(asg)
        alchemy.match(loc[0:5])
        alchemy.match(loc[5:10])
        alchemy.match(loc[10:15])
        alchemy.match(loc[15:20])

        clean = asg[0].assignee
        alchemy.unmatch(asg[0])
        self.assertEqual(None, asg[0].assignee)
        self.assertEqual(19, len(clean.rawassignees))
        self.assertEqual(19, len(clean.patents))

        self.assertEqual(4, session.query(Location).count())
        self.assertEqual(4, session.query(locationassignee).count())

        clean = loc[0].location
        self.assertEqual(5, len(clean.rawlocations))
        alchemy.unmatch(loc[0])
        self.assertEqual(4, len(clean.rawlocations))
        alchemy.unmatch(loc[1])
        self.assertEqual(3, len(clean.rawlocations))
        alchemy.unmatch(loc[2:5])
        self.assertEqual(None, loc[0].location)
        self.assertEqual(3, session.query(Location).count())
        self.assertEqual(3, session.query(locationassignee).count())

        alchemy.unmatch(loc[5].location)
        self.assertEqual(2, session.query(Location).count())
        self.assertEqual(2, session.query(locationassignee).count())
        alchemy.unmatch(loc[10].location)
        self.assertEqual(1, session.query(Location).count())
        self.assertEqual(1, session.query(locationassignee).count())

    def test_unmatch_invloc(self):
        loc = session.query(RawLocation).limit(20).all()
        inv = session.query(RawInventor).limit(20).all()
        alchemy.match(inv)
        alchemy.match(loc[0:5])
        alchemy.match(loc[5:10])
        alchemy.match(loc[10:15])
        alchemy.match(loc[15:20])

        clean = inv[0].inventor
        alchemy.unmatch(inv[0])
        self.assertEqual(None, inv[0].inventor)
        self.assertEqual(19, len(clean.rawinventors))
        self.assertEqual(10, len(clean.patents))

        self.assertEqual(4, session.query(Location).count())
        self.assertEqual(4, session.query(locationinventor).count())

        clean = loc[0].location
        self.assertEqual(5, len(clean.rawlocations))
        alchemy.unmatch(loc[0])
        self.assertEqual(4, len(clean.rawlocations))
        alchemy.unmatch(loc[1])
        self.assertEqual(3, len(clean.rawlocations))
        alchemy.unmatch(loc[2:5])
        self.assertEqual(None, loc[0].location)
        self.assertEqual(3, session.query(Location).count())
        self.assertEqual(3, session.query(locationinventor).count())

        alchemy.unmatch(inv[1])
        self.assertEqual(None, inv[1].inventor)
        self.assertEqual(18, len(clean.rawinventors))
        # this patent is repeated
        self.assertEqual(10, len(clean.patents))

        alchemy.unmatch(inv[2])
        self.assertEqual(None, inv[2].inventor)
        self.assertEqual(17, len(clean.rawinventors))
        self.assertEqual(9, len(clean.patents))

        self.assertEqual(2, session.query(Location).count())
        self.assertEqual(2, session.query(locationinventor).count())
        alchemy.unmatch(loc[0].location)
        self.assertEqual(1, session.query(Location).count())

        self.assertEqual(1, session.query(locationinventor).count())
        alchemy.unmatch(loc[10].location)
        self.assertEqual(0, session.query(Location).count())
        self.assertEqual(0, session.query(locationinventor).count())

    def test_unmatch_lawyer(self):
        law = session.query(RawLawyer).limit(20).all()
        alchemy.match(law)

        alchemy.unmatch(law[0])
        self.assertEqual(None, law[0].lawyer)
        self.assertEqual(19, len(law[1].lawyer.rawlawyers))
        self.assertEqual(14, len(law[1].lawyer.patents))
<commit_msg>added ability to remove multiple objects<commit_after>import unittest
import os
import sys
sys.path.append('../lib/')
sys.path.append(os.path.dirname(os.path.realpath(__file__)))
import alchemy
from alchemy.schema import *


class TestAlchemy(unittest.TestCase):

    def setUp(self):
        # this basically resets our testing database
        os.system("cp {0}/alchemy.raw {0}/test.db".format(config.get("sqlite").get("path")))
        session = alchemy.fetch_session()
        pass

    def tearDown(self):
        # we keep this to tidy up our database if it fails
        session.close()
        pass

    def test_raw_clean(self):
        # add a Clean record to mark something against
        asg0 = session.query(RawAssignee).limit(10).all()
        asg1 = session.query(RawAssignee).limit(10).offset(10).all()

        alchemy.match(asg0)
        alchemy.match(asg1)
        alchemy.match([asg0[0], asg1[0].assignee])

    def test_match_all(self):
        alchemy.match(session.query(RawAssignee))

    def test_set_default(self):
        # create two items
        loc = session.query(RawLocation).all()
        alchemy.match(loc)

        alchemy.match(loc[0], {"city": u"Frisco", "state": u"Cali", "country": u"US", "longitude": 10.0, "latitude": 10.0})
        self.assertEqual("Frisco, Cali, US", loc[0].location.address)

        alchemy.match(loc[0], keepdefault=True)
        self.assertEqual("Frisco, Cali, US", loc[0].location.address)
        self.assertEqual(10.0, loc[0].location.latitude)
        self.assertEqual(10.0, loc[0].location.longitude)

        alchemy.match(loc[0])
        self.assertEqual("Hong Kong, OH, US", loc[0].location.address)
        self.assertEqual(10.0, loc[0].location.latitude)
        self.assertEqual(10.0, loc[0].location.longitude)

        alchemy.match(loc[0], {"city": u"Frisco"}, keepdefault=True)
        self.assertEqual("Frisco, OH, US", loc[0].location.address)
        self.assertEqual(10.0, loc[0].location.latitude)
        self.assertEqual(10.0, loc[0].location.longitude)

    def test_unmatch_asgloc(self):
        loc = session.query(RawLocation).limit(20).all()
        asg = session.query(RawAssignee).limit(20).all()
        alchemy.match(asg)
        alchemy.match(loc[0:5])
        alchemy.match(loc[5:10])
        alchemy.match(loc[10:15])
        alchemy.match(loc[15:20])

        clean = asg[0].assignee
        alchemy.unmatch(asg[0])
        self.assertEqual(None, asg[0].assignee)
        self.assertEqual(19, len(clean.rawassignees))
        self.assertEqual(19, len(clean.patents))

        self.assertEqual(4, session.query(Location).count())
        self.assertEqual(4, session.query(locationassignee).count())

        clean = loc[0].location
        self.assertEqual(5, len(clean.rawlocations))
        alchemy.unmatch(loc[0])
        self.assertEqual(4, len(clean.rawlocations))
        alchemy.unmatch(loc[1])
        self.assertEqual(3, len(clean.rawlocations))
        alchemy.unmatch(loc[2:5])
        self.assertEqual(None, loc[0].location)
        self.assertEqual(3, session.query(Location).count())
        self.assertEqual(3, session.query(locationassignee).count())

        alchemy.unmatch(loc[5].location)
        self.assertEqual(2, session.query(Location).count())
        self.assertEqual(2, session.query(locationassignee).count())
        alchemy.unmatch(loc[10].location)
        self.assertEqual(1, session.query(Location).count())
        self.assertEqual(1, session.query(locationassignee).count())

    def test_unmatch_invloc(self):
        loc = session.query(RawLocation).limit(20).all()
        inv = session.query(RawInventor).limit(20).all()
        alchemy.match(inv)
        alchemy.match(loc[0:5])
        alchemy.match(loc[5:10])
        alchemy.match(loc[10:15])
        alchemy.match(loc[15:20])

        clean = inv[0].inventor
        alchemy.unmatch(inv[0])
        self.assertEqual(None, inv[0].inventor)
        self.assertEqual(19, len(clean.rawinventors))
        self.assertEqual(10, len(clean.patents))

        self.assertEqual(4, session.query(Location).count())
        self.assertEqual(4, session.query(locationinventor).count())

        clean = loc[0].location
        self.assertEqual(5, len(clean.rawlocations))
        alchemy.unmatch(loc[0])
        self.assertEqual(4, len(clean.rawlocations))
        alchemy.unmatch(loc[1])
        self.assertEqual(3, len(clean.rawlocations))
        alchemy.unmatch(loc[2:5])
        self.assertEqual(None, loc[0].location)
        self.assertEqual(3, session.query(Location).count())
        self.assertEqual(3, session.query(locationinventor).count())

        clean = inv[5].inventor
        alchemy.unmatch(inv[1])
        self.assertEqual(None, inv[1].inventor)
        self.assertEqual(18, len(clean.rawinventors))
        # this patent is repeated
        self.assertEqual(10, len(clean.patents))

        alchemy.unmatch(inv[2])
        self.assertEqual(None, inv[2].inventor)
        self.assertEqual(17, len(clean.rawinventors))
        self.assertEqual(9, len(clean.patents))

        alchemy.unmatch(loc[5].location)
        self.assertEqual(2, session.query(Location).count())
        self.assertEqual(2, session.query(locationinventor).count())

        alchemy.unmatch(loc[10].location)
        self.assertEqual(1, session.query(Location).count())
        self.assertEqual(1, session.query(locationinventor).count())

    def test_unmatch_lawyer(self):
        law = session.query(RawLawyer).limit(20).all()
        alchemy.match(law)

        alchemy.unmatch(law[0])
        self.assertEqual(None, law[0].lawyer)
        self.assertEqual(19, len(law[1].lawyer.rawlawyers))
        self.assertEqual(14, len(law[1].lawyer.patents))

{'fraction_non_alphanumeric': 0.1485208640582979, 'fraction_numerical': 0.028541684740175242, 'mean_word_length': 3.0392431674842326, 'pattern_counts': {'":': 12, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3584677', 'n_tokens_mistral': 3894, 'n_tokens_neox': 3649, 'n_words': 515}