<commit_before>#!/usr/bin/env python2
from common import *
from testdc import *

CREATE_DOID_OFFSET = 1 + 8 + 8 + 2 + 4
VERIFY_DELETE_OBJECT = 0x21656944
VERIFY_DELETE_QUERY = 0x6c6c694b

class DatabaseBaseTests(object):
    def createTypeGetId(self, sender, context, type):
        # Create object of type
        dg = Datagram.create([777], sender, DBSERVER_CREATE_OBJECT)
        dg.add_uint32(context)
        dg.add_uint16(type)
        dg.add_uint16(0) # Field count
        self.conn.send(dg)

        dg = self.conn.recv()
        dgi = DatagramIterator(dg)
        dgi.seek(CREATE_DOID_OFFSET)
        return dgi.read_uint32()

    def createGenericGetId(self, sender, context):
        return self.createTypeGetId(sender, context, DistributedTestObject1)

    def deleteObject(self, sender, doid):
        dg = Datagram.create([777], sender, DBSERVER_OBJECT_DELETE)
        dg.add_uint32(doid)
        self.conn.send(dg)

    def test_create_getall(self):
        self.conn.flush()
        self.conn.send(Datagram.create_add_channel(20))

        doids = []

        # Create a stored DistributedTestObject1 with no initial values...
        dg = Datagram.create([777], 20, DBSERVER_CREATE_OBJECT)
        dg.add_uint32(1) # Context
        dg.add_uint16(DistributedTestObject1)
        dg.add_uint16(0) # Field count
        self.conn.send(dg)

        # The Database should return the context and do_id...
        dg = self.conn.recv()
        dgi = DatagramIterator(dg)
        self.assertTrue(dgi.matches_header([20], 777, DBSERVER_CREATE_OBJECT_RESP, remaining=4+4))
        self.assertEquals(dgi.read_uint32(), 1) # Check context
        doids.append(dgi.read_uint32())
        self.assertGreaterEqual(doids[0], 1000000) # do_id in valid range
        self.assertLessEqual(doids[0], 1000010) # do_id in valid range

        # Select all fields from the stored object
        dg = Datagram.create([777], 20, DBSERVER_OBJECT_GET_ALL)
        dg.add_uint32(2) # Context
        dg.add_uint32(doids[0])
        self.conn.send(dg)

        # Retrieve object from the database, we stored no DB values, so get none back
        dg = Datagram.create([20], 777, DBSERVER_OBJECT_GET_ALL_RESP)
        dg.add_uint32(2) # Context
        dg.add_uint8(SUCCESS)
        dg.add_uint16(DistributedTestObject1)
        dg.add_uint16(0) # Field count
        self.assertTrue(*self.conn.expect(dg)) # Expecting SELECT_RESP with no values

        # Create a stored DistributedTestObject3 missing a required value...
        dg = Datagram.create([777], 20, DBSERVER_CREATE_OBJECT)
        dg.add_uint32(3) # Context
        dg.add_uint16(DistributedTestObject3)
        dg.add_uint16(0) # Field count
        self.conn.send(dg)

        # Return should be invalid because it is missing a defaultless required field
        dg = Datagram.create([20], 777, DBSERVER_CREATE_OBJECT_RESP)
        dg.add_uint32(3) # Context
        dg.add_uint32(INVALID_DO_ID)
        self.assertTrue(*self.conn.expect(dg)) # Expecting CREATE_RESP with BAD_DO_ID

        # Create a stored DistributedTestObject3 with an actual values...
        dg = Datagram.create([777], 20, DBSERVER_CREATE_OBJECT)
        dg.add_uint32(4) # Context
        dg.add_uint16(DistributedTestObject3)
        dg.add_uint16(2) # Field count
        dg.add_uint16(setRDB3)
        dg.add_uint32(91849)
        dg.add_uint16(setDb3)
        dg.add_string("You monster...")
        self.conn.send(dg)

        # The Database should return a new do_id...
        dg = self.conn.recv()
        dgi = DatagramIterator(dg)
        self.assertTrue(dgi.matches_header([20], 777, DBSERVER_CREATE_OBJECT_RESP, remaining=4+4))
        self.assertEquals(dgi.read_uint32(), 4) # Check context
        doids.append(dgi.read_uint32())
        self.assertGreaterEqual(doids[1], 1000000) # do_id in valid range
        self.assertLessEqual(doids[1], 1000010) # do_id in valid range
        self.assertTrue(doids[0] != doids[1]) # do_ids should be different

        # Retrieve object from the database...
        dg = Datagram.create([777], 20, DBSERVER_OBJECT_GET_ALL)
        dg.add_uint32(5) # Context
        dg.add_uint32(doids[1])
        self.conn.send(dg)

        # Get values back from server
        dg = self.conn.recv()<commit_msg>Tests: Update test_database to update for new behavior.<commit_after>#!/usr/bin/env python2
from common import *
from testdc import *

CREATE_DOID_OFFSET = 1 + 8 + 8 + 2 + 4
VERIFY_DELETE_OBJECT = 0x21656944
VERIFY_DELETE_QUERY = 0x6c6c694b

class DatabaseBaseTests(object):
    def createTypeGetId(self, sender, context, type):
        # Create object of type
        dg = Datagram.create([777], sender, DBSERVER_CREATE_OBJECT)
        dg.add_uint32(context)
        dg.add_uint16(type)
        dg.add_uint16(0) # Field count
        self.conn.send(dg)

        dg = self.conn.recv()
        dgi = DatagramIterator(dg)
        dgi.seek(CREATE_DOID_OFFSET)
        return dgi.read_uint32()

    def createGenericGetId(self, sender, context):
        return self.createTypeGetId(sender, context, DistributedTestObject1)

    def deleteObject(self, sender, doid):
        dg = Datagram.create([777], sender, DBSERVER_OBJECT_DELETE)
        dg.add_uint32(doid)
        self.conn.send(dg)

    def test_create_getall(self):
        self.conn.flush()
        self.conn.send(Datagram.create_add_channel(20))

        doids = []

        # Create a stored DistributedTestObject1 with no initial values...
        dg = Datagram.create([777], 20, DBSERVER_CREATE_OBJECT)
        dg.add_uint32(1) # Context
        dg.add_uint16(DistributedTestObject1)
        dg.add_uint16(0) # Field count
        self.conn.send(dg)

        # The Database should return the context and do_id...
        dg = self.conn.recv()
        dgi = DatagramIterator(dg)
        self.assertTrue(dgi.matches_header([20], 777, DBSERVER_CREATE_OBJECT_RESP, remaining=4+4))
        self.assertEquals(dgi.read_uint32(), 1) # Check context
        doids.append(dgi.read_uint32())
        self.assertGreaterEqual(doids[0], 1000000) # do_id in valid range
        self.assertLessEqual(doids[0], 1000010) # do_id in valid range

        # Select all fields from the stored object
        dg = Datagram.create([777], 20, DBSERVER_OBJECT_GET_ALL)
        dg.add_uint32(2) # Context
        dg.add_uint32(doids[0])
        self.conn.send(dg)

        # Retrieve object from the database, we stored no DB values, so get none back
        dg = Datagram.create([20], 777, DBSERVER_OBJECT_GET_ALL_RESP)
        dg.add_uint32(2) # Context
        dg.add_uint8(SUCCESS)
        dg.add_uint16(DistributedTestObject1)
        dg.add_uint16(0) # Field count
        self.assertTrue(*self.conn.expect(dg)) # Expecting SELECT_RESP with no values

        # Create a stored DistributedTestObject3 with an actual values...
        dg = Datagram.create([777], 20, DBSERVER_CREATE_OBJECT)
        dg.add_uint32(4) # Context
        dg.add_uint16(DistributedTestObject3)
        dg.add_uint16(2) # Field count
        dg.add_uint16(setRDB3)
        dg.add_uint32(91849)
        dg.add_uint16(setDb3)
        dg.add_string("You monster...")
        self.conn.send(dg)

        # The Database should return a new do_id...
        dg = self.conn.recv()
        dgi = DatagramIterator(dg)
        self.assertTrue(dgi.matches_header([20], 777, DBSERVER_CREATE_OBJECT_RESP, remaining=4+4))
        self.assertEquals(dgi.read_uint32(), 4) # Check context
        doids.append(dgi.read_uint32())
        self.assertGreaterEqual(doids[1], 1000000) # do_id in valid range
        self.assertLessEqual(doids[1], 1000010) # do_id in valid range
        self.assertTrue(doids[0] != doids[1]) # do_ids should be different

        # Retrieve object from the database...
        dg = Datagram.create([777], 20, DBSERVER_OBJECT_GET_ALL)
        dg.add_uint32(5) # Context
        dg.add_uint32(doids[1])
        self.conn.send(dg)

        # Get values back from server
        dg = self.conn.recv()
{'fraction_non_alphanumeric': 0.10271148319380233, 'fraction_numerical': 0.04785705360489816, 'mean_word_length': 2.763046544428773, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4768164', 'n_tokens_mistral': 2938, 'n_tokens_neox': 2596, 'n_words': 547}