<jupyter_start><jupyter_text>lists are quite slow for this purpose because inserting or deleting an element at the beginning requires shifting all of the other elements by one, requiring O(n) time.“First in First Out” Sell = AsksBuy = BidsPrior Values: initial_asks: %s 22670final_asks: %s 12677initial_bids: %s 40436final_bids: %s 28413done_msgs: %s 84084open_msgs: %s 83274 MATCHTaker Order is removed off the Order BookMaker Order is kept with a down or an upA trade occurred between two orders.The aggressor or taker order is the one executing immediately after being received and the maker order is a resting order on the book.The side field indicates the maker order side.If the side is sell this indicates the maker was a sell order and the match is considered an up-tick.A buy side match is a down-tick.<jupyter_code>import logging
import queue

logging.basicConfig(level=20)
#logging.basicConfig(level=2)

logger = logging.getLogger()

class OrderDictTable:
    def __init__(self, order_items):
        self.order_dict = {} #
        self.assign_orders_dict(order_items)
        
    def assign_orders_dict(self,order_items):
        keys = [subli[2] for subli in order_items]#map(lambda price,size,order_id: order_id,order_items)
        self.order_dict = dict([(order_id,order_item) for (order_id,order_item) in zip(keys,order_items) ])
    
    def get_values(self):
        return self.order_dict.values()

    def get_obj(self):
        return self.order_dict

class CLOB:
    def __init__(self,clob):
        self.bids_obj = OrderDictTable(clob['bids']).get_obj()
        self.asks_obj = OrderDictTable(clob['asks']).get_obj()
        
    def order_remove(self,clob_bids_or_ask_dict,order_id):
        #logger.info('order_id: %s', str(order_id))
        removed_order = clob_bids_or_ask_dict.pop(order_id,None)
        #logger.info('clob_bids_or_ask_dict: %s', str(removed_order))
        
    def order_add(self,clob_bids_or_ask_dict,price,size,order_id):
        clob_bids_or_ask_dict[order_id] = [price,size,order_id]
        
    def update_from_message(self,message_obj):
        #logger.info('message_obj: %s', str(message_obj))
        message_type = message_obj['type']
        message_side = message_obj['side']
        clob_bids_or_ask_dict = self.asks_obj if message_side == 'sell' else self.bids_obj 
        #logger.info('message_side: %s', str(message_side))
        #logger.info('message_type: %s', str(message_type))

        if message_type == 'done':
            self.order_remove(clob_bids_or_ask_dict,message_obj['order_id'])
        elif message_type == 'open':
            self.order_add(clob_bids_or_ask_dict,message_obj['price'],message_obj['remaining_size'],message_obj['order_id'])
        
    def get_clob(self):
        return {
            'bids':self.bids_obj.values(),
            'asks':self.asks_obj.values()
        }
    
class CLOBValidate:
    def validate(self,initial_clob,final_clob,messages_queue_data):
        initial_bids = len(initial_clob['bids'])
        initial_asks = len(initial_clob['asks'])
        
        final_bids = len(final_clob['bids'])
        final_asks = len(final_clob['asks'])
        
        done_msgs = len([message_obj for message_obj in messages_queue_data if message_obj['type'] == 'done'])
        open_msgs = len([message_obj for message_obj in messages_queue_data if message_obj['type'] == 'open'])

        print ('initial_asks: %s', str(initial_asks))
        print ('final_asks: %s', str(final_asks))

        print ('initial_bids: %s', str(initial_bids))
        print ('final_bids: %s', str(final_bids))
        
        print ('done_msgs: %s', str(done_msgs))
        print ('open_msgs: %s', str(open_msgs))
        
        
clob = CLOB(initial_clob)
q = queue.Queue()
messages_queue_data = sorted(messages_data_filtered,key=lambda i: i['sequence'])

list(map(q.put,messages_queue_data ))
while not q.empty():
    message_obj = q.get()
    clob.update_from_message(message_obj)

final_clob_processed = clob.get_clob()
CLOBValidate().validate(final_clob_processed,final_clob,messages_queue_data)

#bids_dict_table = OrderDictTable(initial_clob['bids'])
#asks_dict_table = OrderDictTable(initial_clob['asks'])
messages_queue_data_sorted = sorted(messages_queue_data,key=lambda i: i['sequence'])
print (messages_queue_data_sorted[:2])
messages_queue_data_sorted_changed = [i for i in messages_queue_data_sorted if i['type'] == 'match']
print (len(messages_queue_data_sorted_changed))


print (messages_queue_data_sorted_changed[0])
len(list(set([i['order_id'] for i in messages_queue_data_sorted_changed])))
def list_difference(l1,l2):
    l1= list(l1)
    l2=list(l2)
    l1 = [i[2] for i in l1]
    l2 = [i[2] for i in l2]

    return list(set(l1) - set(l2))

#l1 = ['a','b']
#l2 = ['b','c']
#list_difference(l1,l2)
#list(final_clob_processed['asks'])
#list(final_clob['asks'])

asks_difference = list_difference(final_clob_processed['asks'],final_clob['asks'])
print (asks_difference[0])
#final_clob_processed['asks']
#final_clob['asks']

filter_order_id_message = lambda messages_data,order_id: [D for D in messages_data if 'order_id' in D.keys() and D['order_id'] ==order_id]
filter_order_id_clob = lambda clob,asks_or_bids,order_id: [subli for subli in clob[asks_or_bids] if subli[2] == order_id]



filter_order_id_message(messages_data,'1f080093-08f6-44df-86c8-bddf077673e6')
#maker_order_id

#filter_order_id_message(messages_data,'4fae0472-05b0-4c72-a99e-0158185683e4')
list(clob.get_clob()['bids'])[0]
print (len(bids_dict_table.get_values()))
print (len(initial_clob['bids']))
class Dicttable:
    def __init__(self, bids,asks):
        dict([(order_id,order_data) for (order_id,order_data) in bids ])
        self.bucket_size = len(elements)
        self.buckets = [[] for i in range(self.bucket_size)]
        self._assign_buckets(elements)


# Python program to
# demonstrate queue implementation
# using list
 
# Initializing a queue
queue = []
 
# Adding elements to the queue
queue.append('a')
queue.append('b')
queue.append('c')
 
print("Initial queue")
print(queue)
 
# Removing elements from the queue
print("\nElements dequeued from queue")
print(queue.pop(0))
print(queue.pop(0))
print(queue.pop(0))
 
print("\nQueue after removing elements")
print(queue)
 
# Uncommenting print(queue.pop(0))
# will raise and IndexError
# as the queue is now empty
import queue
q = queue.Queue()
list(map(q.put, messages_data_filtered))

q.get()
with q.mutex:
    q.queue.clear()
q.get()
messages_data_filtered[0]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12146676852559206, 'fraction_numerical': 0.01650114591291062, 'mean_word_length': 4.580562659846548, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '161314', 'license': 'MIT', 'n_tokens_mistral': 2383, 'n_tokens_neox': 2275, 'n_words': 469, 'path': 'jupyter/clob_dict_table.ipynb', 'repo_name': 'ChrisCruze/coinbase_clob_simulation', 'size': '6545'}