<commit_before>import sys
import re
import cPickle as pickle
from django.utils import simplejson as json
from test_utils.testmaker import Testmaker
from test_utils.testmaker.processors.django_processor import Processor

if len(sys.argv) == 2:
    in_file = sys.argv[1]
else:
    raise Exception('Need file name')

f = open(in_file)

buffer = []

req_re = re.compile('---REQUEST_BREAK---')
res_re = re.compile('---RESPONSE_BREAK---')

processor = Processor('replay_processor')
tm = Testmaker()

tm.setup_logging('replay_file', '/dev/null')

serial_obj = pickle

class Request(dict):
    'Mocking a dict to allow attribute access'
    def __getattr__(self, name):
        return self[name]

for line in f.readlines():
    if req_re.search(line):
        #process request
        to_pickle = ''.join(buffer)
        request = Request(serial_obj.loads(to_pickle))
        processor.save_request(request)
        print request['path'], request['time']
        buffer = []
    elif res_re.search(line):
        #process response
        to_pickle = ''.join(buffer)
        response = Request(serial_obj.loads(to_pickle))
        processer.save_response(request, response)
        print response['status_code'], response['time']
        buffer = []
    else:
        buffer.append(line)<commit_msg>Classify the replayer, hopefully making this easier to upgrade in the future.<commit_after>import sys
import re
import cPickle as pickle
from test_utils.testmaker import Testmaker
from test_utils.testmaker.processors.django_processor import Processor

class MockRequest(dict):
    'Mocking a dict to allow attribute access'
    def __getattr__(self, name):
        return self[name]

class Replay(object):

    def __init__(self, file_name, replay_file='replay_file'):
        self.file_name = file_name
        self.stream = open(self.file_name).readlines()
        self.tm = Testmaker()
        self.tm.setup_logging(replay_file, '/dev/null')
        self.processor = Processor('replay_processor')
        self.serial_obj = pickle

    def process(self):
        self.log = []

        buffer = []
        req_re = re.compile('---REQUEST_BREAK---')
        res_re = re.compile('---RESPONSE_BREAK---')

        for line in self.stream:
            if req_re.search(line):
                #process request
                to_pickle = ''.join(buffer)
                request = MockRequest(self.serial_obj.loads(to_pickle))
                self.processor.save_request(request)
                print request['path'], request['time']
                buffer = []
            elif res_re.search(line):
                #process response
                to_pickle = ''.join(buffer)
                response = MockRequest(self.serial_obj.loads(to_pickle))
                self.log.append(request, response)
                self.processer.save_response(request, response)
                print response['status_code'], response['time']
                buffer = []
            else:
                buffer.append(line)

if __name__ == '__main__':
    if len(sys.argv) == 2:
        in_file = sys.argv[1]
    else:
        raise Exception('Need file name')

    replay = Replay(in_file)
    replay.process()
{'fraction_non_alphanumeric': 0.11236660389202763, 'fraction_numerical': 0.0012554927809165098, 'mean_word_length': 2.552954292084727, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6694980', 'n_tokens_mistral': 956, 'n_tokens_neox': 930, 'n_words': 211}