<commit_before>import sys, os, traceback
import flow, filt, utils

class DumpError(Exception): pass


class Options(object):
    __slots__ = [
        "anticache",
        "client_replay",
        "keepserving",
        "kill",
        "no_server",
        "refresh_server_playback",
        "request_script",
        "response_script",
        "rfile",
        "rheaders",
        "server_replay",
        "stickycookie",
        "stickyauth",
        "verbosity",
        "wfile",
    ]
    def __init__(self, **kwargs):
        for k, v in kwargs.items():
            setattr(self, k, v)
        for i in self.__slots__:
            if not hasattr(self, i):
                setattr(self, i, None)


def str_response(resp):
    r = "%s %s"%(resp.code, resp.msg)
    if resp.is_replay():
        r = "[replay] " + r
    return r


def str_request(req):
    if req.client_conn:
        c = req.client_conn.address[0]
    else:
        c = "[replay]"
    r = "%s %s %s"%(c, req.method, req.url())
    if req.stickycookie:
        r = "[stickycookie] " + r
    return r


class DumpMaster(flow.FlowMaster):
    def __init__(self, server, options, filtstr, outfile=sys.stdout):
        flow.FlowMaster.__init__(self, server, flow.State())
        self.outfile = outfile
        self.o = options

        if filtstr:
            self.filt = filt.parse(filtstr)
        else:
            self.filt = None

        if self.o.response_script:
            self.set_response_script(self.o.response_script)  
        if self.o.request_script:
            self.set_request_script(self.o.request_script)  

        if options.stickycookie:
            self.set_stickycookie(options.stickycookie)

        if options.stickyauth:
            self.set_stickyauth(options.stickyauth)

        if options.wfile:
            path = os.path.expanduser(options.wfile)
            try:
                f = file(path, "wb")
                self.fwriter = flow.FlowWriter(f)
            except IOError, v:
                raise DumpError(v.strerror)

        if options.server_replay:
            self.start_server_playback(
                self._readflow(options.server_replay),
                options.kill, options.rheaders,
                not options.keepserving
            )

        if options.rfile:
            path = os.path.expanduser(options.rfile)
            try:
                f = file(path, "r")
                freader = flow.FlowReader(f)
            except IOError, v:
                raise DumpError(v.strerror)
            self.load_flows(freader)

        if options.client_replay:
            self.start_client_playback(
                self._readflow(options.client_replay),
                not options.keepserving
            )

        self.anticache = options.anticache
        self.refresh_server_playback = options.refresh_server_playback

    def _readflow(self, path):
        path = os.path.expanduser(path)
        try:
            f = file(path, "r")
            flows = list(flow.FlowReader(f).stream())
        except (IOError, flow.FlowReadError), v:<commit_msg>Anticache and refresh_server_playback options are applied before flows are loaded.<commit_after>import sys, os, traceback
import flow, filt, utils

class DumpError(Exception): pass


class Options(object):
    __slots__ = [
        "anticache",
        "client_replay",
        "keepserving",
        "kill",
        "no_server",
        "refresh_server_playback",
        "request_script",
        "response_script",
        "rfile",
        "rheaders",
        "server_replay",
        "stickycookie",
        "stickyauth",
        "verbosity",
        "wfile",
    ]
    def __init__(self, **kwargs):
        for k, v in kwargs.items():
            setattr(self, k, v)
        for i in self.__slots__:
            if not hasattr(self, i):
                setattr(self, i, None)


def str_response(resp):
    r = "%s %s"%(resp.code, resp.msg)
    if resp.is_replay():
        r = "[replay] " + r
    return r


def str_request(req):
    if req.client_conn:
        c = req.client_conn.address[0]
    else:
        c = "[replay]"
    r = "%s %s %s"%(c, req.method, req.url())
    if req.stickycookie:
        r = "[stickycookie] " + r
    return r


class DumpMaster(flow.FlowMaster):
    def __init__(self, server, options, filtstr, outfile=sys.stdout):
        flow.FlowMaster.__init__(self, server, flow.State())
        self.outfile = outfile
        self.o = options
        self.anticache = options.anticache
        self.refresh_server_playback = options.refresh_server_playback

        if filtstr:
            self.filt = filt.parse(filtstr)
        else:
            self.filt = None

        if self.o.response_script:
            self.set_response_script(self.o.response_script)  
        if self.o.request_script:
            self.set_request_script(self.o.request_script)  

        if options.stickycookie:
            self.set_stickycookie(options.stickycookie)

        if options.stickyauth:
            self.set_stickyauth(options.stickyauth)

        if options.wfile:
            path = os.path.expanduser(options.wfile)
            try:
                f = file(path, "wb")
                self.fwriter = flow.FlowWriter(f)
            except IOError, v:
                raise DumpError(v.strerror)

        if options.server_replay:
            self.start_server_playback(
                self._readflow(options.server_replay),
                options.kill, options.rheaders,
                not options.keepserving
            )

        if options.rfile:
            path = os.path.expanduser(options.rfile)
            try:
                f = file(path, "r")
                freader = flow.FlowReader(f)
            except IOError, v:
                raise DumpError(v.strerror)
            self.load_flows(freader)

        if options.client_replay:
            self.start_client_playback(
                self._readflow(options.client_replay),
                not options.keepserving
            )


    def _readflow(self, path):
        path = os.path.expanduser(path)
        try:
            f = file(path, "r")
            flows = list(flow.FlowReader(f).stream())
        except (IOError, flow.FlowReadError), v:
{'fraction_non_alphanumeric': 0.1051870884856271, 'fraction_numerical': 0.0003211819495744339, 'mean_word_length': 1.7766384306732055, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1256493', 'n_tokens_mistral': 1939, 'n_tokens_neox': 1884, 'n_words': 376}