<commit_before>

class Site(object):

    def __init__(self):
        self.op_events = []
        self.chats = {}
        self.emails = {}
        self.operators = set()
        self.visitors = set()

    def add_operator_event(self, ts, op, state):
        self.transitions = \
            sorted(set(self.op_events + [(ts, op, state)]))
        self.operators.add(op)
    
    def get_state(self, time_stamp):
        op_count = 0
        for ts, op, state in self.transitions:
            if time_stamp >= ts:
                break
            change = 1 if state == 'online' else -1
            op_count += change
        return 'online' if op_count else 'offline'

    def add_chat(self, time_stamp, visitor):
        if time_stamp in self.chats or time_stamp in self.emails:
            return
        state = self.get_state(time_stamp)
        if state == 'online':
            self.chats[time_stamp] = visitor
        else:
            self.emails[time_stamp] = visitor
        self.visitors.add(visitor)

    def report(self, site_id):
        print "{site_id},messages={messages},emails={emails},operators={operators},visitors={visitors}".format(
            site_id=site_id, messages=len(self.chats), emails=len(self.emails), 
            operators=len(self.operators), visitors=len(self.visitors))

def main():
    import sys
    import json
    fname = sys.argv[1]

    sites = {}
    
    with open(fname) as f:
       for line in f.readlines():
           data = json.loads(line)
           site_id = data['site_id']
           site = sites.setdefault(site_id, Site())
           if data['type'] == 'status':
               site.add_operator_event(data['timestamp'], data['from'], data['data']['status'])
          
    with open(fname) as f:
        for line in f.readlines():
            data = json.loads(line.strip())
            site_id = data['site_id']
            site = sites[site_id]
            if data['type'] == 'message':
                site.add_chat(data['timestamp'], data['from'])

    for site_id, site in sites.items():
       site.report(site_id)


if __name__ == '__main__':
    main()
<commit_msg>fix logic for site online / offline seems like I needed to track it per operator, took too long to figure out<commit_after>

class Site(object):

    def __init__(self):
        self.op_events = {}
        self.chats = set()
        self.emails = set()
        self.operators = set()
        self.visitors = set()

    def add_operator_event(self, ts, op, state):
        self.op_events[op] =  sorted(set(self.op_events.get(op, []) + [(ts, state)]))
        self.operators.add(op)
    
    def get_state(self, time_stamp):
#        import pdb; pdb.set_trace()
        states = []
        for op, events in self.op_events.items():
            prev_state = 'offline'# if events[0][1] == 'online' else 'online'
            for ts, state in events:
                if ts > time_stamp:
                    break
                prev_state = state
            states.append(prev_state)
        #print time_stamp, states
        return 'online' if 'online' in states  else 'offline'

    def add_chat(self, time_stamp, visitor):
        if time_stamp in self.chats or time_stamp in self.emails:
            return
        state = self.get_state(time_stamp)
        if state == 'online':
            self.chats.add(time_stamp)
        else:
            self.emails.add(time_stamp)
        self.visitors.add(visitor)

    def report(self, site_id):
        print "{site_id},messages={messages},emails={emails},operators={operators},visitors={visitors}".format(
            site_id=site_id, messages=len(self.chats), emails=len(self.emails), 
            operators=len(self.operators), visitors=len(self.visitors))

def main():
    import sys
    import json
    fname = sys.argv[1]

    sites = {}
    
    i = 0
    with open(fname) as f:
       for line in f.readlines():
#           if i % 10000 == 0: print 'first pass {}'.format(i)
           i+=1
           data = json.loads(line)
           site_id = data['site_id']
           site = sites.setdefault(site_id, Site())
           if data['type'] == 'status':
               site.add_operator_event(data['timestamp'], data['from'], data['data']['status'])
 
#    from pprint import pprint
#    pprint(sites.values()[0].op_events)
         
    i = 0
    with open(fname) as f:
        for line in f.readlines():
#            if i % 10000 == 0: print 'second pass {}'.format(i)
            i+=1
            data = json.loads(line.strip())
            site_id = data['site_id']
            site = sites[site_id]
            if data['type'] == 'message':
                #print data['timestamp']
                site.add_chat(data['timestamp'], data['from'])

    for site_id, site in sorted(sites.items(), key=lambda _e: _e[0]):
       site.report(site_id)


if __name__ == '__main__':
    main()

{'fraction_non_alphanumeric': 0.13079130791307914, 'fraction_numerical': 0.005125051250512505, 'mean_word_length': 2.0762925598991173, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6911120', 'n_tokens_mistral': 1556, 'n_tokens_neox': 1518, 'n_words': 355}