<commit_before>#!/usr/bin/env pytho
# -*- coding: utf-8 -*-

from archery.bow import Daikyu as sdict
import sys
import os
import atexit



from logging.config import dictConfig
import logging

from itertools import islice, product
from dsat.state import construct_info, get_connection


#pyzmq is string agnostic, so we ensure we use bytes
import simplejson as json
from simplejson import loads, dumps


SENTINEL = object
HOUR = 3600
DAY = 24 * HOUR
#### let's 
#time_keeper = scheduler(time, sleep)
if not len(sys.argv) >= 1:
    raise( Exception("Arg"))

def warn(msg):
    if True:
        print msg
    else:
        raise(Exception(msg))

class ToxicSet(set):
    """a set for wich add is a shortcut for union

    Some people want to watch the world burn.
    """
    def copy(self):
        if hasattr(super(Copier, self),"copy"):
            return self.__class__(super(Copier, self).copy())
        else:
            return [ x for x in self]

    def __iadd__(x,y):
        x|=y
        return x

    def __add__(x,y):
        return x|y


    def to_json(self):
        return list(self)

    def __str__(self):
        return str(list(self))

    def __repr__(self):
        return repr(list(self))


#CNX = get_connection(CONFIG, LOCAL_INFO)


class Backend(object):
    """A class that does a garbage collector on a list of event per channel
    each channel must be in _db
    in _ref_counter => channel_name => lurker there is a a ref counters on the events
    that were consumed
    Since union/removal/intersections are used a lot, it is a set that is chosen
    for each channel there is a Cache like API to serialize/unserialize the events
    being stored. 

    Events are stored as channel => event_id => event

    the cache is invalidated if the ref count is 0 on a given event_id for all lurkers
    the cache is destroyed if nb lurkers = 0
    """
    # list of lurkers per region
    # The singleton approach I am trying to have is stupid.

    # On the other hand keeping the file for saving in O_EXCL could help 
    # Ensuring one access possible at a time ... as long as we are living
    # in a non distributed env
    _db = None
    # list of reference counts per channel per lurkers
    ## requires that keys are unique for EVERY region (hence the use of stamper)
    _ref_counter = sdict()
    atexit = None

    def _dump(self):<commit_msg>adding serialized queues preaparing v 0.6.2<commit_after>#!/usr/bin/env pytho
# -*- coding: utf-8 -*-

from archery.bow import Daikyu as sdict
import sys
import os
import atexit



from logging.config import dictConfig
import logging

from itertools import islice, product
from dsat.state import construct_info, get_connection


#pyzmq is string agnostic, so we ensure we use bytes
import simplejson as json
from simplejson import loads, dumps


SENTINEL = object
HOUR = 3600
DAY = 24 * HOUR
#### let's 
#time_keeper = scheduler(time, sleep)
if not len(sys.argv) >= 1:
    raise( Exception("Arg"))

def warn(msg):
    if True:
        print msg
    else:
        raise(Exception(msg))

class ToxicSet(set):
    """a set for wich add is a shortcut for union

    Some people want to watch the world burn.
    """
    def copy(self):
        if hasattr(super(Copier, self),"copy"):
            return self.__class__(super(Copier, self).copy())
        else:
            return [ x for x in self]

    def __iadd__(x,y):
        x|=y
        return x

    def __add__(x,y):
        return x|y


    def to_json(self):
        return list(self)

    def __str__(self):
        return str(list(self))

    def __repr__(self):
        return repr(list(self))


#CNX = get_connection(CONFIG, LOCAL_INFO)
import os
import atexit
from dogpile.cache.region import make_region
from simplejson import load, dump
class SerializedQueue(object):
    def __init__(self, **option):
        self.name = option["name"]
        self.load()
        self.cache = make_region(
            ).configure( 'dogpile.cache.dbm',
                ### 2 days
                expiration_time = 3600 * 7 * 24,
                arguments = {
                    "filename":"./%s.dbm" % option["name"]
                })
        atexit.register(lambda: self.save())
    def pop(self):
        task_id = self._queue.pop()
        vector = self.cache.get(str(task_id))
        return task_id, vector

    def append(self, task_id, vector):
        self._queue.append(str(task_id))
        self.cache.set(str(task_id), vector)

    def insert(self, task_id, vector):
        self._queue.appendleft(str(task_id))
        self.cache.set(str(task_id), vector)

    def load(self):
        if not os.path.exists(self.name):
            self._queue = deque()
            self.save()
        else:
            try:
                self._queue = deque(map(str,load(open("%(name)s" % self.__dict__ ))))
            except Exception as e:
                self._queue = deque()

    def save(self):
        dump(list(self._queue), open("%(name)s" % self.__dict__, "w"))



class Backend(object):
    """A class that does a garbage collector on a list of event per channel
    each channel must be in _db
    in _ref_counter => channel_name => lurker there is a a ref counters on the events
    that were consumed
    Since union/removal/intersections are used a lot, it is a set that is chosen
    for each channel there is a Cache like API to serialize/unserialize the events
    being stored. 

    Events are stored as channel => event_id => event

    the cache is invalidated if the ref count is 0 on a given event_id for all lurkers
    the cache is destroyed if nb lurkers = 0
    """
    # list of lurkers per region
    # The singleton approach I am trying to have is stupid.

    # On the other hand keeping the file for saving in O_EXCL could help 
    # Ensuring one access possible at a time ... as long as we are living
    # in a non distributed env
    _db = None
    # list of reference counts per channel per lurkers
    ## requires that keys are unique for EVERY region (hence the use of stamper)
    _ref_counter = sdict()
    atexit = None

    def _dump(self):
{'fraction_non_alphanumeric': 0.08641571194762684, 'fraction_numerical': 0.005073649754500818, 'mean_word_length': 2.552906976744186, 'pattern_counts': {'":': 1, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4006320', 'n_tokens_mistral': 1897, 'n_tokens_neox': 1794, 'n_words': 704}