<commit_before>
from greenlet import greenlet
import ctypes
import random
import os
import json

# When runnin as a package
# _DIRNAME = os.path.dirname(__file__)
# go = ctypes.cdll.LoadLibrary(os.path.join(_DIRNAME, 'libriffmantle.so'))

# When running locally-- no gopy
# mantle = ctypes.cdll.LoadLibrary('./libriffmantle.so')

# When running with gopy
import riffle

class Deferred(object):

    def __init__(self):
        self._callback, self._errback = None, None
        self._callbackId, self._errbackId = -1, -1

def cbid():
    return random.getrandbits(53)
    
class App(object):

    def __init__(self):
        self._app = riffle.App()
        self._app.Init()

        self.registrations, self.subscriptions, self.meta = {}, {}, {}

    def recv(self):
        while True:
            i, args = json.loads(self._app.Receive())
            args = args if args is not None else []

            # Wrap it all in a try-catch, return publish and call errors
            # Don't return yield errors-- its not clear who should deal with those 

            if i in self.meta:
                # Remove the meta call after called? It should not be called more than once, no?
                self.meta[i](*args)

            elif i in self.subscriptions:
                self.subscriptions[i](*args)

            elif i in self.registrations:
                ret = self.registrations[i](*args)


            else: 
                print "No handler available for ", i


# Internalize this reference into the domain object. For now, its ok global
app = App()


class Domain(object):

    def __init__(self, name):
        self.mantleDomain = app._app.NewDomain(name)
        self.name = name

    def join(self):
        cb, eb = cbid(), cbid()<commit_msg>yield almost in place, bad list creation<commit_after>
from greenlet import greenlet
import ctypes
import random
import os
import json

# When runnin as a package
# _DIRNAME = os.path.dirname(__file__)
# go = ctypes.cdll.LoadLibrary(os.path.join(_DIRNAME, 'libriffmantle.so'))

# When running locally-- no gopy
# mantle = ctypes.cdll.LoadLibrary('./libriffmantle.so')

# When running with gopy
import riffle

class Deferred(object):

    def __init__(self):
        self._callback, self._errback = None, None
        self._callbackId, self._errbackId = -1, -1

def cbid():
    return random.getrandbits(53)

class App(object):

    def __init__(self):
        self._app = riffle.App()
        self._app.Init()

        self.registrations, self.subscriptions, self.meta = {}, {}, {}

    def recv(self):
        while True:
            i, args = json.loads(self._app.Receive())
            args = args if args is not None else []

            # Wrap it all in a try-catch, return publish and call errors
            # Don't return yield errors-- its not clear who should deal with those 

            if i in self.meta:
                # Remove the meta call after called? It should not be called more than once, no?
                self.meta[i](*args)

            elif i in self.subscriptions:
                self.subscriptions[i](*args)

            elif i in self.registrations:
                returnId = args.pop(0)
                ret = self.registrations[i](*args)


            else: 
                print "No handler available for ", i


# Internalize this reference into the domain object. For now, its ok global
app = App()


class Domain(object):

    def __init__(self, name):
        self.mantleDomain = app._app.NewDomain(name)
        self.name = name

    def join(self):
        cb, eb = cbid(), cbid()
{'fraction_non_alphanumeric': 0.10134003350083752, 'fraction_numerical': 0.002512562814070352, 'mean_word_length': 2.2484134179510424, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '604525', 'n_tokens_mistral': 1098, 'n_tokens_neox': 1039, 'n_words': 341}