# Path: wikidat/utils/comutils.py
# def send_ujson(socket, obj, flags=0):
#     """Serialize object using ultra-fast ujson"""
#     m = ujson.dumps(obj)
#     z = zlib.compress(m.encode())
#     return socket.send(z, flags=flags)
# 
# def recv_ujson(socket, flags=0):
#     """Load object from ujson serialization"""
#     z = socket.recv(flags)
#     m = zlib.decompress(z).decode()
#     return ujson.loads(m)
# 
# Path: wikidat/retrieval/page.py
# class Page(DataItem):
#     """
#     Models Page elements in Wikipedia database dumps
#     """
# 
#     def __init__(self, *args, **kwargs):
#         """
#         Constructor method for Page objects. Must forward params to
#         parent class DataItem (mandatory inheritance)
# 
#         The following keys must be populated:
#         ---------
#         * id: Unique numeric identifier for this page
#         * ns: Namespace code for this page (0 is main)
#         * title: Page title (unicode string)
#         * restrictions: List of applicable restrictions to this page
#         """
#         super(Page, self).__init__(*args, **kwargs)
# 
# Path: wikidat/retrieval/revision.py
# class Revision(DataItem):
#     """
#     Models Revision elements in Wikipedia dump files
#     """
# 
#     def __init__(self, *args, **kwargs):
#         """
#         Constructor method for Revision objects. Must forward params to
#         parent class DataItem (mandatory inheritance)
# 
#         The following keys must be populated:
#         ---------
#         * id: Unique numeric identifier for this revision
#         * rev_user: Numeric identifier of author of this revision (0 is anon)
#         * rev_timestamp: Timestamp when this rev was saved in database
#         * rev_len: Length in bytes of this revision
#         * ... (other params)
#         """
#         super(Revision, self).__init__(*args, **kwargs)
# 
# Path: wikidat/retrieval/logitem.py
# class LogItem(DataItem):
#     """
#     Models LogItem elements extracted from the 'logging' DB table in Wikipedia
#     """
# 
#     def __init__(self, *args, **kwargs):
#         """
#         Constructor method for Page objects. Must forward params to
#         parent class DataItem (mandatory inheritance)
# 
#         Parameters
#         ----------
#         data_dict: dictionary of raw data to be processed
#         lang: identifier of Wikipedia language edition from which this
#         element comes from (e.g. frwiki, eswiki, dewiki...)
#         """
#         super(LogItem, self).__init__(*args, **kwargs)


import time
import multiprocessing as mp
import zmq
from wikidat.utils.comutils import send_ujson, recv_ujson
from .page import Page
from .revision import Revision
from .logitem import LogItem
        target = self.target

        # Set up sending ZMQ data and control channels
        context = zmq.Context()

        if (self.push_pages_port):
            channel_pages_send = context.socket(zmq.PUSH)
            channel_pages_send.bind("tcp://127.0.0.1:%s" %
                                    self.push_pages_port)

        if (self.push_revs_port):
            channel_revs_send = context.socket(zmq.PUSH)
            channel_revs_send.bind("tcp://127.0.0.1:%s" %
                                   self.push_revs_port)

        if (self.push_logs_port):
            channel_logs_send = context.socket(zmq.PUSH)
            channel_logs_send.bind("tcp://127.0.0.1:%s" %
                                   self.push_logs_port)

        channel_control = context.socket(zmq.PUB)
        channel_control.bind("tcp://127.0.0.1:%s" %
                             self.control_port)

        # Wait a second to wake up and connect
        time.sleep(1)

        for item in target(*self.args, **self.kwargs):
            # Classify outcome elements in their corresponding queue
            # for later processing

# Path: wikidat/utils/comutils.py
# def send_ujson(socket, obj, flags=0):
#     """Serialize object using ultra-fast ujson"""
#     m = ujson.dumps(obj)
#     z = zlib.compress(m.encode())
#     return socket.send(z, flags=flags)
# 
# def recv_ujson(socket, flags=0):
#     """Load object from ujson serialization"""
#     z = socket.recv(flags)
#     m = zlib.decompress(z).decode()
#     return ujson.loads(m)
# 
# Path: wikidat/retrieval/page.py
# class Page(DataItem):
#     """
#     Models Page elements in Wikipedia database dumps
#     """
# 
#     def __init__(self, *args, **kwargs):
#         """
#         Constructor method for Page objects. Must forward params to
#         parent class DataItem (mandatory inheritance)
# 
#         The following keys must be populated:
#         ---------
#         * id: Unique numeric identifier for this page
#         * ns: Namespace code for this page (0 is main)
#         * title: Page title (unicode string)
#         * restrictions: List of applicable restrictions to this page
#         """
#         super(Page, self).__init__(*args, **kwargs)
# 
# Path: wikidat/retrieval/revision.py
# class Revision(DataItem):
#     """
#     Models Revision elements in Wikipedia dump files
#     """
# 
#     def __init__(self, *args, **kwargs):
#         """
#         Constructor method for Revision objects. Must forward params to
#         parent class DataItem (mandatory inheritance)
# 
#         The following keys must be populated:
#         ---------
#         * id: Unique numeric identifier for this revision
#         * rev_user: Numeric identifier of author of this revision (0 is anon)
#         * rev_timestamp: Timestamp when this rev was saved in database
#         * rev_len: Length in bytes of this revision
#         * ... (other params)
#         """
#         super(Revision, self).__init__(*args, **kwargs)
# 
# Path: wikidat/retrieval/logitem.py
# class LogItem(DataItem):
#     """
#     Models LogItem elements extracted from the 'logging' DB table in Wikipedia
#     """
# 
#     def __init__(self, *args, **kwargs):
#         """
#         Constructor method for Page objects. Must forward params to
#         parent class DataItem (mandatory inheritance)
# 
#         Parameters
#         ----------
#         data_dict: dictionary of raw data to be processed
#         lang: identifier of Wikipedia language edition from which this
#         element comes from (e.g. frwiki, eswiki, dewiki...)
#         """
#         super(LogItem, self).__init__(*args, **kwargs)


# Path: wikidat/retrieval/processors.py
import time
import multiprocessing as mp
import zmq
from wikidat.utils.comutils import send_ujson, recv_ujson
from .page import Page
from .revision import Revision
from .logitem import LogItem

        target = self.target

        # Set up sending ZMQ data and control channels
        context = zmq.Context()

        if (self.push_pages_port):
            channel_pages_send = context.socket(zmq.PUSH)
            channel_pages_send.bind("tcp://127.0.0.1:%s" %
                                    self.push_pages_port)

        if (self.push_revs_port):
            channel_revs_send = context.socket(zmq.PUSH)
            channel_revs_send.bind("tcp://127.0.0.1:%s" %
                                   self.push_revs_port)

        if (self.push_logs_port):
            channel_logs_send = context.socket(zmq.PUSH)
            channel_logs_send.bind("tcp://127.0.0.1:%s" %
                                   self.push_logs_port)

        channel_control = context.socket(zmq.PUB)
        channel_control.bind("tcp://127.0.0.1:%s" %
                             self.control_port)

        # Wait a second to wake up and connect
        time.sleep(1)

        for item in target(*self.args, **self.kwargs):
            # Classify outcome elements in their corresponding queue
            # for later processing

            if isinstance(item, Page):