# Path: wikidat/utils/comutils.py
# def send_ujson(socket, obj, flags=0):
#     """Serialize object using ultra-fast ujson"""
#     m = ujson.dumps(obj)
#     z = zlib.compress(m.encode())
#     return socket.send(z, flags=flags)
# 
# def recv_ujson(socket, flags=0):
#     """Load object from ujson serialization"""
#     z = socket.recv(flags)
#     m = zlib.decompress(z).decode()
#     return ujson.loads(m)
# 
# Path: wikidat/retrieval/page.py
# class Page(DataItem):
#     """
#     Models Page elements in Wikipedia database dumps
#     """
# 
#     def __init__(self, *args, **kwargs):
#         """
#         Constructor method for Page objects. Must forward params to
#         parent class DataItem (mandatory inheritance)
# 
#         The following keys must be populated:
#         ---------
#         * id: Unique numeric identifier for this page
#         * ns: Namespace code for this page (0 is main)
#         * title: Page title (unicode string)
#         * restrictions: List of applicable restrictions to this page
#         """
#         super(Page, self).__init__(*args, **kwargs)
# 
# Path: wikidat/retrieval/revision.py
# class Revision(DataItem):
#     """
#     Models Revision elements in Wikipedia dump files
#     """
# 
#     def __init__(self, *args, **kwargs):
#         """
#         Constructor method for Revision objects. Must forward params to
#         parent class DataItem (mandatory inheritance)
# 
#         The following keys must be populated:
#         ---------
#         * id: Unique numeric identifier for this revision
#         * rev_user: Numeric identifier of author of this revision (0 is anon)
#         * rev_timestamp: Timestamp when this rev was saved in database
#         * rev_len: Length in bytes of this revision
#         * ... (other params)
#         """
#         super(Revision, self).__init__(*args, **kwargs)
# 
# Path: wikidat/retrieval/logitem.py
# class LogItem(DataItem):
#     """
#     Models LogItem elements extracted from the 'logging' DB table in Wikipedia
#     """
# 
#     def __init__(self, *args, **kwargs):
#         """
#         Constructor method for Page objects. Must forward params to
#         parent class DataItem (mandatory inheritance)
# 
#         Parameters
#         ----------
#         data_dict: dictionary of raw data to be processed
#         lang: identifier of Wikipedia language edition from which this
#         element comes from (e.g. frwiki, eswiki, dewiki...)
#         """
#         super(LogItem, self).__init__(*args, **kwargs)


import time
import multiprocessing as mp
import zmq
from wikidat.utils.comutils import send_ujson, recv_ujson
from .page import Page
from .revision import Revision
from .logitem import LogItem


class Consumer(mp.Process):
    """
    Consumes items from ZMQ pipeline, coming from Processors.
    http://nichol.as/zeromq-an-introduction

    The "target" must be a function which expects an iterable as it's
    only argument.  Therefore, the args value is not used here.
    """
    def __init__(self, group=None, target=None, name=None, args=None,
                 kwargs=None, producers=0, pull_port=None):

        super(Consumer, self).__init__(name=name)
        self.target = target
        self.args = args if args is not None else []
        self.kwargs = kwargs if kwargs is not None else {}
        self.producers = producers
        self.pull_port = pull_port

    def items(self):
        context = zmq.Context()
        data_recv = context.socket(zmq.PULL)
        data_recv.bind("tcp://127.0.0.1:"+str(self.pull_port))

        # Wait a second to wake up and connect
        time.sleep(1)

        while self.producers > 0:
            while True:

# Path: wikidat/utils/comutils.py
# def send_ujson(socket, obj, flags=0):
#     """Serialize object using ultra-fast ujson"""
#     m = ujson.dumps(obj)
#     z = zlib.compress(m.encode())
#     return socket.send(z, flags=flags)
# 
# def recv_ujson(socket, flags=0):
#     """Load object from ujson serialization"""
#     z = socket.recv(flags)
#     m = zlib.decompress(z).decode()
#     return ujson.loads(m)
# 
# Path: wikidat/retrieval/page.py
# class Page(DataItem):
#     """
#     Models Page elements in Wikipedia database dumps
#     """
# 
#     def __init__(self, *args, **kwargs):
#         """
#         Constructor method for Page objects. Must forward params to
#         parent class DataItem (mandatory inheritance)
# 
#         The following keys must be populated:
#         ---------
#         * id: Unique numeric identifier for this page
#         * ns: Namespace code for this page (0 is main)
#         * title: Page title (unicode string)
#         * restrictions: List of applicable restrictions to this page
#         """
#         super(Page, self).__init__(*args, **kwargs)
# 
# Path: wikidat/retrieval/revision.py
# class Revision(DataItem):
#     """
#     Models Revision elements in Wikipedia dump files
#     """
# 
#     def __init__(self, *args, **kwargs):
#         """
#         Constructor method for Revision objects. Must forward params to
#         parent class DataItem (mandatory inheritance)
# 
#         The following keys must be populated:
#         ---------
#         * id: Unique numeric identifier for this revision
#         * rev_user: Numeric identifier of author of this revision (0 is anon)
#         * rev_timestamp: Timestamp when this rev was saved in database
#         * rev_len: Length in bytes of this revision
#         * ... (other params)
#         """
#         super(Revision, self).__init__(*args, **kwargs)
# 
# Path: wikidat/retrieval/logitem.py
# class LogItem(DataItem):
#     """
#     Models LogItem elements extracted from the 'logging' DB table in Wikipedia
#     """
# 
#     def __init__(self, *args, **kwargs):
#         """
#         Constructor method for Page objects. Must forward params to
#         parent class DataItem (mandatory inheritance)
# 
#         Parameters
#         ----------
#         data_dict: dictionary of raw data to be processed
#         lang: identifier of Wikipedia language edition from which this
#         element comes from (e.g. frwiki, eswiki, dewiki...)
#         """
#         super(LogItem, self).__init__(*args, **kwargs)


# Path: wikidat/retrieval/processors.py
import time
import multiprocessing as mp
import zmq
from wikidat.utils.comutils import send_ujson, recv_ujson
from .page import Page
from .revision import Revision
from .logitem import LogItem



class Consumer(mp.Process):
    """
    Consumes items from ZMQ pipeline, coming from Processors.
    http://nichol.as/zeromq-an-introduction

    The "target" must be a function which expects an iterable as it's
    only argument.  Therefore, the args value is not used here.
    """
    def __init__(self, group=None, target=None, name=None, args=None,
                 kwargs=None, producers=0, pull_port=None):

        super(Consumer, self).__init__(name=name)
        self.target = target
        self.args = args if args is not None else []
        self.kwargs = kwargs if kwargs is not None else {}
        self.producers = producers
        self.pull_port = pull_port

    def items(self):
        context = zmq.Context()
        data_recv = context.socket(zmq.PULL)
        data_recv.bind("tcp://127.0.0.1:"+str(self.pull_port))

        # Wait a second to wake up and connect
        time.sleep(1)

        while self.producers > 0:
            while True:

                item = recv_ujson(data_recv)