# Path: celeryapp/celery.py
# def save_state(sender=None, conf=None, **kwargs):
# def get_message_queue_size(queue_name):
# IP = re.compile(r'\d{1,3}(?:\.\d{1,3}){3}(?::\d{1,6})?')
# 
# Path: celeryapp/celery.py
# def get_message_queue_size(queue_name):
#     client = redis.Redis(IP.findall(config.broker)[0])
#     length = client.llen(queue_name)
#     return length


from celeryapp.celery import app
from celeryapp.celery import get_message_queue_size
from celery import group
from celery.exceptions import Retry
import datetime
import config
import celeryconfig
# -*- coding: utf-8 -*-

from __future__ import absolute_import



# registe all crawlers's task
for crawler in celeryconfig.crawlers:
    # set the crawler_name so we know the namek when parser add new task 
    # and the know call corresponding registed task
    crawler.get('requestHandler').crawler_name = crawler.get('name')
    crawler.get('parseHandler').crawler_name = crawler.get('name')
    crawler.get('scheduler').crawler_name = crawler.get('name')
    crawler.get('pipeline').crawler_name = crawler.get('name')

    # -.-.-.-.-.-.-.-.-.-.-.-.-.-.request task-.-.-.-.-.-.-.-.-.-.-.-.-.

# Path: celeryapp/celery.py
# def save_state(sender=None, conf=None, **kwargs):
# def get_message_queue_size(queue_name):
# IP = re.compile(r'\d{1,3}(?:\.\d{1,3}){3}(?::\d{1,6})?')
# 
# Path: celeryapp/celery.py
# def get_message_queue_size(queue_name):
#     client = redis.Redis(IP.findall(config.broker)[0])
#     length = client.llen(queue_name)
#     return length


# Path: celeryapp/crawl_tasks.py
from celeryapp.celery import app
from celeryapp.celery import get_message_queue_size
from celery import group
from celery.exceptions import Retry
import datetime
import config
import celeryconfig

# -*- coding: utf-8 -*-

from __future__ import absolute_import



# registe all crawlers's task
for crawler in celeryconfig.crawlers:
    # set the crawler_name so we know the namek when parser add new task 
    # and the know call corresponding registed task
    crawler.get('requestHandler').crawler_name = crawler.get('name')
    crawler.get('parseHandler').crawler_name = crawler.get('name')
    crawler.get('scheduler').crawler_name = crawler.get('name')
    crawler.get('pipeline').crawler_name = crawler.get('name')

    # -.-.-.-.-.-.-.-.-.-.-.-.-.-.request task-.-.-.-.-.-.-.-.-.-.-.-.-.

    @app.task(name=crawler.get('name')+'.request', 