# Path: celeryapp/celery.py
# def save_state(sender=None, conf=None, **kwargs):
# def get_message_queue_size(queue_name):
# IP = re.compile(r'\d{1,3}(?:\.\d{1,3}){3}(?::\d{1,6})?')
# 
# Path: celeryapp/celery.py
# def get_message_queue_size(queue_name):
#     client = redis.Redis(IP.findall(config.broker)[0])
#     length = client.llen(queue_name)
#     return length


from celeryapp.celery import app
from celeryapp.celery import get_message_queue_size
from celery import group
from celery.exceptions import Retry
import datetime
import config
import celeryconfig
            except Exception as exc:
                # retry after some secends
                raise self.retry(exc=exc, countdown=config.retry_time)
        except Retry as exc: 
            pass
        except:
            # put task to new_task queue
            if 'retry_cnt' in dict_item:
              dict_item['retry_cnt'] += 1
            else:
              dict_item['retry_cnt'] = 1
            app.tasks[self.crawler_name + '.schedule'].apply_async(([dict_item], ), compression='zlib')
    # -.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.

    # -.-.-.-.-.-.-.-.-.-.-.-.-.-.request_priority task-.-.-.-.-.-.-.-.-
    @app.task(name=crawler.get('name')+'.request_priority', 
              bind=True,
              handler=crawler.get('requestHandler'),
              crawler_name=crawler.get('name'),
              rate_limit=config.rate_limit, 
              time_limit=config.time_limit,
              max_retries=config.max_retries,
              ignore_result=True)
    # fake task 
    def request_priority(self, dict_item, **kwargs):
        try:
            task = self.handler.handle(dict_item, **kwargs)
            if task.get('priority', None):
                app.tasks[self.crawler_name + '.parse_priority'].delay(task)
            else:

# Path: celeryapp/celery.py
# def save_state(sender=None, conf=None, **kwargs):
# def get_message_queue_size(queue_name):
# IP = re.compile(r'\d{1,3}(?:\.\d{1,3}){3}(?::\d{1,6})?')
# 
# Path: celeryapp/celery.py
# def get_message_queue_size(queue_name):
#     client = redis.Redis(IP.findall(config.broker)[0])
#     length = client.llen(queue_name)
#     return length


# Path: celeryapp/crawl_tasks.py
from celeryapp.celery import app
from celeryapp.celery import get_message_queue_size
from celery import group
from celery.exceptions import Retry
import datetime
import config
import celeryconfig

            except Exception as exc:
                # retry after some secends
                raise self.retry(exc=exc, countdown=config.retry_time)
        except Retry as exc: 
            pass
        except:
            # put task to new_task queue
            if 'retry_cnt' in dict_item:
              dict_item['retry_cnt'] += 1
            else:
              dict_item['retry_cnt'] = 1
            app.tasks[self.crawler_name + '.schedule'].apply_async(([dict_item], ), compression='zlib')
    # -.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.

    # -.-.-.-.-.-.-.-.-.-.-.-.-.-.request_priority task-.-.-.-.-.-.-.-.-
    @app.task(name=crawler.get('name')+'.request_priority', 
              bind=True,
              handler=crawler.get('requestHandler'),
              crawler_name=crawler.get('name'),
              rate_limit=config.rate_limit, 
              time_limit=config.time_limit,
              max_retries=config.max_retries,
              ignore_result=True)
    # fake task 
    def request_priority(self, dict_item, **kwargs):
        try:
            task = self.handler.handle(dict_item, **kwargs)
            if task.get('priority', None):
                app.tasks[self.crawler_name + '.parse_priority'].delay(task)
            else:

                app.tasks[self.crawler_name + '.parse'].delay(task)