# Path: celeryapp/crawl_tasks.py
# def request(self, dict_item, **kwargs):
# def request_priority(self, dict_item, **kwargs):
# def parse(self, dict_item, **kwargs):
# def parse_priority(self, dict_item, **kwargs):
# def pipeline(self, dict_item, **kwargs):
# def schedule(self, tasks=None, check_task=False, save_filter=False):
# def crawl_synchronous(self, dict_item, **kwargs):
# 
# Path: celeryapp/celery.py
# def get_message_queue_size(queue_name):
#     client = redis.Redis(IP.findall(config.broker)[0])
#     length = client.llen(queue_name)
#     return length


import os
import sys
import config
import celeryconfig
import logging
import gevent
import json
import time
from celeryapp.crawl_tasks import app
from celeryapp.celery import get_message_queue_size
from celery import group
from gevent import monkey
from gevent.pool import Pool
from gevent.lock import Semaphore
    #           | tasks[crawler['name'] + '.pipeline'].s()).delay()

monkey.patch_all()

scheduel_lock = Semaphore()

filtered_newtasks = []

def schedule():
    global filtered_newtasks
    while 1:
        try:
            if (get_message_queue_size('parse') < config.max_task_queue_size * 2 and 
                get_message_queue_size('pipeline') < config.max_task_queue_size * 2 and
                get_message_queue_size('schedule') < config.max_task_queue_size * 2 and
                get_message_queue_size('request') < config.max_task_queue_size):
                for crawler in celeryconfig.crawlers:
                    #tasks[crawler['name'] + '.schedule'].delay(check_task=True)
                    #for task in tasks[crawler['name']+'.schedule'].scheduler.new_tasks_generator():
                    #    if task.get('priority', None):
                    #        #tasks[crawler['name']+'.request_priority'].delay(task)
                    #        tasks[crawler['name']+'.request_priority'].apply_async((task, ), compression='zlib')
                    #    else:
                    #        #tasks[crawler['name']+'.request'].delay(task)
                    #        tasks[crawler['name']+'.request'].apply_async((task, ), compression='zlib')
                    with scheduel_lock:
                        db = tasks[crawler['name']+'.schedule'].scheduler.get_redis_connection()
                        new_tasks = db.lpop(config.filtered_task_pool_key)
                        if new_tasks:
                            new_tasks = json.loads(new_tasks)

# Path: celeryapp/crawl_tasks.py
# def request(self, dict_item, **kwargs):
# def request_priority(self, dict_item, **kwargs):
# def parse(self, dict_item, **kwargs):
# def parse_priority(self, dict_item, **kwargs):
# def pipeline(self, dict_item, **kwargs):
# def schedule(self, tasks=None, check_task=False, save_filter=False):
# def crawl_synchronous(self, dict_item, **kwargs):
# 
# Path: celeryapp/celery.py
# def get_message_queue_size(queue_name):
#     client = redis.Redis(IP.findall(config.broker)[0])
#     length = client.llen(queue_name)
#     return length


# Path: bin/run.py
import os
import sys
import config
import celeryconfig
import logging
import gevent
import json
import time
from celeryapp.crawl_tasks import app
from celeryapp.celery import get_message_queue_size
from celery import group
from gevent import monkey
from gevent.pool import Pool
from gevent.lock import Semaphore

    #           | tasks[crawler['name'] + '.pipeline'].s()).delay()

monkey.patch_all()

scheduel_lock = Semaphore()

filtered_newtasks = []

def schedule():
    global filtered_newtasks
    while 1:
        try:
            if (get_message_queue_size('parse') < config.max_task_queue_size * 2 and 
                get_message_queue_size('pipeline') < config.max_task_queue_size * 2 and
                get_message_queue_size('schedule') < config.max_task_queue_size * 2 and
                get_message_queue_size('request') < config.max_task_queue_size):
                for crawler in celeryconfig.crawlers:
                    #tasks[crawler['name'] + '.schedule'].delay(check_task=True)
                    #for task in tasks[crawler['name']+'.schedule'].scheduler.new_tasks_generator():
                    #    if task.get('priority', None):
                    #        #tasks[crawler['name']+'.request_priority'].delay(task)
                    #        tasks[crawler['name']+'.request_priority'].apply_async((task, ), compression='zlib')
                    #    else:
                    #        #tasks[crawler['name']+'.request'].delay(task)
                    #        tasks[crawler['name']+'.request'].apply_async((task, ), compression='zlib')
                    with scheduel_lock:
                        db = tasks[crawler['name']+'.schedule'].scheduler.get_redis_connection()
                        new_tasks = db.lpop(config.filtered_task_pool_key)
                        if new_tasks:
                            new_tasks = json.loads(new_tasks)

                        else: