<commit_before>import httplib,urlparse,time,re,sys,time,datetime,os
from celery.task.base import Task
from django.conf import settings
from kral.models import *

if not hasattr(settings, "KRAL_PLUGINS"): 
    for plugin in [x.lower() for x in os.listdir(os.path.join(settings.PROJECT_PATH,'kral/plugins')) if not x.startswith('__')]:
	    __import__('kral.plugins.'+plugin+'.tasks', fromlist=["*"])
else:
    for plugin in settings.KRAL_PLUGINS:
        plugin = plugin.lower()
        try:
	        __import__('kral.plugins.'+plugin+'.tasks', fromlist=["*"])
        except ImportError:
            raise ImportError('Module %s does not exist.' % plugin)

#class PluginController(PeriodicTask):
class PluginController(Task):
#    run_every = settings.KRAL_WAIT
    def run(self, **kwargs):
        i = inspect()
        logger = self.get_logger(**kwargs)
        if not hasattr(settings, 'KRAL_SLOTS'):
            slots = 1
        else:
            slots = settings.KRAL_SLOTS
        querys = Query.objects.order_by('-last_modified')[:slots]
        for query in querys:
            logger.info("Checking if process is running for query: %s" % (query))
            for process in i.active()[socket.gethostname()]:
                if '(<Query: %s>,)' % query not in process['args']:
                    query_worker = Twitter.delay(query)
                    logger.info("Started Twitter process for query: %s" % (query))

def apply_at_worker_start(**kwargs):
    PluginController.delay();   
worker_ready.connect(apply_at_worker_start) 


class ExpandURL(Task):
    def run(self,url,n=1,original_url=None,**kwargs):
        if n == 1:
            original_url = url
        logger = self.get_logger(**kwargs)
        headers = {"User-Agent": "Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.7.6) Gecko/20050512 Firefox"}
        parsed_url = urlparse.urlsplit(url)
        request = urlparse.urlunsplit(('', '', parsed_url.path, parsed_url.query, parsed_url.fragment))
        connection = httplib.HTTPConnection(parsed_url.netloc)
        try : 
            connection.request('HEAD', request, "", headers)
            response = connection.getresponse()<commit_msg>Small code clean-up.<commit_after>import httplib,urlparse,time,re,sys,time,datetime,os
from celery.task.base import Task
from django.conf import settings
from kral.models import *

if not hasattr(settings, "KRAL_PLUGINS"): 
    for plugin in [x.lower() for x in os.listdir(os.path.join(settings.PROJECT_PATH,'kral/plugins')) if not x.startswith('__')]:
	    __import__('kral.plugins.'+plugin+'.tasks', fromlist=["*"])
else:
    for plugin in settings.KRAL_PLUGINS:
        plugin = plugin.lower()
        try:
	        __import__('kral.plugins.'+plugin+'.tasks', fromlist=["*"])
        except ImportError:
            raise ImportError('Module %s does not exist.' % plugin)

#class PluginController(PeriodicTask):
class PluginController(Task):
#    run_every = settings.KRAL_WAIT
    def run(self, **kwargs):
        i = inspect()
        logger = self.get_logger(**kwargs)
        slots = hasattr(settings, 'KRAL_SLOTS', 1):
        querys = Query.objects.order_by('-last_modified')[:slots]
        for query in querys:
            logger.info("Checking if process is running for query: %s" % (query))
            for process in i.active()[socket.gethostname()]:
                if '(<Query: %s>,)' % query not in process['args']:
                    query_worker = Twitter.delay(query)
                    logger.info("Started Twitter process for query: %s" % (query))

def apply_at_worker_start(**kwargs):
    PluginController.delay();   
worker_ready.connect(apply_at_worker_start) 


class ExpandURL(Task):
    def run(self,url,n=1,original_url=None,**kwargs):
        if n == 1:
            original_url = url
        logger = self.get_logger(**kwargs)
        headers = {"User-Agent": "Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.7.6) Gecko/20050512 Firefox"}
        parsed_url = urlparse.urlsplit(url)
        request = urlparse.urlunsplit(('', '', parsed_url.path, parsed_url.query, parsed_url.fragment))
        connection = httplib.HTTPConnection(parsed_url.netloc)
        try : 
            connection.request('HEAD', request, "", headers)
            response = connection.getresponse()
{'fraction_non_alphanumeric': 0.13567251461988303, 'fraction_numerical': 0.010292397660818714, 'mean_word_length': 3.018796992481203, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3557040', 'n_tokens_mistral': 1373, 'n_tokens_neox': 1316, 'n_words': 290}