<commit_before>from zope.interface import implements

from twisted.python import usage, log
from twisted.plugin import IPlugin
from twisted.application.service import IServiceMaker
from twisted.application import internet
from twisted.web.client import getPage, HTTPClientFactory

from datetime import datetime
import re
import os
from urlparse import urljoin

class Options(usage.Options):
    optParameters = [["settings", "s", None, "Django settings module."],
                     ["time", "t", "1", "Poll every n seconds."],
                     ]

def getPoller(options):
    if options["settings"] is None:
            raise usage.UsageError("specify a settings module")
    os.environ["DJANGO_SETTINGS_MODULE"] = options["settings"]
    from pushes.models import Repository, Forest
    from pushes.utils import getURL, handlePushes

    class PushPoller(object):
        def __init__(self, opts):
            self.timeout = 2
            self.limit = int(opts.get('limit', 200))
            self.runnings = 0
            self.parallels = 2
            self.repos = []
            self.start_cycle = None
            pass
        def poll(self):
            if self.runnings >= self.parallels:
                log.msg("skipping a cycle, I'm too busy")
                return
            if not self.repos:
                n = datetime.now()
                if self.start_cycle is not None:
                    lag = n - self.start_cycle
                    log.msg("Cycle took %d seconds" % lag.seconds)
                self.start_cycle = n
                self.repos = list(Repository.objects.filter(forest__isnull = 
                                                            True))
                self.parallels = 1 # make sure to have all forests
                for forest in Forest.objects.all():
                    url = str(forest.url + '?style=raw')
                    d = getPage(url, timeout = self.timeout)
                    d.addCallback(self.gotForest, forest)
                    d.addErrback(self.failedForest, forest)
                    self.runnings += 1
                return
            repo = self.repos.pop()

            jsonurl = getURL(repo, self.limit)
            d = getPage(str(jsonurl))
            d.addCallback(handlePushes, repo)
            d.addErrback(self.jsonErr, repo)
            def decreaseRunning(ignored):
                self.runnings -= 1
            d.addBoth(decreaseRunning)
            self.runnings += 1
            

        def gotForest(self, page, forest):
            self.runnings -= 1
            if self.runnings == 0:
                # done with forests, reset our parallels back to max parallels
                self.parellels = 2
            links = filter(None, re.split(r'\s+', page))
            urls = map(lambda link: urljoin(forest.url, link), links)
            q = Repository.objects.filter(url__in = urls)
            self.repos += list(q)
            known_urls = q.values_list('url', flat=True)
            for i in xrange(len(urls)):
                if urls[i] in known_urls:
                    continue
                name = links[i].strip('/')
                if not False:
                    log.msg("adding %s: %s" % (name, urls[i]))
                r = Repository.objects.create(name = name, url = urls[i],
                                              forest = forest)
                self.repos.append(r)

        def failedForest(self, failure, forest):
            log.err(failure, "failed to load %s" % forest.name)
        def jsonErr(self, failure, repo):
            log.err(failure, "failed to load json for %s" % repo.name)

    pp = PushPoller(options)
    return pp.poll

class MyServiceMaker(object):
    implements(IServiceMaker, IPlugin)
    tapname = "get-pushes"
    description = "Gotcha grabbin' repos."
    options = Options

    def makeService(self, options):
        """
        Construct a TCPServer from a factory defined in myproject.
        """
        HTTPClientFactory.noisy = False
        poller = getPoller(options)
        return internet.TimerService(1, poller)


# Now construct an object which *provides* the relevant interfaces
# The name of this variable is irrelevant, as long as there is *some*
# name bound to a provider of IPlugin and IServiceMaker.

serviceMaker = MyServiceMaker()<commit_msg>actually use the commandline argument for timer frequency<commit_after>from zope.interface import implements

from twisted.python import usage, log
from twisted.plugin import IPlugin
from twisted.application.service import IServiceMaker
from twisted.application import internet
from twisted.web.client import getPage, HTTPClientFactory

from datetime import datetime
import re
import os
from urlparse import urljoin

class Options(usage.Options):
    optParameters = [["settings", "s", None, "Django settings module."],
                     ["time", "t", "1", "Poll every n seconds."],
                     ]

def getPoller(options):
    if options["settings"] is None:
            raise usage.UsageError("specify a settings module")
    os.environ["DJANGO_SETTINGS_MODULE"] = options["settings"]
    from pushes.models import Repository, Forest
    from pushes.utils import getURL, handlePushes

    class PushPoller(object):
        def __init__(self, opts):
            self.timeout = 2
            self.limit = int(opts.get('limit', 200))
            self.runnings = 0
            self.parallels = 2
            self.repos = []
            self.start_cycle = None
            pass
        def poll(self):
            if self.runnings >= self.parallels:
                log.msg("skipping a cycle, I'm too busy")
                return
            if not self.repos:
                n = datetime.now()
                if self.start_cycle is not None:
                    lag = n - self.start_cycle
                    log.msg("Cycle took %d seconds" % lag.seconds)
                self.start_cycle = n
                self.repos = list(Repository.objects.filter(forest__isnull = 
                                                            True))
                self.parallels = 1 # make sure to have all forests
                for forest in Forest.objects.all():
                    url = str(forest.url + '?style=raw')
                    d = getPage(url, timeout = self.timeout)
                    d.addCallback(self.gotForest, forest)
                    d.addErrback(self.failedForest, forest)
                    self.runnings += 1
                return
            repo = self.repos.pop()

            jsonurl = getURL(repo, self.limit)
            d = getPage(str(jsonurl))
            d.addCallback(handlePushes, repo)
            d.addErrback(self.jsonErr, repo)
            def decreaseRunning(ignored):
                self.runnings -= 1
            d.addBoth(decreaseRunning)
            self.runnings += 1
            

        def gotForest(self, page, forest):
            self.runnings -= 1
            if self.runnings == 0:
                # done with forests, reset our parallels back to max parallels
                self.parellels = 2
            links = filter(None, re.split(r'\s+', page))
            urls = map(lambda link: urljoin(forest.url, link), links)
            q = Repository.objects.filter(url__in = urls)
            self.repos += list(q)
            known_urls = q.values_list('url', flat=True)
            for i in xrange(len(urls)):
                if urls[i] in known_urls:
                    continue
                name = links[i].strip('/')
                if not False:
                    log.msg("adding %s: %s" % (name, urls[i]))
                r = Repository.objects.create(name = name, url = urls[i],
                                              forest = forest)
                self.repos.append(r)

        def failedForest(self, failure, forest):
            log.err(failure, "failed to load %s" % forest.name)
        def jsonErr(self, failure, repo):
            log.err(failure, "failed to load json for %s" % repo.name)

    pp = PushPoller(options)
    return pp.poll

class MyServiceMaker(object):
    implements(IServiceMaker, IPlugin)
    tapname = "get-pushes"
    description = "Gotcha grabbin' repos."
    options = Options

    def makeService(self, options):
        """
        Construct a TCPServer from a factory defined in myproject.
        """
        HTTPClientFactory.noisy = False
        poller = getPoller(options)
        timer = float(options['time'])
        return internet.TimerService(timer, poller)


# Now construct an object which *provides* the relevant interfaces
# The name of this variable is irrelevant, as long as there is *some*
# name bound to a provider of IPlugin and IServiceMaker.

serviceMaker = MyServiceMaker()
{'fraction_non_alphanumeric': 0.09331810394060537, 'fraction_numerical': 0.0033123929183323814, 'mean_word_length': 1.7577952755905513, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4019163', 'n_tokens_mistral': 2420, 'n_tokens_neox': 2294, 'n_words': 668}