<commit_before>#!/usr/bin/env python

import os
import sys
sys.path.append("..")
sys.path.append(os.path.join(sys.path[0], '..'))

from twisted.internet import reactor, protocol, defer, task
from twisted.python import util

from gearman import client

def run_test(j):
    print repr(j)
    return j

# @defer.inlineCallbacks
def worker(gearman):
    w = client.GearmanWorker(gearman)
    w.registerFunction("test", run_test)

    coop = task.Cooperator()
    coop.coiterate(iter(w))

d=protocol.ClientCreator(reactor, client.GearmanProtocol).connectTCP(
    sys.argv[1], 4730)
d.addCallback(worker)

# sys.settrace(util.spewer)

reactor.run()
<commit_msg>Run example jobs concurrently.<commit_after>#!/usr/bin/env python

import os
import sys
sys.path.append("..")
sys.path.append(os.path.join(sys.path[0], '..'))

from twisted.internet import reactor, protocol, defer, task
from twisted.python import util

from gearman import client

def run_test(j):
    print repr(j)
    return j

# @defer.inlineCallbacks
def worker(gearman):
    w = client.GearmanWorker(gearman)
    w.registerFunction("test", run_test)

    coop = task.Cooperator()
    for i in range(5):
        ## Ramp up start
        reactor.callLater(0.1 * i, lambda: coop.coiterate(iter(w)))
        ## Immediate start
        # coop.coiterate(iter(w))

d=protocol.ClientCreator(reactor, client.GearmanProtocol).connectTCP(
    sys.argv[1], 4730)
d.addCallback(worker)

# sys.settrace(util.spewer)

reactor.run()

{'fraction_non_alphanumeric': 0.13734776725304465, 'fraction_numerical': 0.010148849797023005, 'mean_word_length': 4.082474226804123, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5172323', 'n_tokens_mistral': 541, 'n_tokens_neox': 512, 'n_words': 108}