<commit_before>#! /usr/bin/env python
# -*- coding: iso-8859-1 -*-
# vi:ts=4:et

import pycurl
import unittest
import select

from . import app
from . import runwsgi
from . import util

setup_module_1, teardown_module_1 = runwsgi.app_runner_setup((app.app, 8380))
setup_module_2, teardown_module_2 = runwsgi.app_runner_setup((app.app, 8381))
setup_module_3, teardown_module_3 = runwsgi.app_runner_setup((app.app, 8382))

def setup_module(mod):
    setup_module_1(mod)
    setup_module_2(mod)
    setup_module_3(mod)

def teardown_module(mod):
    teardown_module_3(mod)
    teardown_module_2(mod)
    teardown_module_1(mod)

class MultiSocketSelectTest(unittest.TestCase):
    def test_multi_socket_select(self):
        sockets = set()
        timeout = 0

        urls = [
            'http://localhost:8380/success',
            'http://localhost:8381/success',
            'http://localhost:8382/success',
        ]

        socket_events = []
        
        # socket callback
        def socket(event, socket, multi, data):
            if event == pycurl.POLL_REMOVE:
                #print("Remove Socket %d"%socket)
                sockets.remove(socket)
            else:
                if socket not in sockets:
                    #print("Add socket %d"%socket)
                    sockets.add(socket)
            socket_events.append((event, multi))

        # init
        m = pycurl.CurlMulti()
        m.setopt(pycurl.M_PIPELINING, 1)
        m.setopt(pycurl.M_SOCKETFUNCTION, socket)
        m.handles = []
        for url in urls:
            c = pycurl.Curl()
            # save info in standard Python attributes
            c.url = url
            c.body = util.StringIO()
            c.http_code = -1
            m.handles.append(c)
            # pycurl API calls
            c.setopt(c.URL, c.url)
            c.setopt(c.WRITEFUNCTION, c.body.write)
            m.add_handle(c)

        # get data
        num_handles = len(m.handles)

        while (pycurl.E_CALL_MULTI_PERFORM==m.socket_all()[0]):
            pass
            
        timeout = m.timeout()

        # timeout might be -1, indicating that all work is done
        # XXX make sure there is always work to be done here?
        while timeout >= 0:
            (rr, wr, er) = select.select(sockets,sockets,sockets,timeout/1000.0)
            socketSet = set(rr+wr+er)
            if socketSet:
                for s in socketSet:
                    while True:
                        (ret,running) = m.socket_action(s,0)
                        if ret!=pycurl.E_CALL_MULTI_PERFORM:
                            break
            else:
                (ret,running) = m.socket_action(pycurl.SOCKET_TIMEOUT,0)
            if running==0:
                break

        for c in m.handles:
            # save info in standard Python attributes
            c.http_code = c.getinfo(c.HTTP_CODE)

        # at least in and remove events per socket
        assert len(socket_events) >= 6

        # print result
        for c in m.handles:<commit_msg>This test fails intermittently, add diagnostics<commit_after>#! /usr/bin/env python
# -*- coding: iso-8859-1 -*-
# vi:ts=4:et

import pycurl
import unittest
import select

from . import app
from . import runwsgi
from . import util

setup_module_1, teardown_module_1 = runwsgi.app_runner_setup((app.app, 8380))
setup_module_2, teardown_module_2 = runwsgi.app_runner_setup((app.app, 8381))
setup_module_3, teardown_module_3 = runwsgi.app_runner_setup((app.app, 8382))

def setup_module(mod):
    setup_module_1(mod)
    setup_module_2(mod)
    setup_module_3(mod)

def teardown_module(mod):
    teardown_module_3(mod)
    teardown_module_2(mod)
    teardown_module_1(mod)

class MultiSocketSelectTest(unittest.TestCase):
    def test_multi_socket_select(self):
        sockets = set()
        timeout = 0

        urls = [
            'http://localhost:8380/success',
            'http://localhost:8381/success',
            'http://localhost:8382/success',
        ]

        socket_events = []
        
        # socket callback
        def socket(event, socket, multi, data):
            if event == pycurl.POLL_REMOVE:
                #print("Remove Socket %d"%socket)
                sockets.remove(socket)
            else:
                if socket not in sockets:
                    #print("Add socket %d"%socket)
                    sockets.add(socket)
            socket_events.append((event, multi))

        # init
        m = pycurl.CurlMulti()
        m.setopt(pycurl.M_PIPELINING, 1)
        m.setopt(pycurl.M_SOCKETFUNCTION, socket)
        m.handles = []
        for url in urls:
            c = pycurl.Curl()
            # save info in standard Python attributes
            c.url = url
            c.body = util.StringIO()
            c.http_code = -1
            m.handles.append(c)
            # pycurl API calls
            c.setopt(c.URL, c.url)
            c.setopt(c.WRITEFUNCTION, c.body.write)
            m.add_handle(c)

        # get data
        num_handles = len(m.handles)

        while (pycurl.E_CALL_MULTI_PERFORM==m.socket_all()[0]):
            pass
            
        timeout = m.timeout()

        # timeout might be -1, indicating that all work is done
        # XXX make sure there is always work to be done here?
        while timeout >= 0:
            (rr, wr, er) = select.select(sockets,sockets,sockets,timeout/1000.0)
            socketSet = set(rr+wr+er)
            if socketSet:
                for s in socketSet:
                    while True:
                        (ret,running) = m.socket_action(s,0)
                        if ret!=pycurl.E_CALL_MULTI_PERFORM:
                            break
            else:
                (ret,running) = m.socket_action(pycurl.SOCKET_TIMEOUT,0)
            if running==0:
                break

        for c in m.handles:
            # save info in standard Python attributes
            c.http_code = c.getinfo(c.HTTP_CODE)

        # at least in and remove events per socket
        assert len(socket_events) >= 6, 'Less than 6 socket events: %s' % repr(socket_events)

        # print result
        for c in m.handles:
{'fraction_non_alphanumeric': 0.09793730713009582, 'fraction_numerical': 0.01867792756212441, 'mean_word_length': 1.773873873873874, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5505445', 'n_tokens_mistral': 2004, 'n_tokens_neox': 1883, 'n_words': 419}