<commit_before>import gevent
import gevent.queue
import requests


class GreenletRequests():
    NodeResponsesGet = []
    NodeResponsesPost = []
    NodeResponsesPut = []
    NodeResponsesDelete = []
    ng = 0
    np = 0
    npo = 0
    nd = 0

    def __init__(self, resourceList):
        self.resourceList = resourceList

    def parallelGet(self):
        queue = gevent.queue.Queue()
        for i in self.resourceList:
            queue.put(i)

        gList = []
        for t in range(len(self.resourceList)):
            gl = gevent.spawn(getRequest, queue)
            gList.append(gl)

        print str(gList)
        gevent.joinall(gList)

        return GreenletRequests.NodeResponsesGet

    def parallelPost(self, payload):
        queue = gevent.queue.Queue()
        for i in self.resourceList:
            queue.put(i)

        gList = []
        for t in range(len(self.resourceList)):
            gl = gevent.spawn(postRequest, queue, payload)
            gList.append(gl)

        print str(gList)
        gevent.joinall(gList)

        return GreenletRequests.NodeResponsesPost

    def reset(self): # TODO:  Check if reset is required for all attributes separately or general one is ok
        GreenletRequests.NodeResponsesGet = []
        GreenletRequests.NodeResponsesPost = []
        GreenletRequests.NodeResponsesPut = []
        GreenletRequests.NodeResponsesDelete = []




def randomT(queue, name):
    while not queue.empty():
        t = queue.get(timeout = 1)
        gevent.sleep(5)
        print 'I am + '+name+ ' executing ' + str(GreenletRequests.ng)
        GreenletRequests.ng+=1
        gevent.sleep(0)


def getRequest(queue):
    response = {}
    statusCode = {}
    while not queue.empty():
        resURI = queue.get(timeout=1)
        try:
            r = requests.get(resURI, timeout=2)
            data = r.json()
            response['Node'] = resURI
            response['StatusCode'] = r.status_code
            response['Data'] = data
        except requests.exceptions.Timeout:
            response['Node'] = resURI
            statusCode['StatusCode'] = 408
            response['Data'] = 'n/a'
        except requests.exceptions.ConnectionError:
            response['Node'] = resURI
            statusCode['StatusCode'] = 404
            response['Data'] = 'n/a'

        GreenletRequests.NodeResponsesGet.append(response)
        print 'Threaded GET with ID '+str(GreenletRequests.ng)+ ' executed for ' + resURI
        GreenletRequests.ng+=1
        gevent.sleep(0)


def postRequest(queue, payload=None):
    response = {}
    statusCode = {}
    data = {}
    while not queue.empty:
        resourceURI = queue.get(timeout=1)
        response['Node'] = resourceURI
        try:
            if payload is None:
                r = requests.post(resourceURI, timeout=2)
            else:
                r = requests.post(resourceURI, data=payload, timeout=2)
            if r.headers['Content-Type'] == 'application/json':
                data = r.json()
            else:
                data = r.text
            response['StatusCode'] = r.status_code
            response['Data'] = data<commit_msg>fixed unsent status code bug in hreanletThreads<commit_after>import gevent
import gevent.queue
import requests


class GreenletRequests():
    NodeResponsesGet = []
    NodeResponsesPost = []
    NodeResponsesPut = []
    NodeResponsesDelete = []
    ng = 0
    np = 0
    npo = 0
    nd = 0

    def __init__(self, resourceList):
        self.resourceList = resourceList

    def parallelGet(self):
        queue = gevent.queue.Queue()
        for i in self.resourceList:
            queue.put(i)

        gList = []
        for t in range(len(self.resourceList)):
            gl = gevent.spawn(getRequest, queue)
            gList.append(gl)

        print str(gList)
        gevent.joinall(gList)

        return GreenletRequests.NodeResponsesGet

    def parallelPost(self, payload):
        queue = gevent.queue.Queue()
        for i in self.resourceList:
            queue.put(i)

        gList = []
        for t in range(len(self.resourceList)):
            gl = gevent.spawn(postRequest, queue, payload)
            gList.append(gl)

        print str(gList)
        gevent.joinall(gList)

        return GreenletRequests.NodeResponsesPost

    def reset(self): # TODO:  Check if reset is required for all attributes separately or general one is ok
        GreenletRequests.NodeResponsesGet = []
        GreenletRequests.NodeResponsesPost = []
        GreenletRequests.NodeResponsesPut = []
        GreenletRequests.NodeResponsesDelete = []




def randomT(queue, name):
    while not queue.empty():
        t = queue.get(timeout = 1)
        gevent.sleep(5)
        print 'I am + '+name+ ' executing ' + str(GreenletRequests.ng)
        GreenletRequests.ng+=1
        gevent.sleep(0)


def getRequest(queue):
    response = {}
    while not queue.empty():
        resURI = queue.get(timeout=1)
        try:
            r = requests.get(resURI, timeout=2)
            data = r.json()
            response['Node'] = resURI
            response['StatusCode'] = r.status_code
            response['Data'] = data
        except requests.exceptions.Timeout:
            response['Node'] = resURI
            response['StatusCode'] = 408
            response['Data'] = 'n/a'
        except requests.exceptions.ConnectionError:
            response['Node'] = resURI
            response['StatusCode'] = 404
            response['Data'] = 'n/a'

        GreenletRequests.NodeResponsesGet.append(response)
        print 'Threaded GET with ID ' + str(GreenletRequests.ng) + ' executed for ' + resURI
        GreenletRequests.ng += 1
        gevent.sleep(0)


def postRequest(queue, payload=None):
    response = {}
    statusCode = {}
    data = {}
    while not queue.empty:
        resourceURI = queue.get(timeout=1)
        response['Node'] = resourceURI
        try:
            if payload is None:
                r = requests.post(resourceURI, timeout=2)
            else:
                r = requests.post(resourceURI, data=payload, timeout=2)
            if r.headers['Content-Type'] == 'application/json':
                data = r.json()
            else:
                data = r.text
            response['StatusCode'] = r.status_code
            response['Data'] = data
{'fraction_non_alphanumeric': 0.10575256107171001, 'fraction_numerical': 0.006619385342789598, 'mean_word_length': 2.0262279446828804, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2449708', 'n_tokens_mistral': 1851, 'n_tokens_neox': 1812, 'n_words': 403}