<commit_before>#
#    ICRAR - International Centre for Radio Astronomy Research
#    (c) UWA - The University of Western Australia, 2016
#    Copyright by UWA (in the framework of the ICRAR)
#    All rights reserved
#
#    This library is free software; you can redistribute it and/or
#    modify it under the terms of the GNU Lesser General Public
#    License as published by the Free Software Foundation; either
#    version 2.1 of the License, or (at your option) any later version.
#
#    This library is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
#    Lesser General Public License for more details.
#
#    You should have received a copy of the GNU Lesser General Public
#    License along with this library; if not, write to the Free Software
#    Foundation, Inc., 59 Temple Place, Suite 330, Boston,
#    MA 02111-1307  USA
#
import subprocess
import sys
import unittest

from dfms import drop
from dfms.manager import client
import time


hostname = 'localhost'


def memory_drop(uid):
    return drop.dropdict({'node':hostname, 'oid':uid, 'uid':uid, 'type':'plain', 'storage':'memory'})

def create_graph(branches, drops_per_branch):
    graph = []
    completed_uids = []
    final_apps = []
    for branch in range(branches):
        for i in range(drops_per_branch):
            data_uid = 'data_%d_branch_%d' % (i, branch)
            app_uid = 'app_%d_branch_%d' % (i, branch)
            data_drop = memory_drop(data_uid)
            app_drop = drop.dropdict({'node':hostname, 'oid':app_uid, 'uid':app_uid, 'type':'app', 'app':'test.graphsRepository.SleepAndCopyApp', 'sleepTime':0})
            data_drop.addConsumer(app_drop)
            graph.append(data_drop)
            graph.append(app_drop)
            if i == 0:
                completed_uids.append(data_uid)
                prev_app = data_drop
            elif i == drops_per_branch - 1:
                final_apps.append(app_drop)
            else:
                data_drop.addProducer(prev_app)

    final_drop = memory_drop('final')
    for final_app in final_apps:
        final_drop.addProducer(final_app)

    graph.append(final_drop)
    return graph, completed_uids

class TestBigGraph(unittest.TestCase):
    """
    A small class that simply checks that the deployment of a considerable-sized
    graph takes no longer than really expected.
    """

    def setUp(self):
        unittest.TestCase.setUp(self)

        args = [sys.executable, '-m', 'dfms.manager.cmdline', 'dfmsNM', '-H', '0.0.0.0', '-qq']
        self.dmProcess = subprocess.Popen(args)

    def tearDown(self):
        self.dmProcess.kill()
        unittest.TestCase.tearDown(self)

    def test_submit_hugegraph(self):

        sessionId = 'lala'
        restPort  = 8888
        args = [sys.executable, '-m', 'dfms.manager.cmdline', 'dfmsDIM', \
                '--port', str(restPort), '-N',hostname, '-qq']

        # Each branch contains a data drop and an app drop
        # All branches connect to a final data drop
        drops_per_branch=5000
        branches = 5
        n_drops = drops_per_branch * branches * 2 + 1
        graph, completed_uids = create_graph(branches=5, drops_per_branch=drops_per_branch)
        self.assertEqual(n_drops, len(graph))

        c = client.NodeManagerClient(port=restPort)
        dimProcess = subprocess.Popen(args)
        try:
            c.create_session(sessionId)
            c.append_graph(sessionId, graph)

            # What we are actually trying to measure with all this stuff
            start = time.time()
            c.deploy_session(sessionId, completed_uids)
            delta = time.time() - start

            # A minute is more than enough, in my PC it takes around 4 or 5 [s]
            # A minute is also way less than the ~2 [h] we observed in AWS
            self.assertLessEqual(delta, 60, "It took way too much time to create %d drops" % (n_drops,))

        finally:
            with client.NodeManagerClient(port=8000) as nm:
                nm.shutdown_node_manager()
            dimProcess.kill()<commit_msg>with terminating()<commit_after>#
#    ICRAR - International Centre for Radio Astronomy Research
#    (c) UWA - The University of Western Australia, 2016
#    Copyright by UWA (in the framework of the ICRAR)
#    All rights reserved
#
#    This library is free software; you can redistribute it and/or
#    modify it under the terms of the GNU Lesser General Public
#    License as published by the Free Software Foundation; either
#    version 2.1 of the License, or (at your option) any later version.
#
#    This library is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
#    Lesser General Public License for more details.
#
#    You should have received a copy of the GNU Lesser General Public
#    License along with this library; if not, write to the Free Software
#    Foundation, Inc., 59 Temple Place, Suite 330, Boston,
#    MA 02111-1307  USA
#
import subprocess
import sys
import unittest

from dfms import drop
from dfms.manager import client
import time
from test.manager import testutils


hostname = 'localhost'


def memory_drop(uid):
    return drop.dropdict({'node':hostname, 'oid':uid, 'uid':uid, 'type':'plain', 'storage':'memory'})

def create_graph(branches, drops_per_branch):
    graph = []
    completed_uids = []
    final_apps = []
    for branch in range(branches):
        for i in range(drops_per_branch):
            data_uid = 'data_%d_branch_%d' % (i, branch)
            app_uid = 'app_%d_branch_%d' % (i, branch)
            data_drop = memory_drop(data_uid)
            app_drop = drop.dropdict({'node':hostname, 'oid':app_uid, 'uid':app_uid, 'type':'app', 'app':'test.graphsRepository.SleepAndCopyApp', 'sleepTime':0})
            data_drop.addConsumer(app_drop)
            graph.append(data_drop)
            graph.append(app_drop)
            if i == 0:
                completed_uids.append(data_uid)
                prev_app = data_drop
            elif i == drops_per_branch - 1:
                final_apps.append(app_drop)
            else:
                data_drop.addProducer(prev_app)

    final_drop = memory_drop('final')
    for final_app in final_apps:
        final_drop.addProducer(final_app)

    graph.append(final_drop)
    return graph, completed_uids

class TestBigGraph(unittest.TestCase):
    """
    A small class that simply checks that the deployment of a considerable-sized
    graph takes no longer than really expected.
    """

    def setUp(self):
        unittest.TestCase.setUp(self)

        args = [sys.executable, '-m', 'dfms.manager.cmdline', 'dfmsNM', '-H', '0.0.0.0', '-qq']
        self.dmProcess = subprocess.Popen(args)

    def tearDown(self):
        self.dmProcess.kill()
        unittest.TestCase.tearDown(self)

    def test_submit_hugegraph(self):

        sessionId = 'lala'
        restPort  = 8888
        args = [sys.executable, '-m', 'dfms.manager.cmdline', 'dfmsDIM', \
                '--port', str(restPort), '-N',hostname, '-qq']

        # Each branch contains a data drop and an app drop
        # All branches connect to a final data drop
        drops_per_branch=5000
        branches = 5
        n_drops = drops_per_branch * branches * 2 + 1
        graph, completed_uids = create_graph(branches=5, drops_per_branch=drops_per_branch)
        self.assertEqual(n_drops, len(graph))

        c = client.NodeManagerClient(port=restPort)
        dimProcess = subprocess.Popen(args)

        with testutils.terminating(dimProcess, 5):
            c.create_session(sessionId)
            c.append_graph(sessionId, graph)

            # What we are actually trying to measure with all this stuff
            start = time.time()
            c.deploy_session(sessionId, completed_uids)
            delta = time.time() - start

            # A minute is more than enough, in my PC it takes around 4 or 5 [s]
            # A minute is also way less than the ~2 [h] we observed in AWS
            self.assertLessEqual(delta, 60, "It took way too much time to create %d drops" % (n_drops,))
{'fraction_non_alphanumeric': 0.0937650457390467, 'fraction_numerical': 0.011194029850746268, 'mean_word_length': 2.631555944055944, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '500920', 'n_tokens_mistral': 2527, 'n_tokens_neox': 2421, 'n_words': 777}