<commit_before>#!/usr/bin/env python

#python libs
import csv
import os
import re
import signal
import sys
from datetime import datetime
from time import sleep

#nova libs
from novaclient import base
from novaclient.exceptions import NotFound as NovaNotFound
from novaclient.exceptions import OverLimit as NovaOverLimit
from novaclient.exceptions import ClientException as NovaClientException
from novaclient.v1_1 import client

#cube libs
from cube import Cube


class NovaServiceTest(object):
    '''Class to manage creating and deleting nova instances'''

    def __init__(self, username=None, password=<PASSWORD>, tenant=None,
                 auth_url=None, region=None, keypair=None, auth_ver='2.0',
                 count=1, instance_name='NovaServiceTest', debug=False):

        self.username = username
        self.password = password
        self.tenant = tenant
        self.auth_url = auth_url
        self.region = region
        self.keypair = keypair
        self.auth_ver = auth_ver
        self.count = count
        self.test_name = instance_name
        self.debug = debug

        self.nova = None
        self.timeout = 20
        self.server = {}

        self.path = os.path.dirname(__file__)
        if not self.path:
            self.path = '.'


    def connect(self, force=False):
        '''If we haven't already created a nova client, create one.'''
        if self.nova and not force:
            return

        self.nova = client.Client(username=self.username,
                                  api_key=self.password,
                                  project_id=self.tenant,
                                  auth_url=self.auth_url,
                                  region_name=self.region,
                                  service_type="compute")


    def cleanup(self):
        '''Remove any instances with a matching name then exit.'''
        previous = re.compile('^' + self.test_name)
        exit = False
        for _server in self.nova.servers.list():
            if previous.match(_server.name):
                print("\nDetected active instance from another run, deleting")
                self.nova.servers.delete(_server.id)
                exit = True
        if exit:
            self.dieGracefully(0)


    def set_flavor(self, flavor):
        '''Lookup the specified flavor.'''
        self.flavor = self.nova.flavors.find(name=flavor)


    def set_image(self, image):
        '''Lookup the specified image.'''
        self.image = self.nova.images.find(name=image)


    def create(self):
        '''Create self.count number of instances and time how long it takes.'''
        for i in range(self.count):
            try:
                newserver = self.nova.servers.create(self.test_name + str(i),
                                                     image=self.image,
                                                     flavor=self.flavor,
                                                     key_name=self.keypair)
            except NovaOverLimit as nol:
                print(nol)
                self.dieGracefully()
            except NovaClientException as nce:
                print(nce)
                self.dieGracefully()
            newid = newserver._info['id']
            self.server[newid] = {}
            self.server[newid]['time'] = {}
            self.server[newid]['time']['create_start'] = datetime.now()
            print("Creating server {0}".format(newid))
            sleep(0.5) #prevent being rate-limited

        create_list = self.server.keys()
        active_list = []
        backoff = 1
        while create_list:
            for i in create_list:
                try:
                    _server = self.nova.servers.get(i)
                except NovaClientException as nce:
                    print(nce)
                    self.dieGracefully()

                if _server.status.startswith("BUILD"):
                    pass
                elif _server.status == "ACTIVE":
                    active_list.append(i)
                    create_list.remove(i)
                    self.server[i]['time']['create_end'] = datetime.now()
                    self.server[i]['time']['create_total'] = \
                            self.server[i]['time']['create_end'] - \<commit_msg>We are getting rate-limited on NTT<commit_after>#!/usr/bin/env python

#python libs
import csv
import os
import re
import signal
import sys
from datetime import datetime
from time import sleep

#nova libs
from novaclient import base
from novaclient.exceptions import NotFound as NovaNotFound
from novaclient.exceptions import OverLimit as NovaOverLimit
from novaclient.exceptions import ClientException as NovaClientException
from novaclient.v1_1 import client

#cube libs
from cube import Cube


class NovaServiceTest(object):
    '''Class to manage creating and deleting nova instances'''

    def __init__(self, username=None, password=<PASSWORD>, tenant=None,
                 auth_url=None, region=None, keypair=None, auth_ver='2.0',
                 count=1, instance_name='NovaServiceTest', debug=False):

        self.username = username
        self.password = password
        self.tenant = tenant
        self.auth_url = auth_url
        self.region = region
        self.keypair = keypair
        self.auth_ver = auth_ver
        self.count = count
        self.test_name = instance_name
        self.debug = debug

        self.nova = None
        self.timeout = 20
        self.server = {}

        self.path = os.path.dirname(__file__)
        if not self.path:
            self.path = '.'


    def connect(self, force=False):
        '''If we haven't already created a nova client, create one.'''
        if self.nova and not force:
            return

        self.nova = client.Client(username=self.username,
                                  api_key=self.password,
                                  project_id=self.tenant,
                                  auth_url=self.auth_url,
                                  region_name=self.region,
                                  service_type="compute")


    def cleanup(self):
        '''Remove any instances with a matching name then exit.'''
        previous = re.compile('^' + self.test_name)
        exit = False
        for _server in self.nova.servers.list():
            if previous.match(_server.name):
                print("\nDetected active instance from another run, deleting")
                self.nova.servers.delete(_server.id)
                exit = True
        if exit:
            self.dieGracefully(0)


    def set_flavor(self, flavor):
        '''Lookup the specified flavor.'''
        self.flavor = self.nova.flavors.find(name=flavor)


    def set_image(self, image):
        '''Lookup the specified image.'''
        self.image = self.nova.images.find(name=image)


    def create(self):
        '''Create self.count number of instances and time how long it takes.'''
        for i in range(self.count):
            try:
                newserver = self.nova.servers.create(self.test_name + str(i),
                                                     image=self.image,
                                                     flavor=self.flavor,
                                                     key_name=self.keypair)
            except NovaOverLimit as nol:
                print(nol)
                self.dieGracefully()
            except NovaClientException as nce:
                print(nce)
                self.dieGracefully()
            newid = newserver._info['id']
            self.server[newid] = {}
            self.server[newid]['time'] = {}
            self.server[newid]['time']['create_start'] = datetime.now()
            print("Creating server {0}".format(newid))
            sleep(1) #prevent being rate-limited

        create_list = self.server.keys()
        active_list = []
        backoff = 1
        while create_list:
            for i in create_list:
                try:
                    _server = self.nova.servers.get(i)
                except NovaClientException as nce:
                    print(nce)
                    self.dieGracefully()

                if _server.status.startswith("BUILD"):
                    pass
                elif _server.status == "ACTIVE":
                    active_list.append(i)
                    create_list.remove(i)
                    self.server[i]['time']['create_end'] = datetime.now()
                    self.server[i]['time']['create_total'] = \
                            self.server[i]['time']['create_end'] - \
{'fraction_non_alphanumeric': 0.09389232745533108, 'fraction_numerical': 0.0026859745416326053, 'mean_word_length': 1.6157605375687232, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 22, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3247255', 'n_tokens_mistral': 2293, 'n_tokens_neox': 2185, 'n_words': 543}