<commit_before>from unittest2 import TestCase
import os

from pyramid import testing
from mozsvc.config import load_into_settings
from mozsvc.plugin import load_and_register
from sqlalchemy.exc import IntegrityError

from tokenserver import logger
from tokenserver.assignment import INodeAssignment


class TestShardedNode(TestCase):

    def setUp(self):
        super(TestCase, self).setUp()

        # get the options from the config
        self.config = testing.setUp()
        self.ini = os.path.join(os.path.dirname(__file__),
                                'test_sharded.ini')
        settings = {}
        load_into_settings(self.ini, settings)
        self.config.add_settings(settings)

        # instantiate the backend to test
        self.config.include("tokenserver")
        load_and_register("tokenserver", self.config)
        self.backend = self.config.registry.getUtility(INodeAssignment)

        # adding a node with 100 slots
        self.backend._safe_execute('sync',
              """insert into nodes (`node`, `service`, `version`, `available`,
                    `capacity`, `current_load`, `downed`, `backoff`)
                  values ("phx12", "sync", "1.0", 100, 100, 0, 0, 0)""")

        self._sqlite = self.backend._dbs['sync'][0].driver == 'pysqlite'

    def tearDown(self):
        for engine, __, __ in self.backend._dbs.values():<commit_msg>changed the way the sharding works<commit_after>from unittest2 import TestCase
import os

from pyramid import testing
from mozsvc.config import load_into_settings
from mozsvc.plugin import load_and_register
from sqlalchemy.exc import IntegrityError

from tokenserver import logger
from tokenserver.assignment import INodeAssignment


class TestShardedNode(TestCase):

    def setUp(self):
        super(TestCase, self).setUp()

        # get the options from the config
        self.config = testing.setUp()
        self.ini = os.path.join(os.path.dirname(__file__),
                                'test_sharded.ini')
        settings = {}
        load_into_settings(self.ini, settings)
        self.config.add_settings(settings)

        # instantiate the backend to test
        self.config.include("tokenserver")
        load_and_register("tokenserver", self.config)
        self.backend = self.config.registry.getUtility(INodeAssignment)

        # adding a node with 100 slots
        self.backend._safe_execute(
              """insert into nodes (`node`, `service`, `version`, `available`,
                    `capacity`, `current_load`, `downed`, `backoff`)
                  values ("phx12", "sync", "1.0", 100, 100, 0, 0, 0)""",
                  service='sync')

        self._sqlite = self.backend._dbs['sync'][0].driver == 'pysqlite'

    def tearDown(self):
        for val in self.backend._dbs.values():
            engine = val[0]
{'fraction_non_alphanumeric': 0.10745846588900672, 'fraction_numerical': 0.013078826440438318, 'mean_word_length': 2.7041884816753927, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '968196', 'n_tokens_mistral': 840, 'n_tokens_neox': 790, 'n_words': 201}