<commit_before>import logging
import dart.common.database
import dart.common.query


class BaseCommand(object):
    def __init__(self, **kwargs):
        # everything needs a logger!
        self.logger = logging.getLogger(__name__)

        # get configuration from options
        self.verbose = kwargs.get("verbose", False)


class DataCommand(BaseCommand):
    def __init__(self, **kwargs):
        super().__init__(**kwargs)

        # ignore cassandra errors only if we aren't explicitly in verbose mode
        if (not self.logger.isEnabledFor(logging.DEBUG)):
            logging.getLogger("cassandra").setLevel(logging.ERROR)

        # we ignore warnings when trying to connect because our user doesn't
        # really care if a node is down. if all nodes are down then we still
        # bomb out. here we are just establishing the initial session.
        self.logger.debug("getting cassandra session")
        self.session = dart.common.database.session()
        if (self.session is None):
            raise RuntimeError("could not get connection to cassandra")

        # now assign the session function to the dart query handler
        dart.common.query.session = dart.common.database.session

    def __del__(self):
        try:
            self.session.shutdown()
        except Exception as e:
            pass
<commit_msg>use a different pointer that is more clear<commit_after>import logging
import dart.common.database
import dart.common.query


class BaseCommand(object):
    def __init__(self, **kwargs):
        # everything needs a logger!
        self.logger = logging.getLogger(__name__)

        # get configuration from options
        self.verbose = kwargs.get("verbose", False)


class DataCommand(BaseCommand):
    def __init__(self, **kwargs):
        super().__init__(**kwargs)

        # ignore cassandra errors only if we aren't explicitly in verbose mode
        if (not self.logger.isEnabledFor(logging.DEBUG)):
            logging.getLogger("cassandra").setLevel(logging.ERROR)

        # we ignore warnings when trying to connect because our user doesn't
        # really care if a node is down. if all nodes are down then we still
        # bomb out. here we are just establishing the initial session.
        self.logger.debug("getting cassandra session")
        self.session = dart.common.database.session()
        if (self.session is None):
            raise RuntimeError("could not get connection to cassandra")

        # now assign the session function to the dart query handler
        dart.common.query.session = self.session

    def __del__(self):
        try:
            self.session.shutdown()
        except Exception as e:
            pass

{'fraction_non_alphanumeric': 0.0821917808219178, 'fraction_numerical': 0.0, 'mean_word_length': 2.651351351351351, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5311462', 'n_tokens_mistral': 699, 'n_tokens_neox': 663, 'n_words': 243}