<commit_before># Copyright (C) 2011 <NAME>

import collections
import logging
import math
import os
import random
import re
import sqlite3
import time
import types

from .instatrace import Instatrace
from . import scoring
from . import tokenizers

log = logging.getLogger("cobe")

# use an empty string to denote the start/end of a chain
_END_TOKEN_TEXT = ""
_NEXT_TOKEN_TABLE = "next_token"
_PREV_TOKEN_TABLE = "prev_token"

_trace = Instatrace()


class Brain:
    """The main interface for Cobe."""
    def __init__(self, filename, instatrace=None):
        """Construct a brain for the specified filename. If that file
        doesn't exist, it will be initialized with the default brain
        settings."""
        if not os.path.exists(filename):
            log.info("File does not exist. Assuming defaults.")
            Brain.init(filename)

        if instatrace is not None:
            _trace.init(instatrace)

        _start = _trace.now()
        self._db = db = _Sql(sqlite3.connect(filename))
        _trace.trace("Brain.connect_us", _trace.now() - _start)

        self.order = int(db.get_info_text("order"))

        self.scorer = scoring.ScorerGroup()
        self.scorer.add_scorer(1.0, scoring.CobeScorer(self.order))

        tokenizer_name = db.get_info_text("tokenizer")
        if tokenizer_name == "MegaHAL":
            self.tokenizer = tokenizers.MegaHALTokenizer()
        else:
            self.tokenizer = tokenizers.CobeTokenizer()

        self.stemmer = None
        stemmer_name = db.get_info_text("stemmer")

        if stemmer_name is not None:
            try:
                self.stemmer = tokenizers.CobeStemmer(stemmer_name)
            except Exception, e:
                log.error("Error creating stemmer: %s", str(e))

        self._end_token_id = db.get_token_id(_END_TOKEN_TEXT)
        self._learning = False

    def start_batch_learning(self):
        """Begin a series of batch learn operations. Data will not be
        committed to the database until stop_batch_learning is
        called. Learn text using the normal learn(text) method."""
        self._learning = True

    def stop_batch_learning(self):
        """Finish a series of batch learn operations."""
        self._learning = False
        self._db.commit()

    def del_stemmer(self):
        self.stemmer = None

        self._db.delete_token_stems()

        self._db.set_info_text("stemmer", None)
        self._db.commit()

    def set_stemmer(self, language):
        self.stemmer = tokenizers.CobeStemmer(language)

        self._db.delete_token_stems()
        self._db.update_token_stems(self.stemmer)

        self._db.set_info_text("stemmer", language)
        self._db.commit()

    def learn(self, text):
        """Learn a string of text. If the input is not already
        Unicode, it will be decoded as utf-8."""
        if type(text) != types.UnicodeType:
            # Assume that non-Unicode text is encoded as utf-8, which
            # should be somewhat safe in the modern world.
            text = text.decode("utf-8", "ignore")

        tokens = self.tokenizer.split(text)
        _trace.trace("Brain.learn_input_token_count", len(tokens))

        if len(tokens) < self.order:
            log.debug("Input too short to learn: %s", text)
            return

        self._learn_tokens(tokens)

    def _learn_tokens(self, tokens):
        db = self._db
        c = db.cursor()

        token_ids = self._get_or_register_tokens(c, tokens)
        n_exprs = len(token_ids) - self.order

        # increment seen count for each token
        db.inc_token_counts(token_ids, c=c)

        links = []

        for i in xrange(n_exprs + 1):
            expr = token_ids[i:i + self.order]
            expr_id = self._get_or_register_expr(c, expr)

            # increment the expr count
            db.inc_expr_count(expr_id, c=c)

            if i == 0:
                # add link to boundary on prev_token
                links.append((_PREV_TOKEN_TABLE, expr_id, self._end_token_id))

            if i > 0:
                # link prev token to this expr
                prev_token = token_ids[i - 1]
                links.append((_PREV_TOKEN_TABLE, expr_id, prev_token))

            if i < n_exprs:
                # link next token to this expr
                next_token = token_ids[i + self.order]
                links.append((_NEXT_TOKEN_TABLE, expr_id, next_token))

            if i == n_exprs:
                # add link to boundary on next_token
                links.append((_NEXT_TOKEN_TABLE, expr_id, self._end_token_id))

        if len(links) > 0:
            db.add_or_inc_links(links, c=c)

        if not self._learning:
            db.commit()

    def reply(self, text):
        """Reply to a string of text. If the input is not already
        Unicode, it will be decoded as utf-8."""
        if type(text) != types.UnicodeType:
            # Assume that non-Unicode text is encoded as utf-8, which
            # should be somewhat safe in the modern world.
            text = text.decode("utf-8", "ignore")

        c = self._db.cursor()

        # each reply gets its own database cache for now
        db_cache = DbCache(self._db)

        tokens = self.tokenizer.split(text)
        input_ids = db_cache.get_token_ids(tokens)

        # filter out unknown words and non-words from the potential pivots
        pivot_set = self._filter_pivots(input_ids)

        # Conflate the known ids with the stems of their words
        if self.stemmer is not None:
            self._conflate_stems(pivot_set, tokens)

        # If we didn't recognize any word tokens in the input, pick
        # something random from the database and babble.
        if len(pivot_set) == 0:
            pivot_set = self._babble(c)

        if len(pivot_set) == 0:
            # we couldn't find any pivot words in _babble(), so we're
            # working with an essentially empty brain. Use the classic
            # MegaHAL reply:
            return "I don't know enough to answer you yet!"

        best_score = -1.0
        best_reply = None

        # loop for half a second
        start = time.time()
        end = start + 0.5
        count = 0

        all_replies = []

        _start = _trace.now()
        while best_reply is None or time.time() < end:
            _now = _trace.now()
            reply = self._generate_reply(pivot_set, db_cache)
            _trace.trace("Brain.generate_reply_us", _trace.now() - _now)

            if reply is None:
                continue

            token_ids, pivot_idx = reply

            _now = _trace.now()
            score = self._evaluate_reply(input_ids, token_ids, db_cache)
            _trace.trace("Brain.evaluate_reply_us", _trace.now() - _now)

            _trace.trace("Brain.reply_output_token_count", len(token_ids))

            count += 1

            if score > best_score:
                best_score = score
                best_reply = token_ids

            # dump all replies to the console if debugging is enabled
            if log.isEnabledFor(logging.DEBUG):
                all_replies.append((score, token_ids, pivot_idx))

        all_replies.sort()
        for score, token_ids, pivot_idx in all_replies:
            words = db_cache.get_token_texts(token_ids)
            words[pivot_idx] = "[%s]" % words[pivot_idx]

            text = self.tokenizer.join(words)
            log.debug("%f %s", score, text.encode("utf-8"))

        _trace.trace("Brain.reply_input_token_count", len(tokens))
        _trace.trace("Brain.known_word_token_count", len(pivot_set))

        _time = _trace.now() - _start
        _trace.trace("Brain.reply_us", _time)
        _trace.trace("Brain.reply_count", count, _time)
        _trace.trace("Brain.best_reply_score", int(best_score * 1000))
        _trace.trace("Brain.best_reply_length", len(best_reply))
        log.debug("made %d replies in %f seconds" % (count,
                                                     time.time() - start))

        # look up the words for these tokens
        _now = _trace.now()
        text = db_cache.get_token_texts(best_reply)
        _trace.trace("Brain.reply_words_lookup_us", _trace.now() - _now)

        return self.tokenizer.join(text)

    def _conflate_stems(self, pivot_set, tokens):
        for token in tokens:
            stem_ids = self._db.get_token_stem_ids(self.stemmer.stem(token))
            if stem_ids is not None:
                # add the tuple of stems to the pivot set, and then
                # remove the individual token_ids
                pivot_set.add(tuple(stem_ids))

                for stem_id in stem_ids:
                    try:
                        pivot_set.remove(stem_id)
                    except KeyError:
                        pass

    def _babble(self, c):
        babble = set()
        for i in xrange(5):
            # Generate a few random tokens that can be used as pivots
            token_id = self._db.get_random_word_token(c=c)

            if token_id is not None:
                babble.add(token_id)

        return babble

    def _filter_pivots(self, pivot_set):
        # remove pivots that might not give good results
        filtered = set()

        for pivot_id in pivot_set:
            if pivot_id is not None \
                    and self._db.get_token_is_word(pivot_id):
                filtered.add(pivot_id)

        return filtered

    def _choose_pivot(self, pivot_ids):
        pivot = random.choice(tuple(pivot_ids))
<commit_msg>Remove the database cursor from the _babble arguments<commit_after># Copyright (C) 2011 <NAME>

import collections
import logging
import math
import os
import random
import re
import sqlite3
import time
import types

from .instatrace import Instatrace
from . import scoring
from . import tokenizers

log = logging.getLogger("cobe")

# use an empty string to denote the start/end of a chain
_END_TOKEN_TEXT = ""
_NEXT_TOKEN_TABLE = "next_token"
_PREV_TOKEN_TABLE = "prev_token"

_trace = Instatrace()


class Brain:
    """The main interface for Cobe."""
    def __init__(self, filename, instatrace=None):
        """Construct a brain for the specified filename. If that file
        doesn't exist, it will be initialized with the default brain
        settings."""
        if not os.path.exists(filename):
            log.info("File does not exist. Assuming defaults.")
            Brain.init(filename)

        if instatrace is not None:
            _trace.init(instatrace)

        _start = _trace.now()
        self._db = db = _Sql(sqlite3.connect(filename))
        _trace.trace("Brain.connect_us", _trace.now() - _start)

        self.order = int(db.get_info_text("order"))

        self.scorer = scoring.ScorerGroup()
        self.scorer.add_scorer(1.0, scoring.CobeScorer(self.order))

        tokenizer_name = db.get_info_text("tokenizer")
        if tokenizer_name == "MegaHAL":
            self.tokenizer = tokenizers.MegaHALTokenizer()
        else:
            self.tokenizer = tokenizers.CobeTokenizer()

        self.stemmer = None
        stemmer_name = db.get_info_text("stemmer")

        if stemmer_name is not None:
            try:
                self.stemmer = tokenizers.CobeStemmer(stemmer_name)
            except Exception, e:
                log.error("Error creating stemmer: %s", str(e))

        self._end_token_id = db.get_token_id(_END_TOKEN_TEXT)
        self._learning = False

    def start_batch_learning(self):
        """Begin a series of batch learn operations. Data will not be
        committed to the database until stop_batch_learning is
        called. Learn text using the normal learn(text) method."""
        self._learning = True

    def stop_batch_learning(self):
        """Finish a series of batch learn operations."""
        self._learning = False
        self._db.commit()

    def del_stemmer(self):
        self.stemmer = None

        self._db.delete_token_stems()

        self._db.set_info_text("stemmer", None)
        self._db.commit()

    def set_stemmer(self, language):
        self.stemmer = tokenizers.CobeStemmer(language)

        self._db.delete_token_stems()
        self._db.update_token_stems(self.stemmer)

        self._db.set_info_text("stemmer", language)
        self._db.commit()

    def learn(self, text):
        """Learn a string of text. If the input is not already
        Unicode, it will be decoded as utf-8."""
        if type(text) != types.UnicodeType:
            # Assume that non-Unicode text is encoded as utf-8, which
            # should be somewhat safe in the modern world.
            text = text.decode("utf-8", "ignore")

        tokens = self.tokenizer.split(text)
        _trace.trace("Brain.learn_input_token_count", len(tokens))

        if len(tokens) < self.order:
            log.debug("Input too short to learn: %s", text)
            return

        self._learn_tokens(tokens)

    def _learn_tokens(self, tokens):
        db = self._db
        c = db.cursor()

        token_ids = self._get_or_register_tokens(c, tokens)
        n_exprs = len(token_ids) - self.order

        # increment seen count for each token
        db.inc_token_counts(token_ids, c=c)

        links = []

        for i in xrange(n_exprs + 1):
            expr = token_ids[i:i + self.order]
            expr_id = self._get_or_register_expr(c, expr)

            # increment the expr count
            db.inc_expr_count(expr_id, c=c)

            if i == 0:
                # add link to boundary on prev_token
                links.append((_PREV_TOKEN_TABLE, expr_id, self._end_token_id))

            if i > 0:
                # link prev token to this expr
                prev_token = token_ids[i - 1]
                links.append((_PREV_TOKEN_TABLE, expr_id, prev_token))

            if i < n_exprs:
                # link next token to this expr
                next_token = token_ids[i + self.order]
                links.append((_NEXT_TOKEN_TABLE, expr_id, next_token))

            if i == n_exprs:
                # add link to boundary on next_token
                links.append((_NEXT_TOKEN_TABLE, expr_id, self._end_token_id))

        if len(links) > 0:
            db.add_or_inc_links(links, c=c)

        if not self._learning:
            db.commit()

    def reply(self, text):
        """Reply to a string of text. If the input is not already
        Unicode, it will be decoded as utf-8."""
        if type(text) != types.UnicodeType:
            # Assume that non-Unicode text is encoded as utf-8, which
            # should be somewhat safe in the modern world.
            text = text.decode("utf-8", "ignore")

        # each reply gets its own database cache for now
        db_cache = DbCache(self._db)

        tokens = self.tokenizer.split(text)
        input_ids = db_cache.get_token_ids(tokens)

        # filter out unknown words and non-words from the potential pivots
        pivot_set = self._filter_pivots(input_ids)

        # Conflate the known ids with the stems of their words
        if self.stemmer is not None:
            self._conflate_stems(pivot_set, tokens)

        # If we didn't recognize any word tokens in the input, pick
        # something random from the database and babble.
        if len(pivot_set) == 0:
            pivot_set = self._babble()

        if len(pivot_set) == 0:
            # we couldn't find any pivot words in _babble(), so we're
            # working with an essentially empty brain. Use the classic
            # MegaHAL reply:
            return "I don't know enough to answer you yet!"

        best_score = -1.0
        best_reply = None

        # loop for half a second
        start = time.time()
        end = start + 0.5
        count = 0

        all_replies = []

        _start = _trace.now()
        while best_reply is None or time.time() < end:
            _now = _trace.now()
            reply = self._generate_reply(pivot_set, db_cache)
            _trace.trace("Brain.generate_reply_us", _trace.now() - _now)

            if reply is None:
                continue

            token_ids, pivot_idx = reply

            _now = _trace.now()
            score = self._evaluate_reply(input_ids, token_ids, db_cache)
            _trace.trace("Brain.evaluate_reply_us", _trace.now() - _now)

            _trace.trace("Brain.reply_output_token_count", len(token_ids))

            count += 1

            if score > best_score:
                best_score = score
                best_reply = token_ids

            # dump all replies to the console if debugging is enabled
            if log.isEnabledFor(logging.DEBUG):
                all_replies.append((score, token_ids, pivot_idx))

        all_replies.sort()
        for score, token_ids, pivot_idx in all_replies:
            words = db_cache.get_token_texts(token_ids)
            words[pivot_idx] = "[%s]" % words[pivot_idx]

            text = self.tokenizer.join(words)
            log.debug("%f %s", score, text.encode("utf-8"))

        _trace.trace("Brain.reply_input_token_count", len(tokens))
        _trace.trace("Brain.known_word_token_count", len(pivot_set))

        _time = _trace.now() - _start
        _trace.trace("Brain.reply_us", _time)
        _trace.trace("Brain.reply_count", count, _time)
        _trace.trace("Brain.best_reply_score", int(best_score * 1000))
        _trace.trace("Brain.best_reply_length", len(best_reply))
        log.debug("made %d replies in %f seconds" % (count,
                                                     time.time() - start))

        # look up the words for these tokens
        _now = _trace.now()
        text = db_cache.get_token_texts(best_reply)
        _trace.trace("Brain.reply_words_lookup_us", _trace.now() - _now)

        return self.tokenizer.join(text)

    def _conflate_stems(self, pivot_set, tokens):
        for token in tokens:
            stem_ids = self._db.get_token_stem_ids(self.stemmer.stem(token))
            if stem_ids is not None:
                # add the tuple of stems to the pivot set, and then
                # remove the individual token_ids
                pivot_set.add(tuple(stem_ids))

                for stem_id in stem_ids:
                    try:
                        pivot_set.remove(stem_id)
                    except KeyError:
                        pass

    def _babble(self):
        token_ids = []
        for i in xrange(5):
            # Generate a few random tokens that can be used as pivots
            token_id = self._db.get_random_word_token()

            if token_id is not None:
                token_ids.append(token_id)

        return token_ids

    def _filter_pivots(self, pivot_set):
        # remove pivots that might not give good results
        filtered = set()

        for pivot_id in pivot_set:
            if pivot_id is not None \
                    and self._db.get_token_is_word(pivot_id):
                filtered.add(pivot_id)

        return filtered

    def _choose_pivot(self, pivot_ids):
        pivot = random.choice(tuple(pivot_ids))

{'fraction_non_alphanumeric': 0.08952320874730589, 'fraction_numerical': 0.0034694843084686956, 'mean_word_length': 2.000157703832203, 'pattern_counts': {'":': 2, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5343511', 'n_tokens_mistral': 5838, 'n_tokens_neox': 5531, 'n_words': 1517}