<commit_before>from __future__ import annotations

import logging
import math
from typing import Iterable

import numpy
from BTrees.Length import Length
from zope.index.text.cosineindex import CosineIndex
from zope.index.text.lexicon import Lexicon
from zope.index.text.setops import mass_weightedUnion
from zope.index.text.textindex import TextIndex

logger = logging.getLogger(__name__)


class CanopyIndex(TextIndex):  # pragma: no cover
    def __init__(self) -> None:
        lexicon = CanopyLexicon()
        self.index = CosineIndex(lexicon)
        self.lexicon = lexicon

    def initSearch(self) -> None:
        N = len(self.index._docweight)
        threshold = int(max(1000, N * 0.05))

        stop_words = []
        self._wids_dict = {}

        bucket = self.index.family.IF.Bucket
        for wid, docs in self.index._wordinfo.items():
            if len(docs) > threshold:
                stop_words.append(wid)
                continue
            if isinstance(docs, dict):
                docs = bucket(docs)
            idf = numpy.log1p(N / len(docs))
            self.index._wordinfo[wid] = docs
            term = self.lexicon._words[wid]
            self._wids_dict[term] = (wid, idf)

        for wid in stop_words:
            word = self.lexicon._words.pop(wid)
            logger.info(f"Removing stop word {word}")
            del self.index._wordinfo[wid]

    def apply(
        self,
        query_list: Iterable[str],
        threshold: float,
        start: int = 0,
        count: int | None = None,
    ) -> list[tuple[float, int]]:
        _wids_dict = self._wids_dict
        _wordinfo = self.index._wordinfo
        l_pow = float.__pow__

        L = []
        qw = 0.0

        for term in query_list:<commit_msg>also reset canopy wids when removing stop words, closes #1069<commit_after>from __future__ import annotations

import logging
import math
from typing import Iterable

import numpy
from BTrees.Length import Length
from zope.index.text.cosineindex import CosineIndex
from zope.index.text.lexicon import Lexicon
from zope.index.text.setops import mass_weightedUnion
from zope.index.text.textindex import TextIndex

logger = logging.getLogger(__name__)


class CanopyIndex(TextIndex):  # pragma: no cover
    def __init__(self) -> None:
        lexicon = CanopyLexicon()
        self.index = CosineIndex(lexicon)
        self.lexicon = lexicon

    def initSearch(self) -> None:
        N = len(self.index._docweight)
        threshold = int(max(1000, N * 0.05))

        stop_words = []
        self._wids_dict = {}

        bucket = self.index.family.IF.Bucket
        for wid, docs in self.index._wordinfo.items():
            if len(docs) > threshold:
                stop_words.append(wid)
                continue

            if isinstance(docs, dict):
                docs = bucket(docs)
                self.index._wordinfo[wid] = docs

            idf = numpy.log1p(N / len(docs))
            term = self.lexicon._words[wid]

            self._wids_dict[term] = (wid, idf)

        for wid in stop_words:
            word = self.lexicon._words.pop(wid)
            del self.lexicon._wids[word]
            logger.info(f"Removing stop word {word}")
            del self.index._wordinfo[wid]

    def apply(
        self,
        query_list: Iterable[str],
        threshold: float,
        start: int = 0,
        count: int | None = None,
    ) -> list[tuple[float, int]]:
        _wids_dict = self._wids_dict
        _wordinfo = self.index._wordinfo
        l_pow = float.__pow__

        L = []
        qw = 0.0

        for term in query_list:
{'fraction_non_alphanumeric': 0.09534368070953436, 'fraction_numerical': 0.007206208425720621, 'mean_word_length': 2.108527131782946, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1421590', 'n_tokens_mistral': 1133, 'n_tokens_neox': 1074, 'n_words': 268}