<commit_before>from typing import Iterable
from nomenklatura.loader import DS, E
from nomenklatura.resolver import Resolver
from nomenklatura.index import Index


def xref(
    index: Index[DS, E], resolver: Resolver, entities: Iterable[E], limit: int = 15
) -> None:
    for query in entities:
        assert query.id is not None, query
        for match, score in index.match(query, limit=limit):
            assert match.id is not None, match
            resolver.suggest(query.id, match.id, score)<commit_msg>Add cool logging to xref<commit_after>import math
import logging
from typing import Iterable, List

from nomenklatura.loader import DS, E
from nomenklatura.resolver import Resolver
from nomenklatura.index import Index

log = logging.getLogger(__name__)


def _print_stats(num_entities: int, scores: List[float]) -> None:
    matches = len(scores)
    log.info(
        "Xref: %d entities, %d matches, avg: %.2f, min: %.2f, max: %.2f",
        num_entities,
        matches,
        sum(scores) / max(1, matches),
        min(scores, default=0.0),
        max(scores, default=0.0),
    )


def xref(
    index: Index[DS, E], resolver: Resolver, entities: Iterable[E], limit: int = 15
) -> None:
    log.info("Begin xref: %r, resolver: %r", index, resolver)
    scores: List[float] = []
    for num_entities, query in enumerate(entities):
        assert query.id is not None, query
        for match, score in index.match(query, limit=limit):
            assert match.id is not None, match
            log.debug("[%.2f]-> %r x %r", score, query, match)
            resolver.suggest(query.id, match.id, score)
            scores.append(score)

        if num_entities % 100 == 0 and num_entities > 0:
            _print_stats(num_entities, scores)

    _print_stats(num_entities, scores)
{'fraction_non_alphanumeric': 0.11129660545353366, 'fraction_numerical': 0.01001669449081803, 'mean_word_length': 2.9955555555555557, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '312071', 'n_tokens_mistral': 578, 'n_tokens_neox': 564, 'n_words': 178}