<commit_before># -*- coding: utf-8 -*-
# This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later.
# Copyright © <NAME>. See NOTICE for more information.
#
from django.conf import settings
from django.contrib.contenttypes.models import ContentType
from django.core.cache import cache

from .models import Tag, Book
from os.path import getmtime
import cPickle
from collections import defaultdict


BOOK_CATEGORIES = ('author', 'epoch', 'genre', 'kind')

_COUNTERS = None
_COUNTER_TIME = None


def get_top_level_related_tags(tags, categories=None):
    """
    Finds tags related to given tags through books, and counts their usage.

    Takes ancestry into account: if a tag is applied to a book, its
    usage on the book's descendants is ignored.
    """
    global _COUNTERS, _COUNTER_TIME
    # First, check that we have a valid and recent version of the counters.
    if getmtime(settings.CATALOGUE_COUNTERS_FILE) > _COUNTER_TIME:
        with open(settings.CATALOGUE_COUNTERS_FILE) as f:
            _COUNTERS = cPickle.load(f)

    tagids = tuple(sorted(t.pk for t in tags))
    try:
        related_ids = _COUNTERS['next'][tagids]
    except KeyError:
        return
<commit_msg>multiple tries loading counters<commit_after># -*- coding: utf-8 -*-
# This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later.
# Copyright © <NAME>. See NOTICE for more information.
#
from django.conf import settings
from django.contrib.contenttypes.models import ContentType
from django.core.cache import cache

from .models import Tag, Book
from os.path import getmtime
import cPickle
from collections import defaultdict


BOOK_CATEGORIES = ('author', 'epoch', 'genre', 'kind')

_COUNTERS = None
_COUNTER_TIME = None


def get_top_level_related_tags(tags, categories=None):
    """
    Finds tags related to given tags through books, and counts their usage.

    Takes ancestry into account: if a tag is applied to a book, its
    usage on the book's descendants is ignored.
    """
    global _COUNTERS, _COUNTER_TIME
    # First, check that we have a valid and recent version of the counters.
    if getmtime(settings.CATALOGUE_COUNTERS_FILE) > _COUNTER_TIME:
        for i in xrange(10):
            try:
                with open(settings.CATALOGUE_COUNTERS_FILE) as f:
                    _COUNTERS = cPickle.load(f)
            except (EOFError, ValueError):
                if i < 9:
                    continue
                else:
                    raise
            else:
                break

    tagids = tuple(sorted(t.pk for t in tags))
    try:
        related_ids = _COUNTERS['next'][tagids]
    except KeyError:
        return

{'fraction_non_alphanumeric': 0.07617625093353249, 'fraction_numerical': 0.0026138909634055266, 'mean_word_length': 2.910948905109489, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4193813', 'n_tokens_mistral': 837, 'n_tokens_neox': 805, 'n_words': 283}