<commit_before># -*- coding: utf-8 -*-

from vulyk.app import TASKS_TYPES
from vulyk.models.tasks import Batch, AbstractTask


def batch_completeness():
    """
    Gathers completeness stats on every batch using 2 metrics:
    - actually completed tasks
    - actual reports to total planned amount ratio (tasks * redundancy)

    :rtype : dict
    """
    batches = {}
    tasks = lambda: AbstractTask.objects(batch=b.id)
    f_percent = lambda done, total: (float(done) / (total or done or 1)) * 100
    a_percent = lambda got, limit, total: (float(got) / (limit * total)) * 100

    for b in Batch.objects.all():
        batches[b.id] = {
            'total': 0,
            'flag': 0,
            'flag_percent': 0,
            'answers': 0,
            'answers_percent': 0}

        if len(tasks()) > 0:
            answers = sum(tasks().scalar('users_count'))
            type_id = tasks().first().task_type
            task_type = TASKS_TYPES[type_id]

            batches[b.id] = {
                'total': b.tasks_count,
                'flag': b.tasks_processed,
                'flag_percent': f_percent(b.tasks_processed,
                                          b.tasks_count),
                'answers': answers,
                'answers_percent': a_percent(answers,
                                             task_type.redundancy,
                                             b.tasks_count)
            }

    return batches<commit_msg>fixed calculations logic<commit_after># -*- coding: utf-8 -*-

from vulyk.app import TASKS_TYPES
from vulyk.models.tasks import Batch, AbstractTask


def batch_completeness():
    """
    Gathers completeness stats on every batch using 2 metrics:
    - actually completed tasks
    - actual reports to total planned amount ratio (tasks * redundancy)

    :rtype : dict
    """
    batches = {}
    tasks = lambda batch: AbstractTask.objects(batch=batch.id)
    percent = lambda done, total: (float(done) / (total or done or 1)) * 100

    for b in Batch.objects.all():
        batches[b.id] = {
            'total': 0,
            'flag': 0,
            'flag_percent': 0,
            'answers': 0,
            'answers_percent': 0}

        if len(tasks(b)) > 0:
            type_id = tasks(b).first().task_type
            task_type = TASKS_TYPES[type_id]
            answers = sum(tasks(b).filter(closed=False).scalar('users_count'))
            done = answers + (task_type.redundancy * b.tasks_processed)

            batches[b.id] = {
                'total': b.tasks_count,
                'flag': b.tasks_processed,
                'flag_percent': percent(b.tasks_processed, b.tasks_count),
                'answers': done,
                'answers_percent': percent(done, b.tasks_count)
            }

    return batches
{'fraction_non_alphanumeric': 0.11422413793103449, 'fraction_numerical': 0.009698275862068966, 'mean_word_length': 1.8131313131313131, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3904570', 'n_tokens_mistral': 860, 'n_tokens_neox': 796, 'n_words': 198}