<commit_before>
import commonware.log
import json
import requests

from django.conf import settings
from django.contrib.auth.models import Permission
from django.http import HttpResponse
from translate.filters import checks


log = commonware.log.getLogger('pontoon')


def add_can_localize(user):

    email = user.email
    log.debug(email)

    # Grant permission to Mozilla localizers
    url = "https://mozillians.org/api/v1/users/"
    payload = {
        "app_name": "pontoon",
        "app_key": settings.MOZILLIANS_API_KEY,
        "email": email,
        "is_vouched": True,
        "groups": "localization",
    }

    try:
        response = requests.get(url, params=payload)
        mozillians = response.json()["objects"]

        if len(mozillians) > 0:
            can_localize = Permission.objects.get(codename="can_localize")
            user.user_permissions.add(can_localize)
            log.debug("Permission can_localize set.")

            # Fallback if profile does not allow accessing data
            user.first_name = mozillians[0].get("full_name", email)
            user.save()

    except Exception as e:
        log.debug(e)
        log.debug("Is your MOZILLIANS_API_KEY set?")
        user.save()


def quality_check(original, string, ignore):
    """Check for obvious errors like blanks and missing interpunction."""

    if not ignore:
        warnings = checks.runtests(original, string)
        if warnings:

            # https://github.com/translate/pootle/
            check_names = {
                'accelerators': 'Accelerators',
                'acronyms': 'Acronyms',
                'blank': 'Blank',
                'brackets': 'Brackets',
                'compendiumconflicts': 'Compendium conflict',
                'credits': 'Translator credits',
                'doublequoting': 'Double quotes',
                'doublespacing': 'Double spaces',
                'doublewords': 'Repeated word',
                'emails': 'E-mail',
                'endpunc': 'Ending punctuation',
                'endwhitespace': 'Ending whitespace',
                'escapes': 'Escapes',<commit_msg>Replace check.runtest with own function to avoid UnicodeEncodeErrors<commit_after>
import commonware.log
import json
import requests

from django.conf import settings
from django.contrib.auth.models import Permission
from django.http import HttpResponse
from translate.filters import checks
from translate.storage import base as storage_base
from translate.lang import data as lang_data


log = commonware.log.getLogger('pontoon')


def add_can_localize(user):

    email = user.email
    log.debug(email)

    # Grant permission to Mozilla localizers
    url = "https://mozillians.org/api/v1/users/"
    payload = {
        "app_name": "pontoon",
        "app_key": settings.MOZILLIANS_API_KEY,
        "email": email,
        "is_vouched": True,
        "groups": "localization",
    }

    try:
        response = requests.get(url, params=payload)
        mozillians = response.json()["objects"]

        if len(mozillians) > 0:
            can_localize = Permission.objects.get(codename="can_localize")
            user.user_permissions.add(can_localize)
            log.debug("Permission can_localize set.")

            # Fallback if profile does not allow accessing data
            user.first_name = mozillians[0].get("full_name", email)
            user.save()

    except Exception as e:
        log.debug(e)
        log.debug("Is your MOZILLIANS_API_KEY set?")
        user.save()


def quality_check(original, string, ignore):
    """Check for obvious errors like blanks and missing interpunction."""

    if not ignore:
        original = lang_data.normalized_unicode(original)
        string = lang_data.normalized_unicode(string)

        unit = storage_base.TranslationUnit(original)
        unit.target = string
        checker = checks.StandardChecker()

        warnings = checker.run_filters(unit)
        if warnings:

            # https://github.com/translate/pootle/
            check_names = {
                'accelerators': 'Accelerators',
                'acronyms': 'Acronyms',
                'blank': 'Blank',
                'brackets': 'Brackets',
                'compendiumconflicts': 'Compendium conflict',
                'credits': 'Translator credits',
                'doublequoting': 'Double quotes',
                'doublespacing': 'Double spaces',
                'doublewords': 'Repeated word',
                'emails': 'E-mail',
                'endpunc': 'Ending punctuation',
                'endwhitespace': 'Ending whitespace',
                'escapes': 'Escapes',
{'fraction_non_alphanumeric': 0.10625941062594106, 'fraction_numerical': 0.001290600129060013, 'mean_word_length': 2.3025568181818183, 'pattern_counts': {'":': 10, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6679766', 'n_tokens_mistral': 1370, 'n_tokens_neox': 1287, 'n_words': 309}