<commit_before>from __future__ import unicode_literals

import codecs
import glob
import os

from django.core.management.base import BaseCommand, CommandError
from django.core.management.utils import find_command, popen_wrapper
from django.utils._os import npath, upath


def has_bom(fn):
    with open(fn, 'rb') as f:
        sample = f.read(4)
    return (sample[:3] == b'\xef\xbb\xbf' or
        sample.startswith((codecs.BOM_UTF16_LE, codecs.BOM_UTF16_BE)))


def is_writable(path):
    # Known side effect: updating file access/modified time to current time if
    # it is writable.
    try:
        with open(path, 'a'):
            os.utime(path, None)
    except (IOError, OSError):
        return False
    return True


class Command(BaseCommand):
    help = 'Compiles .po files to .mo files for use with builtin gettext support.'

    requires_system_checks = False
    leave_locale_alone = True

    program = 'msgfmt'
    program_options = ['--check-format']

    def add_arguments(self, parser):
        parser.add_argument('--locale', '-l', dest='locale', action='append', default=[],
            help='Locale(s) to process (e.g. de_AT). Default is to process all. '<commit_msg>Removed hardcoded utf-8 BOM, used value from codecs instead.<commit_after>from __future__ import unicode_literals

import codecs
import glob
import os

from django.core.management.base import BaseCommand, CommandError
from django.core.management.utils import find_command, popen_wrapper
from django.utils._os import npath, upath


def has_bom(fn):
    with open(fn, 'rb') as f:
        sample = f.read(4)
    return sample.startswith((codecs.BOM_UTF8, codecs.BOM_UTF16_LE, codecs.BOM_UTF16_BE))


def is_writable(path):
    # Known side effect: updating file access/modified time to current time if
    # it is writable.
    try:
        with open(path, 'a'):
            os.utime(path, None)
    except (IOError, OSError):
        return False
    return True


class Command(BaseCommand):
    help = 'Compiles .po files to .mo files for use with builtin gettext support.'

    requires_system_checks = False
    leave_locale_alone = True

    program = 'msgfmt'
    program_options = ['--check-format']

    def add_arguments(self, parser):
        parser.add_argument('--locale', '-l', dest='locale', action='append', default=[],
            help='Locale(s) to process (e.g. de_AT). Default is to process all. '
{'fraction_non_alphanumeric': 0.10511009555463233, 'fraction_numerical': 0.00540091400083091, 'mean_word_length': 3.3465703971119134, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3864897', 'n_tokens_mistral': 767, 'n_tokens_neox': 741, 'n_words': 226}