<commit_before># -*- coding: utf-8 -*-
import os
import fnmatch

from django.apps import apps
from django.conf import settings
from django.core.management.base import BaseCommand, CommandError
from django.core.management.color import color_style
from django.template.loader import get_template

from django_extensions.compat import get_template_setting
from django_extensions.management.utils import signalcommand


#
# TODO: Render the template with fake request object ?
#


class Command(BaseCommand):
    args = ''
    help = "Validate templates on syntax and compile errors"
    ignores = set([
        ".DS_Store",
        "*.swp",
        "*~",
    ])

    def add_arguments(self, parser):
        super(Command, self).add_arguments(parser)
        parser.add_argument(
            '--no-apps', action='store_true', dest='no_apps',
            default=False, help="Do not automatically include apps.")
        parser.add_argument(
            '--break', '-b', action='store_true', dest='break',
            default=False, help="Break on first error.")
        parser.add_argument(
            '--include', '-i', action='append', dest='includes',
            default=[], help="Append these paths to TEMPLATE DIRS")
        parser.add_argument(
            '--ignore-app', action='append', dest='ignore_apps',
            default=[], help="Ignore these apps")

    def ignore_filename(self, filename):
        filename = os.path.basename(filename)
        for ignore_pattern in self.ignores:
            if fnmatch.fnmatch(filename, ignore_pattern):
                return True
        return False

    @signalcommand
    def handle(self, *args, **options):
        if hasattr(settings, 'VALIDATE_TEMPLATES_IGNORES'):
            self.ignores = getattr(settings, 'VALIDATE_TEMPLATES_IGNORES')

        style = color_style()
        template_dirs = set(get_template_setting('DIRS'))
        template_dirs |= set(options['includes'])
        template_dirs |= set(getattr(settings, 'VALIDATE_TEMPLATES_EXTRA_TEMPLATE_DIRS', []))

        if not options['no_apps']:
            ignore_apps = options['ignore_apps']
            if not ignore_apps and hasattr(settings, 'VALIDATE_TEMPLATES_IGNORE_APPS'):
                ignore_apps = getattr(settings, 'VALIDATE_TEMPLATES_IGNORE_APPS')
            for app in apps.get_app_configs():
                if app.name in ignore_apps:
                    continue
                app_template_dir = os.path.join(app.path, 'templates')
                if os.path.isdir(app_template_dir):
                    template_dirs.add(app_template_dir)

        settings.TEMPLATES[0]['DIRS'] = list(template_dirs)
        settings.TEMPLATE_DEBUG = True
        verbosity = options["verbosity"]
        errors = 0

        for template_dir in template_dirs:
            for root, dirs, filenames in os.walk(template_dir):
                for filename in filenames:
                    if self.ignore_filename(filename):
                        continue

                    filepath = os.path.join(root, filename)
                    if verbosity > 1:
                        self.stdout.write(filepath)
                    try:
                        get_template(filepath)
                    except Exception as e:
                        errors += 1
                        self.stdout.write("%s: %s" % (filepath, style.ERROR("%s %s" % (e.__class__.__name__, str(e)))))
                    if errors and options['break']:
                        raise CommandError("Errors found")

        if errors:
            raise CommandError("%s errors found" % errors)
        self.stdout.write("%s errors found" % errors)
<commit_msg>Allow empty DIRS in validate_templates<commit_after># -*- coding: utf-8 -*-
import os
import fnmatch

from django.apps import apps
from django.conf import settings
from django.core.management.base import BaseCommand, CommandError
from django.core.management.color import color_style
from django.template.loader import get_template

from django_extensions.compat import get_template_setting
from django_extensions.management.utils import signalcommand


#
# TODO: Render the template with fake request object ?
#


class Command(BaseCommand):
    args = ''
    help = "Validate templates on syntax and compile errors"
    ignores = set([
        ".DS_Store",
        "*.swp",
        "*~",
    ])

    def add_arguments(self, parser):
        super(Command, self).add_arguments(parser)
        parser.add_argument(
            '--no-apps', action='store_true', dest='no_apps',
            default=False, help="Do not automatically include apps.")
        parser.add_argument(
            '--break', '-b', action='store_true', dest='break',
            default=False, help="Break on first error.")
        parser.add_argument(
            '--include', '-i', action='append', dest='includes',
            default=[], help="Append these paths to TEMPLATE DIRS")
        parser.add_argument(
            '--ignore-app', action='append', dest='ignore_apps',
            default=[], help="Ignore these apps")

    def ignore_filename(self, filename):
        filename = os.path.basename(filename)
        for ignore_pattern in self.ignores:
            if fnmatch.fnmatch(filename, ignore_pattern):
                return True
        return False

    @signalcommand
    def handle(self, *args, **options):
        if hasattr(settings, 'VALIDATE_TEMPLATES_IGNORES'):
            self.ignores = getattr(settings, 'VALIDATE_TEMPLATES_IGNORES')

        style = color_style()
        template_dirs = set(get_template_setting('DIRS', []))
        template_dirs |= set(options['includes'])
        template_dirs |= set(getattr(settings, 'VALIDATE_TEMPLATES_EXTRA_TEMPLATE_DIRS', []))

        if not options['no_apps']:
            ignore_apps = options['ignore_apps']
            if not ignore_apps and hasattr(settings, 'VALIDATE_TEMPLATES_IGNORE_APPS'):
                ignore_apps = getattr(settings, 'VALIDATE_TEMPLATES_IGNORE_APPS')
            for app in apps.get_app_configs():
                if app.name in ignore_apps:
                    continue
                app_template_dir = os.path.join(app.path, 'templates')
                if os.path.isdir(app_template_dir):
                    template_dirs.add(app_template_dir)

        settings.TEMPLATES[0]['DIRS'] = list(template_dirs)
        settings.TEMPLATE_DEBUG = True
        verbosity = options["verbosity"]
        errors = 0

        for template_dir in template_dirs:
            for root, dirs, filenames in os.walk(template_dir):
                for filename in filenames:
                    if self.ignore_filename(filename):
                        continue

                    filepath = os.path.join(root, filename)
                    if verbosity > 1:
                        self.stdout.write(filepath)
                    try:
                        get_template(filepath)
                    except Exception as e:
                        errors += 1
                        self.stdout.write("%s: %s" % (filepath, style.ERROR("%s %s" % (e.__class__.__name__, str(e)))))
                    if errors and options['break']:
                        raise CommandError("Errors found")

        if errors:
            raise CommandError("%s errors found" % errors)
        self.stdout.write("%s errors found" % errors)

{'fraction_non_alphanumeric': 0.10428280081577158, 'fraction_numerical': 0.0013596193065941536, 'mean_word_length': 2.1816608996539792, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4553066', 'n_tokens_mistral': 2027, 'n_tokens_neox': 1941, 'n_words': 472}