<commit_before># -*- coding: utf-8 -*-
from __future__ import with_statement

import os
import re

from django.conf import settings
from django.core.management.base import BaseCommand

from django_extensions.compat import get_template_setting
from django_extensions.management.utils import signalcommand

ANNOTATION_RE = re.compile("\{?#[\s]*?(TODO|FIXME|BUG|HACK|WARNING|NOTE|XXX)[\s:]?(.+)")
ANNOTATION_END_RE = re.compile("(.*)#\}(.*)")


class Command(BaseCommand):
    help = 'Show all annotations like TODO, FIXME, BUG, HACK, WARNING, NOTE or XXX in your py and HTML files.'
    args = 'tag'
    label = 'annotation tag (TODO, FIXME, BUG, HACK, WARNING, NOTE, XXX)'

    @signalcommand
    def handle(self, *args, **options):
        # don't add django internal code
        apps = [app for app in filter(lambda app: not app.startswith('django.contrib'), settings.INSTALLED_APPS)]
        template_dirs = get_template_setting('DIRS', [])
        base_dir = getattr(settings, 'BASE_DIR')
        if template_dirs:
            apps += template_dirs
        for app_dir in apps:
            app_dir = app_dir.replace(".", "/")
            if base_dir:
                app_dir = os.path.join(base_dir, app_dir)
            for top, dirs, files in os.walk(app_dir):
                for fn in files:
                    if os.path.splitext(fn)[1] in ('.py', '.html'):
                        fpath = os.path.join(top, fn)
                        annotation_lines = []
                        with open(fpath, 'r') as fd:
                            i = 0
                            for line in fd.readlines():
                                i += 1
                                if ANNOTATION_RE.search(line):
                                    tag, msg = ANNOTATION_RE.findall(line)[0]
                                    if len(args) == 1:
                                        search_for_tag = args[0].upper()
                                        if not search_for_tag == tag:
                                            break

                                    if ANNOTATION_END_RE.search(msg.strip()):
                                        msg = ANNOTATION_END_RE.findall(msg.strip())[0][0]

                                    annotation_lines.append("[%3s] %-5s %s" % (i, tag, msg.strip()))
                            if annotation_lines:
                                self.stdout.write("%s:" % fpath)
                                for annotation in annotation_lines:
                                    self.stdout.write(u"  * %s" % annotation.decode('utf-8'))
                                self.stdout.write("")
<commit_msg>Fix for issue #1210 ("notes" command raises exception on utf-8 content)<commit_after># -*- coding: utf-8 -*-
from __future__ import with_statement

import os
import re
import six

from django.conf import settings
from django.core.management.base import BaseCommand

from django_extensions.compat import get_template_setting
from django_extensions.management.utils import signalcommand

ANNOTATION_RE = re.compile("\{?#[\s]*?(TODO|FIXME|BUG|HACK|WARNING|NOTE|XXX)[\s:]?(.+)")
ANNOTATION_END_RE = re.compile("(.*)#\}(.*)")


class Command(BaseCommand):
    help = 'Show all annotations like TODO, FIXME, BUG, HACK, WARNING, NOTE or XXX in your py and HTML files.'
    args = 'tag'
    label = 'annotation tag (TODO, FIXME, BUG, HACK, WARNING, NOTE, XXX)'

    @signalcommand
    def handle(self, *args, **options):
        # don't add django internal code
        apps = [app for app in filter(lambda app: not app.startswith('django.contrib'), settings.INSTALLED_APPS)]
        template_dirs = get_template_setting('DIRS', [])
        base_dir = getattr(settings, 'BASE_DIR')
        if template_dirs:
            apps += template_dirs
        for app_dir in apps:
            app_dir = app_dir.replace(".", "/")
            if base_dir:
                app_dir = os.path.join(base_dir, app_dir)
            for top, dirs, files in os.walk(app_dir):
                for fn in files:
                    if os.path.splitext(fn)[1] in ('.py', '.html'):
                        fpath = os.path.join(top, fn)
                        annotation_lines = []
                        with open(fpath, 'r') as fd:
                            i = 0
                            for line in fd.readlines():
                                i += 1
                                if ANNOTATION_RE.search(line):
                                    tag, msg = ANNOTATION_RE.findall(line)[0]
                                    if len(args) == 1:
                                        search_for_tag = args[0].upper()
                                        if not search_for_tag == tag:
                                            break

                                    if ANNOTATION_END_RE.search(msg.strip()):
                                        msg = ANNOTATION_END_RE.findall(msg.strip())[0][0]

                                    annotation_lines.append("[%3s] %-5s %s" % (i, tag, msg.strip()))
                            if annotation_lines:
                                self.stdout.write("%s:" % fpath)
                                for annotation in annotation_lines:
                                    self.stdout.write("  * %s" % annotation.decode('utf-8') if six.PY2 else annotation)
                                self.stdout.write("")

{'fraction_non_alphanumeric': 0.11016478429920386, 'fraction_numerical': 0.005554526939455656, 'mean_word_length': 1.4410302756439224, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 32, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3824665', 'n_tokens_mistral': 1494, 'n_tokens_neox': 1423, 'n_words': 362}