<commit_before># Copyright (c) Ecometrica. All rights reserved.
# Distributed under the BSD license. See LICENSE for details.

import codecs
from optparse import make_option
import os
import re

import vinaigrette

from django.core.management.base import BaseCommand, CommandError
from django.core.management.commands import makemessages as django_makemessages
from django.utils.translation import ugettext

def _get_po_paths(locale=None):
    """Returns paths to all relevant po files in the current project."""
    basedirs = [os.path.join('conf', 'locale'), 'locale']
    if os.environ.get('DJANGO_SETTINGS_MODULE'):
        from django.conf import settings
        basedirs.extend(settings.LOCALE_PATHS)

    # Gather existing directories.
    basedirs = set(map(os.path.abspath, filter(os.path.isdir, basedirs)))

    if not basedirs:
        raise CommandError("This script should be run from the Django SVN tree or your project or app tree, or with the settings module specified.")

    po_paths = []
    for basedir in basedirs:
        if locale:
            basedir = os.path.join(basedir, locale, 'LC_MESSAGES')
        for dirpath, dirnames, filenames in os.walk(basedir):
            for f in filenames:
                if f.endswith('.po'):
                    po_paths.append(os.path.join(dirpath, f))
    return po_paths

class Command(django_makemessages.Command):
    
    option_list = django_makemessages.Command.option_list + (
        make_option('--no-vinaigrette', default=True, action='store_false', dest='avec-vinaigrette',
            help="Don't include strings from database fields handled by vinaigrette."),
        make_option('--keep-obsolete', default=False, action='store_true', dest='keep-obsolete',
            help="Don't obsolete strings no longer referenced in code or Viniagrette's fields.")
    )
    
    help = "Runs over the entire source tree of the current directory and pulls out all strings marked for translation. It creates (or updates) a message file in the conf/locale (in the django tree) or locale (for project and application) directory. Also includes strings from database fields handled by vinaigrette."
    requires_model_validation = True
    
    def handle(self, *args, **options):
        if not options.get('avec-vinaigrette'):
            return super(Command, self).handle(*args, **options)
        
        verbosity = int(options.get('verbosity'))
        vinfilepath = 'vinaigrette-deleteme.py'
        sources = ['', '']
        
        # Because Django makemessages isn't very extensible, we're writing a
        # fake Python file, calling makemessages, then deleting it after.
        vinfile = codecs.open(vinfilepath, 'w', encoding='utf8')
        try:
            vinfile.write('#coding:utf-8\n')
            if verbosity > 0:
                print 'Vinaigrette is processing database values...'
            
            for model in sorted(vinaigrette._registry.keys(),
              key=lambda m: m._meta.object_name):
                strings_seen = set()
                modelname = model._meta.object_name
                reg = vinaigrette._registry[model]
                fields = reg['fields']
                manager = reg['manager'] if reg['manager'] else model._default_manager
                qs = manager.filter(reg['restrict_to']) if reg['restrict_to'] \
                    else manager.all()
            
                for instance in qs.order_by('pk').values('pk', *fields):
                    # In the reference comment in the po file, use the object's primary
                    # key as the line number, but only if it's an integer primary key
                    idnum = instance.pop('pk')
                    idnum = idnum if isinstance(idnum, (int, long)) or idnum.isdigit() else 0
                    for (fieldname, val) in instance.items():
                        if val and val not in strings_seen:
                            strings_seen.add(val)
                            sources.append('%s/%s:%s' % (modelname, fieldname, idnum))
                            vinfile.write(
                                'gettext(%r)\n' 
                                % val.replace('\r', '').replace('%', '%%')
                            )
        finally:
            vinfile.close()
        
        try:
            super(Command, self).handle(*args, **options)
        finally:
            os.unlink(vinfilepath)
        
        r_lineref = re.compile(r'%s:(\d+)' % re.escape(vinfilepath))
        def lineref_replace(match):
            try:
                return sources[int(match.group(1))]
            except IndexError, ValueError:
                return match.group(0)
        
        # The PO file has been generated. Now, swap out the line-number
        # references to our fake python file for more descriptive
        # references.
        if options.get('all'):
            po_paths = _get_po_paths()
        else:
            po_paths = _get_po_paths(options.get('locale'))<commit_msg>In django 1.6, makemessages can take the -l option multiple times.  Making Vinaigrette compatible with 1.6 and still compatible with previous versions.<commit_after># Copyright (c) Ecometrica. All rights reserved.
# Distributed under the BSD license. See LICENSE for details.

import codecs
from optparse import make_option
import os
import re

import vinaigrette

from django.core.management.base import BaseCommand, CommandError
from django.core.management.commands import makemessages as django_makemessages
from django.utils.translation import ugettext

def _get_po_paths(locales=None):
    """Returns paths to all relevant po files in the current project."""
    basedirs = [os.path.join('conf', 'locale'), 'locale']
    if os.environ.get('DJANGO_SETTINGS_MODULE'):
        from django.conf import settings
        basedirs.extend(settings.LOCALE_PATHS)

    # Gather existing directories.
    basedirs = set(map(os.path.abspath, filter(os.path.isdir, basedirs)))

    if not basedirs:
        raise CommandError("This script should be run from the Django SVN tree or your project or app tree, or with the settings module specified.")

    po_paths = []
    for basedir in basedirs:
        for locale in locales:
            basedir_locale = os.path.join(basedir, locale, 'LC_MESSAGES')
            for dirpath, dirnames, filenames in os.walk(basedir_locale):
                for f in filenames:
                    if f.endswith('.po'):
                        po_paths.append(os.path.join(dirpath, f))
    return po_paths

class Command(django_makemessages.Command):
    
    option_list = django_makemessages.Command.option_list + (
        make_option('--no-vinaigrette', default=True, action='store_false', dest='avec-vinaigrette',
            help="Don't include strings from database fields handled by vinaigrette."),
        make_option('--keep-obsolete', default=False, action='store_true', dest='keep-obsolete',
            help="Don't obsolete strings no longer referenced in code or Viniagrette's fields.")
    )
    
    help = "Runs over the entire source tree of the current directory and pulls out all strings marked for translation. It creates (or updates) a message file in the conf/locale (in the django tree) or locale (for project and application) directory. Also includes strings from database fields handled by vinaigrette."
    requires_model_validation = True
    
    def handle(self, *args, **options):
        if not options.get('avec-vinaigrette'):
            return super(Command, self).handle(*args, **options)
        
        verbosity = int(options.get('verbosity'))
        vinfilepath = 'vinaigrette-deleteme.py'
        sources = ['', '']
        
        # Because Django makemessages isn't very extensible, we're writing a
        # fake Python file, calling makemessages, then deleting it after.
        vinfile = codecs.open(vinfilepath, 'w', encoding='utf8')
        try:
            vinfile.write('#coding:utf-8\n')
            if verbosity > 0:
                print 'Vinaigrette is processing database values...'
            
            for model in sorted(vinaigrette._registry.keys(),
              key=lambda m: m._meta.object_name):
                strings_seen = set()
                modelname = model._meta.object_name
                reg = vinaigrette._registry[model]
                fields = reg['fields']
                manager = reg['manager'] if reg['manager'] else model._default_manager
                qs = manager.filter(reg['restrict_to']) if reg['restrict_to'] \
                    else manager.all()
            
                for instance in qs.order_by('pk').values('pk', *fields):
                    # In the reference comment in the po file, use the object's primary
                    # key as the line number, but only if it's an integer primary key
                    idnum = instance.pop('pk')
                    idnum = idnum if isinstance(idnum, (int, long)) or idnum.isdigit() else 0
                    for (fieldname, val) in instance.items():
                        if val and val not in strings_seen:
                            strings_seen.add(val)
                            sources.append('%s/%s:%s' % (modelname, fieldname, idnum))
                            vinfile.write(
                                'gettext(%r)\n' 
                                % val.replace('\r', '').replace('%', '%%')
                            )
        finally:
            vinfile.close()
        
        try:
            super(Command, self).handle(*args, **options)
        finally:
            os.unlink(vinfilepath)
        
        r_lineref = re.compile(r'%s:(\d+)' % re.escape(vinfilepath))
        def lineref_replace(match):
            try:
                return sources[int(match.group(1))]
            except IndexError, ValueError:
                return match.group(0)
        
        # The PO file has been generated. Now, swap out the line-number
        # references to our fake python file for more descriptive
        # references.
        if options.get('all'):
            po_paths = _get_po_paths()
        else:
            locales = options.get('locale')

            # In django 1.6+ one or more locales can be specified, so we
            # make sure to handle both versions here.
            if isinstance(locales, basestring):
                locales = [locales]

            po_paths = _get_po_paths(locales)
{'fraction_non_alphanumeric': 0.09624233128834356, 'fraction_numerical': 0.0017254601226993865, 'mean_word_length': 2.1124701670644392, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3130821', 'n_tokens_mistral': 2878, 'n_tokens_neox': 2762, 'n_words': 844}