<commit_before>import logging
from inspect import getmembers
from optparse import make_option
from os.path import join, abspath, exists, dirname, isdir
from os import getcwd, makedirs

from django.core.management.base import BaseCommand, CommandError
from django.conf import settings
from django.utils.importlib import import_module

from devilry.restful import RestfulManager
from devilry.utils.command import setup_logging, get_verbosity
from devilry.apps.extjshelpers.modelintegration import (restfulcls_to_extjsmodel,
                                                        get_extjs_modelname)


def get_installed_apps():
    installed_apps = []
    for app in settings.INSTALLED_APPS:
        if not app.startswith('django.'):
            mod = import_module(app)
            if exists(mod.__file__) and isdir(dirname(mod.__file__)):
                moddir = dirname(mod.__file__)
                installed_apps.append((moddir, mod, mod.__name__.split('.')[-1]))
    return installed_apps

def get_restful_apps():
    apps = []
    for moddir, mod, appname in get_installed_apps():
        try:
            restfulmodule = import_module('{0}.{1}'.format(mod.__name__, 'restful'))
        except ImportError, e:
            pass
        else:
            apps.append((moddir, restfulmodule, appname))
    return apps


class Command(BaseCommand):
    help = 'Autogenerate ExtJS models from restful interfaces.'

    fileheader = '// Autogenerated by the dev_coreextjsmodels script. DO NOT CHANGE MANUALLY'

    def handle(self, *args, **options):
        setup_logging(get_verbosity(options))
        self._create_files_for_all_modules(get_restful_apps())

    def _create_files_for_all_modules(self, restful_apps):
        all_modelnames = []
        for moddir, restfulmodule, appname in restful_apps:
            modelnames = self._create_files_for_module(moddir, restfulmodule, appname)
            all_modelnames += modelnames
        return all_modelnames

    def _create_files_for_module(self, moddir, restfulmodule, appname):
        modelnames = []
        dirname = self._create_extjsclass_dir(moddir, appname)
        logging.info('Parsing app: %s', appname)
        self._get_restfulmanagers(restfulmodule)
        for restfulmanager in self._get_restfulmanagers(restfulmodule):
            for restfulcls in restfulmanager.iter_restfulclasses():
                logging.debug('Generating JS code for: %s', restfulcls.__name__)
                js = self._get_js_for_model(restfulcls)
                modelname = get_extjs_modelname(restfulcls)
                self._create_extjsclassfile(dirname, modelname, js)
                modelnames.append(modelname)
        return modelnames

    def _get_restfulmanagers(self, restfulmodule):
        def is_restfulmananager_obj(obj):
            return isinstance(obj, RestfulManager)
        return [manager for name, manager in getmembers(restfulmodule, is_restfulmananager_obj)]

    def _create_requireall_file(self, dirname, modelnames):
        requires = ',\n'.join(["    '{0}'".format(modelname) for modelname in modelnames])
        content = '{0}\nExt.require([\n{1}\n]);'.format(self.fileheader, requires)
        open(join(dirname, 'require_all.js'), 'w').write(content)

    def _get_js_for_model(self, restfulcls):
        result_fieldgroups =  restfulcls._meta.simplified._meta.resultfields.additional_aslist()
        js = restfulcls_to_extjsmodel(restfulcls, result_fieldgroups, pretty=True)
        return js + ';'

    def _create_extjsclass_dir(self, moddir, appname):
        dirname = join(moddir, 'static', 'extjs_classes', 'apps', appname, 'simplified')
        if not exists(dirname):
            logging.info('Creating directory: %s', dirname)
            makedirs(dirname)
        return dirname

    def _create_extjsclassfile(self, dirname, modelname, js):
        clsname = modelname.split('.')[-1]
        data = '{0}\n{1}'.format(self.fileheader, js)
        path = join(dirname, clsname + '.js')
        logging.info('Creating: %s', path)
        logging.debug('%s: %s', path, data)
        open(path, 'w').write(data)
<commit_msg>apps.developertools...dev_autogen_extjsmodels: Create require templatate.<commit_after>import logging
from inspect import getmembers
from os.path import join, exists, dirname, isdir
from os import makedirs

from django.core.management.base import BaseCommand
from django.conf import settings
from django.utils.importlib import import_module
from devilry.apps import extjshelpers

from devilry.restful import RestfulManager
from devilry.utils.command import setup_logging, get_verbosity
from devilry.apps.extjshelpers.modelintegration import (restfulcls_to_extjsmodel,
                                                        get_extjs_modelname)


def get_installed_apps():
    installed_apps = []
    for app in settings.INSTALLED_APPS:
        if not app.startswith('django.'):
            mod = import_module(app)
            if exists(mod.__file__) and isdir(dirname(mod.__file__)):
                moddir = dirname(mod.__file__)
                installed_apps.append((moddir, mod, mod.__name__.split('.')[-1]))
    return installed_apps

def get_restful_apps():
    apps = []
    for moddir, mod, appname in get_installed_apps():
        try:
            restfulmodule = import_module('{0}.{1}'.format(mod.__name__, 'restful'))
        except ImportError, e:
            pass
        else:
            apps.append((moddir, restfulmodule, appname))
    return apps


class Command(BaseCommand):
    help = 'Autogenerate ExtJS models from restful interfaces.'

    fileheader = '// Autogenerated by the dev_coreextjsmodels script. DO NOT CHANGE MANUALLY'

    def handle(self, *args, **options):
        setup_logging(get_verbosity(options))
        all_modelnames = self._create_files_for_all_modules(get_restful_apps())
        self._create_requireall_file(all_modelnames)

    def _create_files_for_all_modules(self, restful_apps):
        all_modelnames = []
        for moddir, restfulmodule, appname in restful_apps:
            modelnames = self._create_files_for_module(moddir, restfulmodule, appname)
            all_modelnames += modelnames
        return all_modelnames

    def _create_files_for_module(self, moddir, restfulmodule, appname):
        modelnames = []
        directory = self._create_extjsclass_dir(moddir, appname)
        logging.info('Parsing app: %s', appname)
        self._get_restfulmanagers(restfulmodule)
        for restfulmanager in self._get_restfulmanagers(restfulmodule):
            for restfulcls in restfulmanager.iter_restfulclasses():
                logging.debug('Generating JS code for: %s', restfulcls.__name__)
                js = self._get_js_for_model(restfulcls)
                modelname = get_extjs_modelname(restfulcls)
                self._create_extjsclassfile(directory, modelname, js)
                modelnames.append(modelname)
        return modelnames

    def _get_restfulmanagers(self, restfulmodule):
        def is_restfulmananager_obj(obj):
            return isinstance(obj, RestfulManager)
        return [manager for name, manager in getmembers(restfulmodule, is_restfulmananager_obj)]

    def _create_requireall_file(self, modelnames):
        extjshelpersdir = dirname(extjshelpers.__file__)
        requires = ',\n'.join(["    '{0}'".format(modelname) for modelname in modelnames])
        content = ('{{% comment %}}\n'
                   '{0}\n'
                   '// This is a Django template because we want to make it easy to dump it into the page template.\n'
                   '{{% endcomment %}}'
                   '\nExt.require([\n{1}\n]);').format(self.fileheader, requires)
        path = join(extjshelpersdir, 'templates', 'extjshelpers', 'restful-generated-models.django.js')
        logging.info('Creating: %s', path)
        logging.debug('%s: %s', path, content)
        open(path, 'w').write(content)

    def _get_js_for_model(self, restfulcls):
        result_fieldgroups =  restfulcls._meta.simplified._meta.resultfields.additional_aslist()
        js = restfulcls_to_extjsmodel(restfulcls, result_fieldgroups, pretty=True)
        return js + ';'

    def _create_extjsclass_dir(self, moddir, appname):
        directory = join(moddir, 'static', 'extjs_classes', 'apps', appname, 'simplified')
        if not exists(directory):
            logging.info('Creating directory: %s', directory)
            makedirs(directory)
        return directory

    def _create_extjsclassfile(self, directory, modelname, js):
        clsname = modelname.split('.')[-1]
        data = '{0}\n{1}'.format(self.fileheader, js)
        path = join(directory, clsname + '.js')
        logging.info('Creating: %s', path)
        logging.debug('%s: %s', path, data)
        open(path, 'w').write(data)

{'fraction_non_alphanumeric': 0.10112359550561797, 'fraction_numerical': 0.0020429009193054137, 'mean_word_length': 3.02007299270073, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1892707', 'n_tokens_mistral': 2622, 'n_tokens_neox': 2563, 'n_words': 567}