<commit_before>from django.conf import settings
from django.core import formfields, validators
from django.core import db
from django.core.exceptions import ObjectDoesNotExist
from django.core.meta.fields import *
from django.utils.functional import curry
from django.utils.text import capfirst
import copy, datetime, os, re, sys, types

# Admin stages.
ADD, CHANGE, BOTH = 1, 2, 3

# Size of each "chunk" for get_iterator calls.
# Larger values are slightly faster at the expense of more storage space.
GET_ITERATOR_CHUNK_SIZE = 100

# Prefix (in Python path style) to location of models.
MODEL_PREFIX = 'django.models'

# Methods on models with the following prefix will be removed and
# converted to module-level functions.
MODEL_FUNCTIONS_PREFIX = '_module_'

# Methods on models with the following prefix will be removed and
# converted to manipulator methods.
MANIPULATOR_FUNCTIONS_PREFIX = '_manipulator_'

LOOKUP_SEPARATOR = '__'

####################
# HELPER FUNCTIONS #
####################

# Django currently supports two forms of ordering.
# Form 1 (deprecated) example:
#     order_by=(('pub_date', 'DESC'), ('headline', 'ASC'), (None, 'RANDOM'))
# Form 2 (new-style) example:
#     order_by=('-pub_date', 'headline', '?')
# Form 1 is deprecated and will no longer be supported for Django's first
# official release. The following code converts from Form 1 to Form 2.

LEGACY_ORDERING_MAPPING = {'ASC': '_', 'DESC': '-_', 'RANDOM': '?'}

def handle_legacy_orderlist(order_list):
    if not order_list or isinstance(order_list[0], basestring):
        return order_list
    else:
        import warnings
        new_order_list = [LEGACY_ORDERING_MAPPING[j.upper()].replace('_', str(i)) for i, j in order_list]
        warnings.warn("%r ordering syntax is deprecated. Use %r instead." % (order_list, new_order_list), DeprecationWarning)
        return new_order_list

def orderlist2sql(order_list, prefix=''):
    output = []
    for f in handle_legacy_orderlist(order_list):
        if f.startswith('-'):
            output.append('%s%s DESC' % (prefix, f[1:]))
        elif f == '?':
            output.append('RANDOM()')
        else:
            output.append('%s%s ASC' % (prefix, f))
    return ', '.join(output)

def get_module(app_label, module_name):
    return __import__('%s.%s.%s' % (MODEL_PREFIX, app_label, module_name), '', '', [''])

def get_app(app_label):
    return __import__('%s.%s' % (MODEL_PREFIX, app_label), '', '', [''])

_installed_models_cache = None
def get_installed_models():
    """
    Returns a list of installed "models" packages, such as foo.models,
    ellington.news.models, etc. This does NOT include django.models.
    """
    global _installed_models_cache
    if _installed_models_cache is not None:
        return _installed_models_cache
    _installed_models_cache = []
    for a in settings.INSTALLED_APPS:
        try:
            _installed_models_cache.append(__import__(a + '.models', '', '', ['']))
        except ImportError:
            pass
    return _installed_models_cache

_installed_modules_cache = None
def get_installed_model_modules(core_models=None):
    """
    Returns a list of installed models, such as django.models.core,
    ellington.news.models.news, foo.models.bar, etc.
    """
    global _installed_modules_cache
    if _installed_modules_cache is not None:
        return _installed_modules_cache
    _installed_modules_cache = []

    # django.models is a special case.
    for submodule in (core_models or []):
        _installed_modules_cache.append(__import__('django.models.%s' % submodule, '', '', ['']))
    for m in get_installed_models():
        for submodule in getattr(m, '__all__', []):
            _installed_modules_cache.append(__import__('django.models.%s' % submodule, '', '', ['']))
    return _installed_modules_cache

class LazyDate:
    """
    Use in limit_choices_to to compare the field to dates calculated at run time
    instead of when the model is loaded.  For example::

        ... limit_choices_to = {'date__gt' : meta.LazyDate(days=-3)} ...

    which will limit the choices to dates greater than three days ago.
    """
    def __init__(self, **kwargs):
        self.delta = datetime.timedelta(**kwargs)

    def __str__(self):
        return str(self.__get_value__())<commit_msg>Fixed #257 -- Empty model modules no longer cause an error. Thanks, <NAME><commit_after>from django.conf import settings
from django.core import formfields, validators
from django.core import db
from django.core.exceptions import ObjectDoesNotExist
from django.core.meta.fields import *
from django.utils.functional import curry
from django.utils.text import capfirst
import copy, datetime, os, re, sys, types

# Admin stages.
ADD, CHANGE, BOTH = 1, 2, 3

# Size of each "chunk" for get_iterator calls.
# Larger values are slightly faster at the expense of more storage space.
GET_ITERATOR_CHUNK_SIZE = 100

# Prefix (in Python path style) to location of models.
MODEL_PREFIX = 'django.models'

# Methods on models with the following prefix will be removed and
# converted to module-level functions.
MODEL_FUNCTIONS_PREFIX = '_module_'

# Methods on models with the following prefix will be removed and
# converted to manipulator methods.
MANIPULATOR_FUNCTIONS_PREFIX = '_manipulator_'

LOOKUP_SEPARATOR = '__'

####################
# HELPER FUNCTIONS #
####################

# Django currently supports two forms of ordering.
# Form 1 (deprecated) example:
#     order_by=(('pub_date', 'DESC'), ('headline', 'ASC'), (None, 'RANDOM'))
# Form 2 (new-style) example:
#     order_by=('-pub_date', 'headline', '?')
# Form 1 is deprecated and will no longer be supported for Django's first
# official release. The following code converts from Form 1 to Form 2.

LEGACY_ORDERING_MAPPING = {'ASC': '_', 'DESC': '-_', 'RANDOM': '?'}

def handle_legacy_orderlist(order_list):
    if not order_list or isinstance(order_list[0], basestring):
        return order_list
    else:
        import warnings
        new_order_list = [LEGACY_ORDERING_MAPPING[j.upper()].replace('_', str(i)) for i, j in order_list]
        warnings.warn("%r ordering syntax is deprecated. Use %r instead." % (order_list, new_order_list), DeprecationWarning)
        return new_order_list

def orderlist2sql(order_list, prefix=''):
    output = []
    for f in handle_legacy_orderlist(order_list):
        if f.startswith('-'):
            output.append('%s%s DESC' % (prefix, f[1:]))
        elif f == '?':
            output.append('RANDOM()')
        else:
            output.append('%s%s ASC' % (prefix, f))
    return ', '.join(output)

def get_module(app_label, module_name):
    return __import__('%s.%s.%s' % (MODEL_PREFIX, app_label, module_name), '', '', [''])

def get_app(app_label):
    return __import__('%s.%s' % (MODEL_PREFIX, app_label), '', '', [''])

_installed_models_cache = None
def get_installed_models():
    """
    Returns a list of installed "models" packages, such as foo.models,
    ellington.news.models, etc. This does NOT include django.models.
    """
    global _installed_models_cache
    if _installed_models_cache is not None:
        return _installed_models_cache
    _installed_models_cache = []
    for a in settings.INSTALLED_APPS:
        try:
            _installed_models_cache.append(__import__(a + '.models', '', '', ['']))
        except ImportError:
            pass
    return _installed_models_cache

_installed_modules_cache = None
def get_installed_model_modules(core_models=None):
    """
    Returns a list of installed models, such as django.models.core,
    ellington.news.models.news, foo.models.bar, etc.
    """
    global _installed_modules_cache
    if _installed_modules_cache is not None:
        return _installed_modules_cache
    _installed_modules_cache = []

    # django.models is a special case.
    for submodule in (core_models or []):
        _installed_modules_cache.append(__import__('django.models.%s' % submodule, '', '', ['']))
    for m in get_installed_models():
        for submodule in getattr(m, '__all__', []):
            mod = __import__('django.models.%s' % submodule, '', '', [''])
            try:
                mod._MODELS
            except AttributeError:
                pass # Skip model modules that don't actually have models in them.
            else:
                _installed_modules_cache.append(mod)
    return _installed_modules_cache

class LazyDate:
    """
    Use in limit_choices_to to compare the field to dates calculated at run time
    instead of when the model is loaded.  For example::

        ... limit_choices_to = {'date__gt' : meta.LazyDate(days=-3)} ...

    which will limit the choices to dates greater than three days ago.
    """
    def __init__(self, **kwargs):
        self.delta = datetime.timedelta(**kwargs)

    def __str__(self):
        return str(self.__get_value__())
{'fraction_non_alphanumeric': 0.12074408117249154, 'fraction_numerical': 0.003720405862457723, 'mean_word_length': 3.572680412371134, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6584898', 'n_tokens_mistral': 2747, 'n_tokens_neox': 2632, 'n_words': 784}