<commit_before>import re
import random
import django
from django.db import DatabaseError, models, transaction
from django.apps import apps
from django.core.exceptions import ObjectDoesNotExist, ValidationError, AppRegistryNotReady
from django.db.models import fields
from django.core.files import File

from cinp.server_common import Converter, Namespace, Model, Action, Paramater, Field, ServerError, checkAuth_true, checkAuth_false

__MODEL_REGISTRY__ = {}

# TODO: take advantage of .save( update_fields=.... ) on UPDATE

NEW_REMOTE_FIELD = int( django.get_version().split( '.' )[1] ) > 8  # last known (to me) use of related was 1.8
HAS_VIEW_PERMISSION = int( django.get_version().split( '.' )[0] ) >= 2 and int( django.get_version().split( '.' )[1] ) >= 1


def field_model_resolver( django_field ):
  mode = None
  is_array = None

  if NEW_REMOTE_FIELD:
    remote_field = django_field.remote_field
  else:
    remote_field = django_field.related

  if django_field.get_internal_type() == 'ManyToManyField':
    # detect if "through" is autogenerated, if not, make the field RO
    # we can't reach into the through model from here, they will have to
    # expose the through model to work with the relationship

    auto_created = None
    try:
      auto_created = remote_field.through._meta.auto_created
    except AttributeError:
      raise ValueError( 'Unknown target model "{0}" make sure it is registered, field: "{1}" model: "{2}"'.format( remote_field.through, django_field.name, django_field.model.__qualname__ ) )

    if auto_created is False:
      mode = 'RO'
      django_model = remote_field.through

    elif auto_created is None:
      mode = 'RO'
      django_model = remote_field.through

    else:
      mode = 'RW'
      django_model = remote_field.model

    is_array = True

  else:
    django_model = remote_field.model

  if isinstance( django_model, str ):  # this catches the case where there is a circular app refrence, so django has yet to fully map one of the sides when this is run
    try:
      ( app_name, model_name ) = django_model.split( '.' )
    except ValueError:
      raise ValueError( 'Remote Field model name "{0}" in unexpected format, field: "{1}" model: "{2}"'.format( django_model, django_field.name, django_field.model.__qualname__ ) )

    try:  # it is expected that this will throw an exception on the first try, but the late resolve should suceede
      app_config = apps.get_app_config( app_name )
      django_model = app_config.get_model( model_name )
    except AppRegistryNotReady as e:
      raise ValueError( 'App Registry Not Ready: "{0}", when resolving model name "{1}" in unexpected format, field: "{2}" model: "{3}"'.format( e, django_model, django_field.name, django_field.model.__qualname__ ) )

  if not isinstance( django_model, models.base.ModelBase ):
    raise ValueError( 'Remote Field model is not a model type, got "{0}"({1}), field: "{2}" model: "{3}"'.format( django_model, type( django_model ), django_field.name, django_field.model.__qualname__ ) )

  target_model_name = '{0}.{1}'.format( django_model.__module__, django_model.__qualname__ )
  try:
    model = __MODEL_REGISTRY__[ target_model_name ]
  except KeyError:
    raise ValueError( 'Unknown field model "{0}" make sure it is registered, field: "{1}" model: "{2}"'.format( target_model_name, django_field.name, django_field.model.__qualname__ ) )

  return ( mode, is_array, model )


def paramater_model_resolver( model_name ):
  if not isinstance( model_name, str ):
    model_name = '{0}.{1}'.format( model_name.__module__, model_name.__qualname__ )

  try:
    model = __MODEL_REGISTRY__[ model_name ]
  except KeyError:
    raise ValueError( 'Unknown paramater model "{0}" make sure it is registered'.format( model_name ) )

  return model


def property_model_resolver( model_name ):
  try:
    model = __MODEL_REGISTRY__[ model_name ]
  except KeyError:
    raise ValueError( 'Unknown paramater model "{0}" make sure it is registered'.format( model_name ) )

  return ( None, None, model )


def paramater_type_to_kwargs( paramater_type ):
  result = {}

  if isinstance( paramater_type, dict ):
    result[ 'type' ] = paramater_type[ 'type' ]

    try:
      result[ 'doc' ] = paramater_type[ 'doc' ]
    except KeyError:
      pass

    try:
      result[ 'length' ] = paramater_type[ 'length' ]
    except KeyError:
      pass

    try:
      result[ 'is_array' ] = paramater_type[ 'is_array' ]
    except KeyError:
      pass

    try:
      result[ 'allowed_scheme_list' ] = paramater_type[ 'allowed_scheme_list' ]
    except KeyError:
      pass

    paramater_model_name = paramater_type.get( 'model', None )
    if paramater_model_name is not None:
      try:
        model = paramater_model_resolver( paramater_model_name )
      except ValueError:  # model_resolver had issues, try late resolving
        result[ 'model' ] = paramater_model_name
        result[ 'model_resolve' ] = paramater_model_resolver
      else:
        result[ 'model' ] = model<commit_msg>added choice list to describe<commit_after>import re
import random
import django
from django.db import DatabaseError, models, transaction
from django.apps import apps
from django.core.exceptions import ObjectDoesNotExist, ValidationError, AppRegistryNotReady
from django.db.models import fields
from django.core.files import File

from cinp.server_common import Converter, Namespace, Model, Action, Paramater, Field, ServerError, checkAuth_true, checkAuth_false

__MODEL_REGISTRY__ = {}

# TODO: take advantage of .save( update_fields=.... ) on UPDATE

NEW_REMOTE_FIELD = int( django.get_version().split( '.' )[1] ) > 8  # last known (to me) use of related was 1.8
HAS_VIEW_PERMISSION = int( django.get_version().split( '.' )[0] ) >= 2 and int( django.get_version().split( '.' )[1] ) >= 1


def field_model_resolver( django_field ):
  mode = None
  is_array = None

  if NEW_REMOTE_FIELD:
    remote_field = django_field.remote_field
  else:
    remote_field = django_field.related

  if django_field.get_internal_type() == 'ManyToManyField':
    # detect if "through" is autogenerated, if not, make the field RO
    # we can't reach into the through model from here, they will have to
    # expose the through model to work with the relationship

    auto_created = None
    try:
      auto_created = remote_field.through._meta.auto_created
    except AttributeError:
      raise ValueError( 'Unknown target model "{0}" make sure it is registered, field: "{1}" model: "{2}"'.format( remote_field.through, django_field.name, django_field.model.__qualname__ ) )

    if auto_created is False:
      mode = 'RO'
      django_model = remote_field.through

    elif auto_created is None:
      mode = 'RO'
      django_model = remote_field.through

    else:
      mode = 'RW'
      django_model = remote_field.model

    is_array = True

  else:
    django_model = remote_field.model

  if isinstance( django_model, str ):  # this catches the case where there is a circular app refrence, so django has yet to fully map one of the sides when this is run
    try:
      ( app_name, model_name ) = django_model.split( '.' )
    except ValueError:
      raise ValueError( 'Remote Field model name "{0}" in unexpected format, field: "{1}" model: "{2}"'.format( django_model, django_field.name, django_field.model.__qualname__ ) )

    try:  # it is expected that this will throw an exception on the first try, but the late resolve should suceede
      app_config = apps.get_app_config( app_name )
      django_model = app_config.get_model( model_name )
    except AppRegistryNotReady as e:
      raise ValueError( 'App Registry Not Ready: "{0}", when resolving model name "{1}" in unexpected format, field: "{2}" model: "{3}"'.format( e, django_model, django_field.name, django_field.model.__qualname__ ) )

  if not isinstance( django_model, models.base.ModelBase ):
    raise ValueError( 'Remote Field model is not a model type, got "{0}"({1}), field: "{2}" model: "{3}"'.format( django_model, type( django_model ), django_field.name, django_field.model.__qualname__ ) )

  target_model_name = '{0}.{1}'.format( django_model.__module__, django_model.__qualname__ )
  try:
    model = __MODEL_REGISTRY__[ target_model_name ]
  except KeyError:
    raise ValueError( 'Unknown field model "{0}" make sure it is registered, field: "{1}" model: "{2}"'.format( target_model_name, django_field.name, django_field.model.__qualname__ ) )

  return ( mode, is_array, model )


def paramater_model_resolver( model_name ):
  if not isinstance( model_name, str ):
    model_name = '{0}.{1}'.format( model_name.__module__, model_name.__qualname__ )

  try:
    model = __MODEL_REGISTRY__[ model_name ]
  except KeyError:
    raise ValueError( 'Unknown paramater model "{0}" make sure it is registered'.format( model_name ) )

  return model


def property_model_resolver( model_name ):
  try:
    model = __MODEL_REGISTRY__[ model_name ]
  except KeyError:
    raise ValueError( 'Unknown paramater model "{0}" make sure it is registered'.format( model_name ) )

  return ( None, None, model )


def paramater_type_to_kwargs( paramater_type ):
  result = {}

  if isinstance( paramater_type, dict ):
    result[ 'type' ] = paramater_type[ 'type' ]

    try:
      result[ 'doc' ] = paramater_type[ 'doc' ]
    except KeyError:
      pass

    try:
      result[ 'length' ] = paramater_type[ 'length' ]
    except KeyError:
      pass

    try:
      result[ 'is_array' ] = paramater_type[ 'is_array' ]
    except KeyError:
      pass

    try:
      result[ 'choice_list' ] = paramater_type[ 'choice_list' ]
    except KeyError:
      pass

    try:
      result[ 'allowed_scheme_list' ] = paramater_type[ 'allowed_scheme_list' ]
    except KeyError:
      pass

    paramater_model_name = paramater_type.get( 'model', None )
    if paramater_model_name is not None:
      try:
        model = paramater_model_resolver( paramater_model_name )
      except ValueError:  # model_resolver had issues, try late resolving
        result[ 'model' ] = paramater_model_name
        result[ 'model_resolve' ] = paramater_model_resolver
      else:
        result[ 'model' ] = model
{'fraction_non_alphanumeric': 0.10088408644400786, 'fraction_numerical': 0.006090373280943025, 'mean_word_length': 3.7508166122258517, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3903042', 'n_tokens_mistral': 3167, 'n_tokens_neox': 3089, 'n_words': 955}