<commit_before># vim: set ts=2 et:
import argparse
import inspect

from .exceptions import DependencyException
from .fields_core import Field, Pointer, Pointers, SelfPointer, SelfPointers, Slice
from .meta import Ann, Doc


__all__ = ['AnnSchema', 'DocSchema', 'FieldSchema', 'StoreSchema', 'create_schema']


class ArgparseAction(argparse.Action):
  __slots__ = ('schema',)

  def __init__(self, schema, *args, **kwargs):
    argparse.Action.__init__(self, *args, **kwargs)
    self.schema = schema

  def __call__(self, parser, namespace, values, option_string=None):
    self.schema.serial = values


def argparse_action(self):
  def fn(*args, **kwargs):
    return ArgparseAction(self, *args, **kwargs)
  return fn


class BaseSchema(object):
  __slots__ = ('_name', '_help', '_defn', 'serial')

  def __init__(self, name, help, serial, defn):
    self._name = name
    self._help = '' if help is None else help
    self.serial = name if serial is None else serial
    self._defn = defn

  @property
  def defn(self):
    return self._defn

  @property
  def help(self):
    return self._help

  @property
  def name(self):
    return self._name

  def to_argparse(self, parser, prefix):
    arg = prefix + '-' + self.name + '-serial'
    parser.add_argument(arg, action=argparse_action(self), default=self.serial, help=self.help, metavar=self.serial)


class AnnSchema(BaseSchema):
  __slots__ = ('_fields',)

  def __init__(self, name, help, serial, defn):
    super(AnnSchema, self).__init__(name, help, serial, defn)
    self._fields = {}

  def __contains__(self, name):
    if not isinstance(name, (str, unicode)):
      raise TypeError('__contains__ needs a str or unicode field name')
    return name in self._fields

  def __getitem__(self, name):
    if not isinstance(name, (str, unicode)):
      raise TypeError('__getitem__ needs a str or unicode field name')
    return self._fields[name]

  def add_field(self, name, field):
    if not isinstance(field, FieldSchema):
      raise TypeError('argument must be a FieldSchema instance')
    self._fields[name] = field

  def fields(self):
    return self._fields.itervalues()

  def to_argparse(self, parser, prefix='-'):
    pre = prefix + '-' + self.name
    group = parser.add_argument_group(self.name, self.help)
    group.add_argument(pre + '-serial', action=argparse_action(self), default=self.serial, metavar=self.serial)
    for field in self.fields():
      field.to_argparse(group, pre)


class DocSchema(BaseSchema):
  __slots__ = ('_fields', '_stores', '_stores_by_klass', 'klasses')

  def __init__(self, name, help, serial, defn):
    super(DocSchema, self).__init__(name, help, serial, defn)
    self._fields = {}
    self._stores = {}
    self._stores_by_klass = {}
    self.klasses = []

  def __contains__(self, arg):
    if inspect.isclass(arg) and issubclass(arg, Ann):
      for klass in self.klasses:
        if klass.defn == arg:
          return True
      return False
    elif isinstance(arg, (str, unicode)):
      return arg in self._fields or arg in self._stores
    else:
      raise TypeError('__contains__ needs a str or unicode field name or an Ann subclass')

  def __getitem__(self, arg):
    if inspect.isclass(arg) and issubclass(arg, Ann):
      for klass in self.klasses:
        if klass.defn == arg:
          return klass
      raise ValueError('Class {0} was not found'.format(arg))
    elif isinstance(arg, (str, unicode)):
      if arg in self.fields:
        return self._fields[arg]
      else:
        return self._stores[arg]
    else:
      raise TypeError('__getitem__ needs a str or unicode field name or an Ann subclass')

  def add_field(self, name, field):
    if not isinstance(field, FieldSchema):
      raise TypeError('argument must be a FieldSchema instance')
    self._fields[name] = field

  def add_store(self, name, store):
    if not isinstance(store, StoreSchema):
      raise TypeError('argument must be a StoreSchema instance')
    self._stores[name] = store
    if store.stored_type.defn in self._stores_by_klass:
      self._stores_by_klass[store.stored_type.defn].append(store)
    else:
      self._stores_by_klass[store.stored_type.defn] = [store]

  def has_klass_by_serial(self, serial):
    for klass in self.klasses:
      if klass.serial == serial:
        return True
    return False

  def has_store_by_name(self, name):
    return name in self._stores

  def klass_by_serial(self, serial):
    for klass in self.klasses:
      if klass.serial == serial:
        return klass

  def store_count_by_type(self, klass):
    klasses = self._stores_by_klass.get(klass, [])
    return len(klasses)

  def store_by_name(self, name):
    return self._stores[name]

  def store_by_type(self, klass):
    return self._stores_by_klass[klass][0]

  def fields(self):
    return self._fields.itervalues()

  def stores(self):
    return self._stores.itervalues()

  def to_argparse(self, parser, prefix='-'):
    pre = prefix + '-' + self.name
    group = parser.add_argument_group(self.name, self.help)
    group.add_argument(pre + '-serial', action=argparse_action(self), default=self.serial, metavar=self.serial)
    for field in self.fields():
      field.to_argparse(group, pre)
    for store in self.stores():
      store.to_argparse(group, pre)

    for schema in self.klasses:
      schema.to_argparse(parser, prefix)


class FieldSchema(BaseSchema):
  __slots__ = ('_is_pointer', '_is_self_pointer', '_is_slice', '_points_to')

  def __init__(self, name, help, serial, defn, is_pointer, is_self_pointer, is_slice, points_to=None):
    super(FieldSchema, self).__init__(name, help, serial, defn)
    self._is_pointer = is_pointer
    self._is_self_pointer = is_self_pointer
    self._is_slice = is_slice
    self._points_to = points_to  # StoreSchema
    if is_pointer and points_to is None:
      raise ValueError('is_pointer requires points_to to point to an AnnSchema instance')

  @property
  def is_pointer(self):
    return self._is_pointer

  @property
  def is_self_pointer(self):
    return self._is_self_pointer
<commit_msg>* renamed to_argparse to add_to_argparse<commit_after># vim: set ts=2 et:
import argparse
import inspect

from .exceptions import DependencyException
from .fields_core import Field, Pointer, Pointers, SelfPointer, SelfPointers, Slice
from .meta import Ann, Doc


__all__ = ['AnnSchema', 'DocSchema', 'FieldSchema', 'StoreSchema', 'create_schema']


class ArgparseAction(argparse.Action):
  __slots__ = ('schema',)

  def __init__(self, schema, *args, **kwargs):
    argparse.Action.__init__(self, *args, **kwargs)
    self.schema = schema

  def __call__(self, parser, namespace, values, option_string=None):
    self.schema.serial = values


def argparse_action(self):
  def fn(*args, **kwargs):
    return ArgparseAction(self, *args, **kwargs)
  return fn


class BaseSchema(object):
  __slots__ = ('_name', '_help', '_defn', 'serial')

  def __init__(self, name, help, serial, defn):
    self._name = name
    self._help = '' if help is None else help
    self.serial = name if serial is None else serial
    self._defn = defn

  @property
  def defn(self):
    return self._defn

  @property
  def help(self):
    return self._help

  @property
  def name(self):
    return self._name

  def add_to_argparse(self, parser, prefix):
    arg = prefix + '-' + self.name + '-serial'
    parser.add_argument(arg, action=argparse_action(self), default=self.serial, help=self.help, metavar=self.serial)


class AnnSchema(BaseSchema):
  __slots__ = ('_fields',)

  def __init__(self, name, help, serial, defn):
    super(AnnSchema, self).__init__(name, help, serial, defn)
    self._fields = {}

  def __contains__(self, name):
    if not isinstance(name, (str, unicode)):
      raise TypeError('__contains__ needs a str or unicode field name')
    return name in self._fields

  def __getitem__(self, name):
    if not isinstance(name, (str, unicode)):
      raise TypeError('__getitem__ needs a str or unicode field name')
    return self._fields[name]

  def add_field(self, name, field):
    if not isinstance(field, FieldSchema):
      raise TypeError('argument must be a FieldSchema instance')
    self._fields[name] = field

  def fields(self):
    return self._fields.itervalues()

  def add_to_argparse(self, parser, prefix='-'):
    pre = prefix + '-' + self.name
    group = parser.add_argument_group(self.name, self.help)
    group.add_argument(pre + '-serial', action=argparse_action(self), default=self.serial, metavar=self.serial)
    for field in self.fields():
      field.add_to_argparse(group, pre)


class DocSchema(BaseSchema):
  __slots__ = ('_fields', '_stores', '_stores_by_klass', 'klasses')

  def __init__(self, name, help, serial, defn):
    super(DocSchema, self).__init__(name, help, serial, defn)
    self._fields = {}
    self._stores = {}
    self._stores_by_klass = {}
    self.klasses = []

  def __contains__(self, arg):
    if inspect.isclass(arg) and issubclass(arg, Ann):
      for klass in self.klasses:
        if klass.defn == arg:
          return True
      return False
    elif isinstance(arg, (str, unicode)):
      return arg in self._fields or arg in self._stores
    else:
      raise TypeError('__contains__ needs a str or unicode field name or an Ann subclass')

  def __getitem__(self, arg):
    if inspect.isclass(arg) and issubclass(arg, Ann):
      for klass in self.klasses:
        if klass.defn == arg:
          return klass
      raise ValueError('Class {0} was not found'.format(arg))
    elif isinstance(arg, (str, unicode)):
      if arg in self.fields:
        return self._fields[arg]
      else:
        return self._stores[arg]
    else:
      raise TypeError('__getitem__ needs a str or unicode field name or an Ann subclass')

  def add_field(self, name, field):
    if not isinstance(field, FieldSchema):
      raise TypeError('argument must be a FieldSchema instance')
    self._fields[name] = field

  def add_store(self, name, store):
    if not isinstance(store, StoreSchema):
      raise TypeError('argument must be a StoreSchema instance')
    self._stores[name] = store
    if store.stored_type.defn in self._stores_by_klass:
      self._stores_by_klass[store.stored_type.defn].append(store)
    else:
      self._stores_by_klass[store.stored_type.defn] = [store]

  def has_klass_by_serial(self, serial):
    for klass in self.klasses:
      if klass.serial == serial:
        return True
    return False

  def has_store_by_name(self, name):
    return name in self._stores

  def klass_by_serial(self, serial):
    for klass in self.klasses:
      if klass.serial == serial:
        return klass

  def store_count_by_type(self, klass):
    klasses = self._stores_by_klass.get(klass, [])
    return len(klasses)

  def store_by_name(self, name):
    return self._stores[name]

  def store_by_type(self, klass):
    return self._stores_by_klass[klass][0]

  def fields(self):
    return self._fields.itervalues()

  def stores(self):
    return self._stores.itervalues()

  def add_to_argparse(self, parser, prefix='-'):
    pre = prefix + '-' + self.name
    group = parser.add_argument_group(self.name, self.help)
    group.add_argument(pre + '-serial', action=argparse_action(self), default=self.serial, metavar=self.serial)
    for field in self.fields():
      field.add_to_argparse(group, pre)
    for store in self.stores():
      store.add_to_argparse(group, pre)

    for schema in self.klasses:
      schema.add_to_argparse(parser, prefix)


class FieldSchema(BaseSchema):
  __slots__ = ('_is_pointer', '_is_self_pointer', '_is_slice', '_points_to')

  def __init__(self, name, help, serial, defn, is_pointer, is_self_pointer, is_slice, points_to=None):
    super(FieldSchema, self).__init__(name, help, serial, defn)
    self._is_pointer = is_pointer
    self._is_self_pointer = is_self_pointer
    self._is_slice = is_slice
    self._points_to = points_to  # StoreSchema
    if is_pointer and points_to is None:
      raise ValueError('is_pointer requires points_to to point to an AnnSchema instance')

  @property
  def is_pointer(self):
    return self._is_pointer

  @property
  def is_self_pointer(self):
    return self._is_self_pointer

{'fraction_non_alphanumeric': 0.11183941007783695, 'fraction_numerical': 0.0004916018025399426, 'mean_word_length': 3.69281045751634, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4509423', 'n_tokens_mistral': 3971, 'n_tokens_neox': 3918, 'n_words': 1066}