<commit_before>import logging
import calendar
import copy
import dateutil
import numpy
import datetime
from couchdbkit.ext.django.schema import Document, StringProperty, IntegerProperty, DateTimeProperty
from couchdbkit.schema.base import DocumentSchema
from couchdbkit.schema.properties import LazyDict
from casexml.apps.case.models import CommCareCase
from corehq.apps.crud.models import AdminCRUDDocumentMixin
from corehq.apps.indicators.admin.crud import (IndicatorAdminCRUDManager,
                                               FormAliasIndicatorAdminCRUDManager,
                                               FormLabelIndicatorAdminCRUDManager, CaseDataInFormIndicatorAdminCRUDManager, FormDataInCaseAdminCRUDManager, CouchIndicatorCRUDManager, BaseDynamicIndicatorCRUDManager, CombinedCouchIndicatorCRUDManager)
from couchforms.models import XFormInstance
from dimagi.utils.couch.database import get_db
from dimagi.utils.dates import DateSpan, add_months, months_between
from dimagi.utils.decorators.memoized import memoized
from dimagi.utils.modules import to_function
from dimagi.utils.couch.cache import cache_core


class DocumentNotInDomainError(Exception):
    pass

class DocumentMismatchError(Exception):
    pass


class IndicatorDefinition(Document, AdminCRUDDocumentMixin):
    """
        An Indicator Definition defines how to compute the indicator that lives
        in the namespaced computed_ property of a case or form.
    """
    namespace = StringProperty()
    domain = StringProperty()
    slug = StringProperty()
    version = IntegerProperty()
    class_path = StringProperty()
    last_modified = DateTimeProperty()

    _admin_crud_class = IndicatorAdminCRUDManager

    _class_path = "corehq.apps.indicators.models"
    _returns_multiple = False

    def __init__(self, _d=None, **kwargs):
        super(IndicatorDefinition, self).__init__(_d, **kwargs)
        self.class_path = self._class_path

    def __str__(self):
        return "\n\n%(class_name)s - Modified %(last_modified)s\n %(slug)s, domain: %(domain)s," \
            " version: %(version)s, namespace: %(namespace)s. ID: %(indicator_id)s." % {
                'class_name': self.__class__.__name__,
                'slug': self.slug,
                'domain': self.domain,
                'version': self.version,
                'namespace': self.namespace,
                'last_modified': (self.last_modified.strftime('%m %B %Y at %H:%M')
                                  if self.last_modified else "<NAME>"),
                'indicator_id': self._id,
            }

    @classmethod
    def key_properties(cls):
        """
            The ordering of these property names should match the ordering of what's emitted in the first part of
            the couch views used for fetching these indicators. These views currently are:
            - indicators/dynamic_indicator_definitions (Couch View Indicator Defs)
            - indicators/indicator_definitions (Form and Case Indicator Defs)
        """
        return ["namespace", "domain", "slug"]

    @classmethod
    def indicator_list_view(cls):
        return "indicators/indicator_definitions"

    @classmethod
    def _generate_couch_key(cls, version=None, reverse=False, **kwargs):
        key = list()
        key_prefix = list()
        for p in cls.key_properties():
            k = kwargs.get(p)
            if k is not None:
                key_prefix.append(p)
                key.append(k)
        key = [" ".join(key_prefix)] + key
        couch_key = dict(startkey=key, endkey=key+[{}]) if version is None else dict(key=key+[version])
        if reverse:
            return dict(startkey=couch_key.get('endkey'), endkey=couch_key.get('startkey'))
        return couch_key

    @classmethod
    def increment_or_create_unique(cls, namespace, domain, slug=None, version=None, **kwargs):
        """
        If an indicator with the same namespace, domain, and version exists, create a new indicator with the
        version number incremented.
        # todo, this feels a bit buggy, so replace bulk copy indicators with
        # copy to domain at some point
        """
        couch_key = cls._generate_couch_key(
            namespace=namespace,
            domain=domain,
            slug=slug,
            reverse=True,
            **kwargs
        )

        existing_indicator = cls.view(
            cls.indicator_list_view(),
            reduce=False,
            include_docs=True,
            descending=True,
            limit=1,
            **couch_key
        ).first()
        if existing_indicator:
            version = existing_indicator.version + 1
        elif version is None:
            version = 1

        new_indicator = cls(
            version=version,
            namespace=namespace,
            domain=domain,
            slug=slug,
            **kwargs
        )
        new_indicator.last_modified = datetime.datetime.utcnow()

        new_indicator.save()
        return new_indicator

    @classmethod
    def copy_to_domain(cls, domain, doc, override=False):
        """
        This copies an indicator doc to the current domain. Intended to be used
        by the export indicators feature.
        :param domain: the name of the domain the indicator should be copied to
        :param doc: the dictionary of kwargs to create the indicator
        :param override: Whether to override the existing indicator
        :return: True if indicator was copied, False if not
        """
        if '_id' in doc:
            del doc['_id']
        if '_rev' in doc:
            del doc['_rev']
        if 'last_modified' in doc:
            del doc['last_modified']

        couch_key = cls._generate_couch_key(
            domain=domain,
            reverse=True,
            **doc
        )
        existing_indicator = cls.view(
            cls.indicator_list_view(),
            reduce=False,
            include_docs=False,
            descending=True,
            limit=1,
            **couch_key
        ).first()<commit_msg>cleanup<commit_after>import logging
import calendar
import copy
import dateutil
import numpy
import datetime
from couchdbkit.ext.django.schema import Document, StringProperty, IntegerProperty, DateTimeProperty
from couchdbkit.schema.base import DocumentSchema
from couchdbkit.schema.properties import LazyDict
from casexml.apps.case.models import CommCareCase
from corehq.apps.crud.models import AdminCRUDDocumentMixin
from corehq.apps.indicators.admin.crud import (IndicatorAdminCRUDManager,
                                               FormAliasIndicatorAdminCRUDManager,
                                               FormLabelIndicatorAdminCRUDManager, CaseDataInFormIndicatorAdminCRUDManager, FormDataInCaseAdminCRUDManager, CouchIndicatorCRUDManager, BaseDynamicIndicatorCRUDManager, CombinedCouchIndicatorCRUDManager)
from couchforms.models import XFormInstance
from dimagi.utils.couch.database import get_db
from dimagi.utils.dates import DateSpan, add_months, months_between
from dimagi.utils.decorators.memoized import memoized
from dimagi.utils.modules import to_function
from dimagi.utils.couch.cache import cache_core


class DocumentNotInDomainError(Exception):
    pass

class DocumentMismatchError(Exception):
    pass


class IndicatorDefinition(Document, AdminCRUDDocumentMixin):
    """
        An Indicator Definition defines how to compute the indicator that lives
        in the namespaced computed_ property of a case or form.
    """
    namespace = StringProperty()
    domain = StringProperty()
    slug = StringProperty()
    version = IntegerProperty()
    class_path = StringProperty()
    last_modified = DateTimeProperty()

    _admin_crud_class = IndicatorAdminCRUDManager

    _class_path = "corehq.apps.indicators.models"
    _returns_multiple = False

    def __init__(self, _d=None, **kwargs):
        super(IndicatorDefinition, self).__init__(_d, **kwargs)
        self.class_path = self._class_path

    def __str__(self):
        return "\n\n%(class_name)s - Modified %(last_modified)s\n %(slug)s, domain: %(domain)s," \
            " version: %(version)s, namespace: %(namespace)s. ID: %(indicator_id)s." % {
                'class_name': self.__class__.__name__,
                'slug': self.slug,
                'domain': self.domain,
                'version': self.version,
                'namespace': self.namespace,
                'last_modified': (self.last_modified.strftime('%m %B %Y at %H:%M')
                                  if self.last_modified else "Ages Ago"),
                'indicator_id': self._id,
            }

    @classmethod
    def key_properties(cls):
        """
            The ordering of these property names should match the ordering of what's emitted in the first part of
            the couch views used for fetching these indicators. These views currently are:
            - indicators/dynamic_indicator_definitions (Couch View Indicator Defs)
            - indicators/indicator_definitions (Form and Case Indicator Defs)
        """
        return ["namespace", "domain", "slug"]

    @classmethod
    def indicator_list_view(cls):
        return "indicators/indicator_definitions"

    @classmethod
    def _generate_couch_key(cls, version=None, reverse=False, **kwargs):
        key = list()
        key_prefix = list()
        for p in cls.key_properties():
            k = kwargs.get(p)
            if k is not None:
                key_prefix.append(p)
                key.append(k)
        key = [" ".join(key_prefix)] + key
        couch_key = dict(startkey=key, endkey=key+[{}]) if version is None else dict(key=key+[version])
        if reverse:
            return dict(startkey=couch_key.get('endkey'), endkey=couch_key.get('startkey'))
        return couch_key

    @classmethod
    def increment_or_create_unique(cls, namespace, domain, slug=None, version=None, **kwargs):
        """
        If an indicator with the same namespace, domain, and version exists, create a new indicator with the
        version number incremented.
        # todo, this feels a bit buggy, so replace bulk copy indicators with
        # copy to domain at some point
        """
        couch_key = cls._generate_couch_key(
            namespace=namespace,
            domain=domain,
            slug=slug,
            reverse=True,
            **kwargs
        )

        existing_indicator = cls.view(
            cls.indicator_list_view(),
            reduce=False,
            include_docs=True,
            descending=True,
            limit=1,
            **couch_key
        ).first()
        if existing_indicator:
            version = existing_indicator.version + 1
        elif version is None:
            version = 1

        new_indicator = cls(
            version=version,
            namespace=namespace,
            domain=domain,
            slug=slug,
            **kwargs
        )
        new_indicator.last_modified = datetime.datetime.utcnow()

        new_indicator.save()
        return new_indicator

    @classmethod
    def copy_to_domain(cls, domain, doc, override=False):
        """
        This copies an indicator doc to the current domain. Intended to be used
        by the export indicators feature.
        :param domain: the name of the domain the indicator should be copied to
        :param doc: the dictionary of kwargs to create the indicator
        :param override: Whether to override the existing indicator
        :return: True if indicator was copied, False if not
        """
        for reserved in ['_id', '_rev', 'last_modified']:
            if reserved in doc:
                del doc[reserved]

        couch_key = cls._generate_couch_key(
            domain=domain,
            reverse=True,
            **doc
        )
        existing_indicator = cls.view(
            cls.indicator_list_view(),
            reduce=False,
            include_docs=False,
            descending=True,
            limit=1,
            **couch_key
        ).first()
{'fraction_non_alphanumeric': 0.08065590144831031, 'fraction_numerical': 0.000665889795238888, 'mean_word_length': 2.3375, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6865509', 'n_tokens_mistral': 3213, 'n_tokens_neox': 3180, 'n_words': 878}