<commit_before>##
# Copyright (c) 2006-2009 Apple Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
##

"""
Apple OpenDirectory directory service implementation.
"""

__all__ = [
    "OpenDirectoryService",
    "OpenDirectoryInitError",
]

import sys
import time
from uuid import UUID

from twext.python.plistlib import readPlistFromString

from xml.parsers.expat import ExpatError

import opendirectory
import dsattributes
import dsquery

from twisted.internet.threads import deferToThread
from twisted.cred.credentials import UsernamePassword
from twisted.web2.auth.digest import DigestedCredentials
from twistedcaldav.config import config

from twistedcaldav.directory.cachingdirectory import CachingDirectoryService,\
    CachingDirectoryRecord
from twistedcaldav.directory.directory import DirectoryService, DirectoryRecord
from twistedcaldav.directory.directory import DirectoryError, UnknownRecordTypeError
from twistedcaldav.directory.principal import cuAddressConverter

class OpenDirectoryService(CachingDirectoryService):
    """
    OpenDirectory implementation of L{IDirectoryService}.
    """
    baseGUID = "891F8321-ED02-424C-BA72-89C32F215C1E"

    def __repr__(self):
        return "<%s %r: %r>" % (self.__class__.__name__, self.realmName, self.node)


    def __init__(self, params, dosetup=True):
        """
        @param params: a dictionary containing the following keys:
            node: an OpenDirectory node name to bind to.
            restrictEnabledRecords: C{True} if a group in the
              directory is to be used to determine which calendar
              users are enabled.
            restrictToGroup: C{str} guid or name of group used to
              restrict enabled users.
            cacheTimeout: C{int} number of minutes before cache is invalidated.
        @param dosetup: if C{True} then the directory records are initialized,
                        if C{False} they are not.
                        This should only be set to C{False} when doing unit tests.
        """

        defaults = {
            'node' : '/Search',
            'restrictEnabledRecords' : False,
            'restrictToGroup' : '',
            'cacheTimeout' : 30,
        }
        ignored = ('requireComputerRecord',)
        params = self.getParams(params, defaults, ignored)

        super(OpenDirectoryService, self).__init__(params['cacheTimeout'])

        try:
            directory = opendirectory.odInit(params['node'])
        except opendirectory.ODError, e:
            self.log_error("OpenDirectory (node=%s) Initialization error: %s" % (params['node'], e))
            raise

        self.realmName = params['node']
        self.directory = directory
        self.node = params['node']
        self.restrictEnabledRecords = params['restrictEnabledRecords']
        self.restrictToGroup = params['restrictToGroup']
        try:
            UUID(self.restrictToGroup)
        except:
            self.restrictToGUID = False
        else:
            self.restrictToGUID = True
        self.restrictedGUIDs = None
        self.restrictedTimestamp = 0
        self._records = {}
        self._delayedCalls = set()

    def __cmp__(self, other):
        if not isinstance(other, DirectoryRecord):
            return super(DirectoryRecord, self).__eq__(other)

        for attr in ("directory", "node"):
            diff = cmp(getattr(self, attr), getattr(other, attr))
            if diff != 0:
                return diff
        return 0

    def __hash__(self):
        h = hash(self.__class__)
        for attr in ("directory", "node"):
            h = (h + hash(getattr(self, attr))) & sys.maxint
        return h

    def _expandGroupMembership(self, members, nestedGroups, processedGUIDs=None, returnGroups=False):

        if processedGUIDs is None:
            processedGUIDs = set()

        if isinstance(members, str):
            members = [members]

        if isinstance(nestedGroups, str):
            nestedGroups = [nestedGroups]

        for memberGUID in members:
            if memberGUID not in processedGUIDs:
                processedGUIDs.add(memberGUID)
                yield memberGUID

        for groupGUID in nestedGroups:
            if groupGUID in processedGUIDs:
                continue

            self.log_debug("opendirectory.queryRecordsWithAttribute_list(%r,%r,%r,%r,%r,%r,%r)" % (
                self.directory,
                dsattributes.kDS1AttrGeneratedUID,
                groupGUID,
                dsattributes.eDSExact,
                False,
                dsattributes.kDSStdRecordTypeGroups,
                [dsattributes.kDSNAttrGroupMembers, dsattributes.kDSNAttrNestedGroups]
            ))
            result = opendirectory.queryRecordsWithAttribute_list(
                self.directory,
                dsattributes.kDS1AttrGeneratedUID,
                groupGUID,
                dsattributes.eDSExact,
                False,
                dsattributes.kDSStdRecordTypeGroups,
                [dsattributes.kDSNAttrGroupMembers, dsattributes.kDSNAttrNestedGroups]
            )

            if not result:
                self.log_error("Couldn't find group %s when trying to expand nested groups."
                             % (groupGUID,))
                continue

            group = result[0][1]

            processedGUIDs.add(groupGUID)
            if returnGroups:
                yield groupGUID

            for GUID in self._expandGroupMembership(
                group.get(dsattributes.kDSNAttrGroupMembers, []),
                group.get(dsattributes.kDSNAttrNestedGroups, []),
                processedGUIDs,
                returnGroups,
            ):
                yield GUID

    def recordTypes(self):
        return (
            self.recordType_users,
            self.recordType_groups,
            self.recordType_locations,
            self.recordType_resources,
        )

    def groupsForGUID(self, guid):
        
        attrs = [
            dsattributes.kDS1AttrGeneratedUID,
        ]

        recordType = dsattributes.kDSStdRecordTypeGroups

        guids = set()

        query = dsquery.match(dsattributes.kDSNAttrGroupMembers, guid, dsattributes.eDSExact)
        try:
            self.log_debug("opendirectory.queryRecordsWithAttribute_list(%r,%r,%r,%r,%r,%r,%r)" % (
                self.directory,
                query.attribute,
                query.value,
                query.matchType,
                False,
                recordType,
                attrs,
            ))
            results = opendirectory.queryRecordsWithAttribute_list(
                self.directory,
                query.attribute,
                query.value,
                query.matchType,
                False,
                recordType,
                attrs,
            )
        except opendirectory.ODError, ex:
            self.log_error("OpenDirectory (node=%s) error: %s" % (self.realmName, str(ex)))
            raise

        for (_ignore_recordShortName, value) in results:

            # Now get useful record info.
            recordGUID = value.get(dsattributes.kDS1AttrGeneratedUID)
            if recordGUID:
                guids.add(recordGUID)

        query = dsquery.match(dsattributes.kDSNAttrNestedGroups, guid, dsattributes.eDSExact)
        try:
            self.log_debug("opendirectory.queryRecordsWithAttribute_list(%r,%r,%r,%r,%r,%r,%r)" % (
                self.directory,
                query.attribute,
                query.value,
                query.matchType,
                False,
                recordType,
                attrs,
            ))
            results = opendirectory.queryRecordsWithAttribute_list(
                self.directory,
                query.attribute,
                query.value,
                query.matchType,
                False,
                recordType,
                attrs,
            )
        except opendirectory.ODError, ex:
            self.log_error("OpenDirectory (node=%s) error: %s" % (self.realmName, str(ex)))
            raise

        for (_ignore_recordShortName, value) in results:

            # Now get useful record info.
            recordGUID = value.get(dsattributes.kDS1AttrGeneratedUID)
            if recordGUID:
                guids.add(recordGUID)

        return guids

    def proxiesForGUID(self, recordType, guid):
        
        # Lookup in index
        try:
            # TODO:
            return ()
        except KeyError:
            return ()

    def readOnlyProxiesForGUID(self, recordType, guid):
        
        # Lookup in index
        try:
            # TODO:
            return ()
        except KeyError:
            return ()

    _ODFields = {
        'fullName' : {
            'odField' : dsattributes.kDS1AttrDistinguishedName,
            'appliesTo' : set([
                dsattributes.kDSStdRecordTypeUsers,
                dsattributes.kDSStdRecordTypeGroups,
                dsattributes.kDSStdRecordTypePlaces,
                dsattributes.kDSStdRecordTypeResources,
            ]),
        },
        'firstName' : {
            'odField' : dsattributes.kDS1AttrFirstName,
            'appliesTo' : set([
                dsattributes.kDSStdRecordTypeUsers,
            ]),
        },
        'lastName' : {
            'odField' : dsattributes.kDS1AttrLastName,
            'appliesTo' : set([
                dsattributes.kDSStdRecordTypeUsers,
            ]),
        },
        'emailAddresses' : {
            'odField' : dsattributes.kDSNAttrEMailAddress,
            'appliesTo' : set([
                dsattributes.kDSStdRecordTypeUsers,
                dsattributes.kDSStdRecordTypeGroups,
            ]),
        },
        'recordName' : {
            'odField' : dsattributes.kDSNAttrRecordName,
            'appliesTo' : set([
                dsattributes.kDSStdRecordTypeUsers,
                dsattributes.kDSStdRecordTypeGroups,
                dsattributes.kDSStdRecordTypePlaces,
                dsattributes.kDSStdRecordTypeResources,
            ]),
        },
        'guid' : {
            'odField' : dsattributes.kDS1AttrGeneratedUID,
            'appliesTo' : set([
                dsattributes.kDSStdRecordTypeUsers,
                dsattributes.kDSStdRecordTypeGroups,
                dsattributes.kDSStdRecordTypePlaces,
                dsattributes.kDSStdRecordTypeResources,
            ]),
        },
    }

    _toODRecordTypes = {
        DirectoryService.recordType_users :
            dsattributes.kDSStdRecordTypeUsers,
        DirectoryService.recordType_locations :
            dsattributes.kDSStdRecordTypePlaces,
        DirectoryService.recordType_groups :
            dsattributes.kDSStdRecordTypeGroups,
        DirectoryService.recordType_resources :
            dsattributes.kDSStdRecordTypeResources,
    }

    _fromODRecordTypes = dict([(b, a) for a, b in _toODRecordTypes.iteritems()])

    def _uniqueTupleFromAttribute(self, attribute):
        if attribute:
            if isinstance(attribute, str):
                return (attribute,)
            else:
                s = set()
                return tuple([(s.add(x), x)[1] for x in attribute if x not in s])
        else:
            return ()

    def _setFromAttribute(self, attribute, lower=False):
        if attribute:
            if isinstance(attribute, str):
                return set((attribute.lower() if lower else attribute,))
            else:
                return set([item.lower() if lower else item for item in attribute])
        else:
            return ()

    def recordsMatchingFields(self, fields, operand="or", recordType=None,
        lookupMethod=opendirectory.queryRecordsWithAttributes_list):

        # Note that OD applies case-sensitivity globally across the entire
        # query, not per expression, so the current code uses whatever is
        # specified in the last field in the fields list

        def collectResults(results):
            self.log_debug("Got back %d records from OD" % (len(results),))
            for key, value in results:
                self.log_debug("OD result: %s %s" % (key, value))
                try:
                    recordNodeName = value.get(
                        dsattributes.kDSNAttrMetaNodeLocation)
                    recordShortNames = self._uniqueTupleFromAttribute(
                        value.get(dsattributes.kDSNAttrRecordName))
                    if (recordNodeName == "/Local/Default" and not
                        (config.Scheduling.iMIP.Username in recordShortNames)):
                        continue

                    recordGUID = value.get(dsattributes.kDS1AttrGeneratedUID)
                    recordType = value.get(dsattributes.kDSNAttrRecordType)
                    if isinstance(recordType, list):
                        recordType = recordType[0]
                    if not recordType:
                        continue
                    recordType = self._fromODRecordTypes[recordType]

                    recordAuthIDs = self._setFromAttribute(
                        value.get(dsattributes.kDSNAttrAltSecurityIdentities))
                    recordFullName = value.get(
                        dsattributes.kDS1AttrDistinguishedName)
                    recordFirstName = value.get(dsattributes.kDS1AttrFirstName)
                    recordLastName = value.get(dsattributes.kDS1AttrLastName)
                    recordEmailAddresses = self._setFromAttribute(
                        value.get(dsattributes.kDSNAttrEMailAddress),
                        lower=True)

                    # Create records but don't store them in our index or
                    # send them to memcached, because these are transient,
                    # existing only so we can create principal resource
                    # objects that are used to generate the REPORT result.

                    record = OpenDirectoryRecord(
                        service               = self,
                        recordType            = recordType,
                        guid                  = recordGUID,
                        nodeName              = recordNodeName,
                        shortNames            = recordShortNames,
                        authIDs               = recordAuthIDs,
                        fullName              = recordFullName,
                        firstName             = recordFirstName,
                        lastName              = recordLastName,
                        emailAddresses        = recordEmailAddresses,
                        enabledForCalendaring = True,
                        memberGUIDs           = (),
                    )
                    yield record

                except KeyError:
                    pass

        def multiQuery(directory, queries, attrs, operand):
            results = []

            for query, recordTypes in queries.iteritems():
                if not query:
                    continue

                expressions = []
                for ODField, value, caseless, matchType in query:
                    if matchType == "starts-with":
                        comparison = dsattributes.eDSStartsWith
                    elif matchType == "contains":
                        comparison = dsattributes.eDSContains
                    else:
                        comparison = dsattributes.eDSExact
                    expressions.append(dsquery.match(ODField, value, comparison))

                complexExpression = dsquery.expression(operand, expressions).generate()

                self.log_debug("Calling OD: Types %s, Operand %s, Caseless %s, %s" %
                    (recordTypes, operand, caseless, complexExpression))

                results.extend(
                    lookupMethod(
                        directory,
                        complexExpression,
                        caseless,
                        recordTypes,
                        attrs,
                    )
                )

            return results


        operand = (dsquery.expression.OR if operand == "or"
            else dsquery.expression.AND)

        if recordType is None:
            # The client is looking for records in any of the four types
            recordTypes = set(self._toODRecordTypes.values())
        else:
            # The client is after only one recordType
            recordTypes = [self._toODRecordTypes[recordType]]

        queries = buildQueries(recordTypes, fields, self._ODFields)

        deferred = deferToThread(
            multiQuery,
            self.directory,
            queries,
            [
                dsattributes.kDS1AttrGeneratedUID,
                dsattributes.kDSNAttrRecordName,
                dsattributes.kDSNAttrAltSecurityIdentities,
                dsattributes.kDSNAttrRecordType,
                dsattributes.kDS1AttrDistinguishedName,
                dsattributes.kDS1AttrFirstName,
                dsattributes.kDS1AttrLastName,
                dsattributes.kDSNAttrEMailAddress,
                dsattributes.kDSNAttrMetaNodeLocation,
            ],
            operand
        )
        deferred.addCallback(collectResults)
        return deferred


    def queryDirectory(self, recordTypes, indexType, indexKey,
        lookupMethod=opendirectory.queryRecordsWithAttribute_list):
        
        attrs = [
            dsattributes.kDS1AttrGeneratedUID,
            dsattributes.kDSNAttrRecordName,
            dsattributes.kDSNAttrAltSecurityIdentities,
            dsattributes.kDSNAttrRecordType,
            dsattributes.kDS1AttrDistinguishedName,
            dsattributes.kDS1AttrFirstName,
            dsattributes.kDS1AttrLastName,
            dsattributes.kDSNAttrEMailAddress,
            dsattributes.kDSNAttrMetaNodeLocation,
        ]

        origIndexKey = indexKey
        if indexType == self.INDEX_TYPE_CUA:
            # The directory doesn't contain CUAs, so we need to convert
            # the CUA to the appropriate field name and value:
            queryattr, indexKey = cuAddressConverter(indexKey)
            # queryattr will be one of:
            # guid, emailAddresses, or recordName
            # ...which will need to be mapped to DS
            queryattr = self._ODFields[queryattr]['odField']

        else:
            queryattr = {
                self.INDEX_TYPE_SHORTNAME : dsattributes.kDSNAttrRecordName,
                self.INDEX_TYPE_GUID      : dsattributes.kDS1AttrGeneratedUID,
                self.INDEX_TYPE_AUTHID    : dsattributes.kDSNAttrAltSecurityIdentities,
            }.get(indexType)
            assert queryattr is not None, "Invalid type for record faulting query"

        query = dsquery.match(queryattr, indexKey, dsattributes.eDSExact)


        listRecordTypes = []
        for recordType in recordTypes:
            if recordType == DirectoryService.recordType_users:
                listRecordTypes.append(dsattributes.kDSStdRecordTypeUsers)
    
            elif recordType == DirectoryService.recordType_groups:
                if queryattr != dsattributes.kDSNAttrEMailAddress:
                    listRecordTypes.append(dsattributes.kDSStdRecordTypeGroups)
                    attrs.append(dsattributes.kDSNAttrGroupMembers)
                    attrs.append(dsattributes.kDSNAttrNestedGroups)
    
            elif recordType == DirectoryService.recordType_locations:
                if queryattr != dsattributes.kDSNAttrEMailAddress:
                    listRecordTypes.append(dsattributes.kDSStdRecordTypePlaces)
                # MOR: possibly can be removed
                attrs.append(dsattributes.kDSNAttrResourceInfo)
            
            elif recordType == DirectoryService.recordType_resources:
                if queryattr != dsattributes.kDSNAttrEMailAddress:
                    listRecordTypes.append(dsattributes.kDSStdRecordTypeResources)
                # MOR: possibly can be removed
                attrs.append(dsattributes.kDSNAttrResourceInfo)
            
            else:
                raise UnknownRecordTypeError("Unknown OpenDirectory record type: %s" % (recordType))


        try:
            self.log_debug("opendirectory.queryRecordsWithAttribute_list(%r,%r,%r,%r,%r,%r,%r)" % (
                self.directory,
                query.attribute,
                query.value,
                query.matchType,
                False,
                listRecordTypes,
                attrs,
            ))
            results = lookupMethod(
                self.directory,
                query.attribute,
                query.value,
                query.matchType,
                False,
                listRecordTypes,
                attrs,
            )
            self.log_debug("opendirectory.queryRecordsWithAttribute_list matched records: %s" % (len(results),))

        except opendirectory.ODError, ex:
            if ex.message[1] == -14140 or ex.message[1] == -14200:
                # Unsupported attribute on record - don't fail
                return
            else:
                self.log_error("OpenDirectory (node=%s) error: %s" % (self.realmName, str(ex)))
                raise


        enabledRecords = []
        disabledRecords = []

        for (recordShortName, value) in results:

            # Now get useful record info.
            recordGUID           = value.get(dsattributes.kDS1AttrGeneratedUID)
            recordShortNames     = self._uniqueTupleFromAttribute(value.get(dsattributes.kDSNAttrRecordName))
            recordType           = value.get(dsattributes.kDSNAttrRecordType)
            if isinstance(recordType, list):
                recordType = recordType[0]
            recordAuthIDs        = self._setFromAttribute(value.get(dsattributes.kDSNAttrAltSecurityIdentities))
            recordFullName       = value.get(dsattributes.kDS1AttrDistinguishedName)
            recordFirstName      = value.get(dsattributes.kDS1AttrFirstName)
            recordLastName       = value.get(dsattributes.kDS1AttrLastName)
            recordEmailAddresses = self._setFromAttribute(value.get(dsattributes.kDSNAttrEMailAddress), lower=True)
            recordNodeName       = value.get(dsattributes.kDSNAttrMetaNodeLocation)

            if recordNodeName == "/Local/Default" and not (config.Scheduling.iMIP.Username in recordShortNames):
                self.log_warn("Local record (%s)%s is not eligible for calendaring."
                              % (recordType, recordShortName))
                continue

            if not recordType:
                self.log_debug("Record (unknown)%s in node %s has no recordType; ignoring."
                               % (recordShortName, recordNodeName))
                continue

            recordType = self._fromODRecordTypes[recordType]

            if not recordGUID:<commit_msg>Another log level tweak<commit_after>##
# Copyright (c) 2006-2009 Apple Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
##

"""
Apple OpenDirectory directory service implementation.
"""

__all__ = [
    "OpenDirectoryService",
    "OpenDirectoryInitError",
]

import sys
import time
from uuid import UUID

from twext.python.plistlib import readPlistFromString

from xml.parsers.expat import ExpatError

import opendirectory
import dsattributes
import dsquery

from twisted.internet.threads import deferToThread
from twisted.cred.credentials import UsernamePassword
from twisted.web2.auth.digest import DigestedCredentials
from twistedcaldav.config import config

from twistedcaldav.directory.cachingdirectory import CachingDirectoryService,\
    CachingDirectoryRecord
from twistedcaldav.directory.directory import DirectoryService, DirectoryRecord
from twistedcaldav.directory.directory import DirectoryError, UnknownRecordTypeError
from twistedcaldav.directory.principal import cuAddressConverter

class OpenDirectoryService(CachingDirectoryService):
    """
    OpenDirectory implementation of L{IDirectoryService}.
    """
    baseGUID = "891F8321-ED02-424C-BA72-89C32F215C1E"

    def __repr__(self):
        return "<%s %r: %r>" % (self.__class__.__name__, self.realmName, self.node)


    def __init__(self, params, dosetup=True):
        """
        @param params: a dictionary containing the following keys:
            node: an OpenDirectory node name to bind to.
            restrictEnabledRecords: C{True} if a group in the
              directory is to be used to determine which calendar
              users are enabled.
            restrictToGroup: C{str} guid or name of group used to
              restrict enabled users.
            cacheTimeout: C{int} number of minutes before cache is invalidated.
        @param dosetup: if C{True} then the directory records are initialized,
                        if C{False} they are not.
                        This should only be set to C{False} when doing unit tests.
        """

        defaults = {
            'node' : '/Search',
            'restrictEnabledRecords' : False,
            'restrictToGroup' : '',
            'cacheTimeout' : 30,
        }
        ignored = ('requireComputerRecord',)
        params = self.getParams(params, defaults, ignored)

        super(OpenDirectoryService, self).__init__(params['cacheTimeout'])

        try:
            directory = opendirectory.odInit(params['node'])
        except opendirectory.ODError, e:
            self.log_error("OpenDirectory (node=%s) Initialization error: %s" % (params['node'], e))
            raise

        self.realmName = params['node']
        self.directory = directory
        self.node = params['node']
        self.restrictEnabledRecords = params['restrictEnabledRecords']
        self.restrictToGroup = params['restrictToGroup']
        try:
            UUID(self.restrictToGroup)
        except:
            self.restrictToGUID = False
        else:
            self.restrictToGUID = True
        self.restrictedGUIDs = None
        self.restrictedTimestamp = 0
        self._records = {}
        self._delayedCalls = set()

    def __cmp__(self, other):
        if not isinstance(other, DirectoryRecord):
            return super(DirectoryRecord, self).__eq__(other)

        for attr in ("directory", "node"):
            diff = cmp(getattr(self, attr), getattr(other, attr))
            if diff != 0:
                return diff
        return 0

    def __hash__(self):
        h = hash(self.__class__)
        for attr in ("directory", "node"):
            h = (h + hash(getattr(self, attr))) & sys.maxint
        return h

    def _expandGroupMembership(self, members, nestedGroups, processedGUIDs=None, returnGroups=False):

        if processedGUIDs is None:
            processedGUIDs = set()

        if isinstance(members, str):
            members = [members]

        if isinstance(nestedGroups, str):
            nestedGroups = [nestedGroups]

        for memberGUID in members:
            if memberGUID not in processedGUIDs:
                processedGUIDs.add(memberGUID)
                yield memberGUID

        for groupGUID in nestedGroups:
            if groupGUID in processedGUIDs:
                continue

            self.log_debug("opendirectory.queryRecordsWithAttribute_list(%r,%r,%r,%r,%r,%r,%r)" % (
                self.directory,
                dsattributes.kDS1AttrGeneratedUID,
                groupGUID,
                dsattributes.eDSExact,
                False,
                dsattributes.kDSStdRecordTypeGroups,
                [dsattributes.kDSNAttrGroupMembers, dsattributes.kDSNAttrNestedGroups]
            ))
            result = opendirectory.queryRecordsWithAttribute_list(
                self.directory,
                dsattributes.kDS1AttrGeneratedUID,
                groupGUID,
                dsattributes.eDSExact,
                False,
                dsattributes.kDSStdRecordTypeGroups,
                [dsattributes.kDSNAttrGroupMembers, dsattributes.kDSNAttrNestedGroups]
            )

            if not result:
                self.log_error("Couldn't find group %s when trying to expand nested groups."
                             % (groupGUID,))
                continue

            group = result[0][1]

            processedGUIDs.add(groupGUID)
            if returnGroups:
                yield groupGUID

            for GUID in self._expandGroupMembership(
                group.get(dsattributes.kDSNAttrGroupMembers, []),
                group.get(dsattributes.kDSNAttrNestedGroups, []),
                processedGUIDs,
                returnGroups,
            ):
                yield GUID

    def recordTypes(self):
        return (
            self.recordType_users,
            self.recordType_groups,
            self.recordType_locations,
            self.recordType_resources,
        )

    def groupsForGUID(self, guid):
        
        attrs = [
            dsattributes.kDS1AttrGeneratedUID,
        ]

        recordType = dsattributes.kDSStdRecordTypeGroups

        guids = set()

        query = dsquery.match(dsattributes.kDSNAttrGroupMembers, guid, dsattributes.eDSExact)
        try:
            self.log_debug("opendirectory.queryRecordsWithAttribute_list(%r,%r,%r,%r,%r,%r,%r)" % (
                self.directory,
                query.attribute,
                query.value,
                query.matchType,
                False,
                recordType,
                attrs,
            ))
            results = opendirectory.queryRecordsWithAttribute_list(
                self.directory,
                query.attribute,
                query.value,
                query.matchType,
                False,
                recordType,
                attrs,
            )
        except opendirectory.ODError, ex:
            self.log_error("OpenDirectory (node=%s) error: %s" % (self.realmName, str(ex)))
            raise

        for (_ignore_recordShortName, value) in results:

            # Now get useful record info.
            recordGUID = value.get(dsattributes.kDS1AttrGeneratedUID)
            if recordGUID:
                guids.add(recordGUID)

        query = dsquery.match(dsattributes.kDSNAttrNestedGroups, guid, dsattributes.eDSExact)
        try:
            self.log_debug("opendirectory.queryRecordsWithAttribute_list(%r,%r,%r,%r,%r,%r,%r)" % (
                self.directory,
                query.attribute,
                query.value,
                query.matchType,
                False,
                recordType,
                attrs,
            ))
            results = opendirectory.queryRecordsWithAttribute_list(
                self.directory,
                query.attribute,
                query.value,
                query.matchType,
                False,
                recordType,
                attrs,
            )
        except opendirectory.ODError, ex:
            self.log_error("OpenDirectory (node=%s) error: %s" % (self.realmName, str(ex)))
            raise

        for (_ignore_recordShortName, value) in results:

            # Now get useful record info.
            recordGUID = value.get(dsattributes.kDS1AttrGeneratedUID)
            if recordGUID:
                guids.add(recordGUID)

        return guids

    def proxiesForGUID(self, recordType, guid):
        
        # Lookup in index
        try:
            # TODO:
            return ()
        except KeyError:
            return ()

    def readOnlyProxiesForGUID(self, recordType, guid):
        
        # Lookup in index
        try:
            # TODO:
            return ()
        except KeyError:
            return ()

    _ODFields = {
        'fullName' : {
            'odField' : dsattributes.kDS1AttrDistinguishedName,
            'appliesTo' : set([
                dsattributes.kDSStdRecordTypeUsers,
                dsattributes.kDSStdRecordTypeGroups,
                dsattributes.kDSStdRecordTypePlaces,
                dsattributes.kDSStdRecordTypeResources,
            ]),
        },
        'firstName' : {
            'odField' : dsattributes.kDS1AttrFirstName,
            'appliesTo' : set([
                dsattributes.kDSStdRecordTypeUsers,
            ]),
        },
        'lastName' : {
            'odField' : dsattributes.kDS1AttrLastName,
            'appliesTo' : set([
                dsattributes.kDSStdRecordTypeUsers,
            ]),
        },
        'emailAddresses' : {
            'odField' : dsattributes.kDSNAttrEMailAddress,
            'appliesTo' : set([
                dsattributes.kDSStdRecordTypeUsers,
                dsattributes.kDSStdRecordTypeGroups,
            ]),
        },
        'recordName' : {
            'odField' : dsattributes.kDSNAttrRecordName,
            'appliesTo' : set([
                dsattributes.kDSStdRecordTypeUsers,
                dsattributes.kDSStdRecordTypeGroups,
                dsattributes.kDSStdRecordTypePlaces,
                dsattributes.kDSStdRecordTypeResources,
            ]),
        },
        'guid' : {
            'odField' : dsattributes.kDS1AttrGeneratedUID,
            'appliesTo' : set([
                dsattributes.kDSStdRecordTypeUsers,
                dsattributes.kDSStdRecordTypeGroups,
                dsattributes.kDSStdRecordTypePlaces,
                dsattributes.kDSStdRecordTypeResources,
            ]),
        },
    }

    _toODRecordTypes = {
        DirectoryService.recordType_users :
            dsattributes.kDSStdRecordTypeUsers,
        DirectoryService.recordType_locations :
            dsattributes.kDSStdRecordTypePlaces,
        DirectoryService.recordType_groups :
            dsattributes.kDSStdRecordTypeGroups,
        DirectoryService.recordType_resources :
            dsattributes.kDSStdRecordTypeResources,
    }

    _fromODRecordTypes = dict([(b, a) for a, b in _toODRecordTypes.iteritems()])

    def _uniqueTupleFromAttribute(self, attribute):
        if attribute:
            if isinstance(attribute, str):
                return (attribute,)
            else:
                s = set()
                return tuple([(s.add(x), x)[1] for x in attribute if x not in s])
        else:
            return ()

    def _setFromAttribute(self, attribute, lower=False):
        if attribute:
            if isinstance(attribute, str):
                return set((attribute.lower() if lower else attribute,))
            else:
                return set([item.lower() if lower else item for item in attribute])
        else:
            return ()

    def recordsMatchingFields(self, fields, operand="or", recordType=None,
        lookupMethod=opendirectory.queryRecordsWithAttributes_list):

        # Note that OD applies case-sensitivity globally across the entire
        # query, not per expression, so the current code uses whatever is
        # specified in the last field in the fields list

        def collectResults(results):
            self.log_debug("Got back %d records from OD" % (len(results),))
            for key, value in results:
                self.log_debug("OD result: %s %s" % (key, value))
                try:
                    recordNodeName = value.get(
                        dsattributes.kDSNAttrMetaNodeLocation)
                    recordShortNames = self._uniqueTupleFromAttribute(
                        value.get(dsattributes.kDSNAttrRecordName))
                    if (recordNodeName == "/Local/Default" and not
                        (config.Scheduling.iMIP.Username in recordShortNames)):
                        continue

                    recordGUID = value.get(dsattributes.kDS1AttrGeneratedUID)
                    recordType = value.get(dsattributes.kDSNAttrRecordType)
                    if isinstance(recordType, list):
                        recordType = recordType[0]
                    if not recordType:
                        continue
                    recordType = self._fromODRecordTypes[recordType]

                    recordAuthIDs = self._setFromAttribute(
                        value.get(dsattributes.kDSNAttrAltSecurityIdentities))
                    recordFullName = value.get(
                        dsattributes.kDS1AttrDistinguishedName)
                    recordFirstName = value.get(dsattributes.kDS1AttrFirstName)
                    recordLastName = value.get(dsattributes.kDS1AttrLastName)
                    recordEmailAddresses = self._setFromAttribute(
                        value.get(dsattributes.kDSNAttrEMailAddress),
                        lower=True)

                    # Create records but don't store them in our index or
                    # send them to memcached, because these are transient,
                    # existing only so we can create principal resource
                    # objects that are used to generate the REPORT result.

                    record = OpenDirectoryRecord(
                        service               = self,
                        recordType            = recordType,
                        guid                  = recordGUID,
                        nodeName              = recordNodeName,
                        shortNames            = recordShortNames,
                        authIDs               = recordAuthIDs,
                        fullName              = recordFullName,
                        firstName             = recordFirstName,
                        lastName              = recordLastName,
                        emailAddresses        = recordEmailAddresses,
                        enabledForCalendaring = True,
                        memberGUIDs           = (),
                    )
                    yield record

                except KeyError:
                    pass

        def multiQuery(directory, queries, attrs, operand):
            results = []

            for query, recordTypes in queries.iteritems():
                if not query:
                    continue

                expressions = []
                for ODField, value, caseless, matchType in query:
                    if matchType == "starts-with":
                        comparison = dsattributes.eDSStartsWith
                    elif matchType == "contains":
                        comparison = dsattributes.eDSContains
                    else:
                        comparison = dsattributes.eDSExact
                    expressions.append(dsquery.match(ODField, value, comparison))

                complexExpression = dsquery.expression(operand, expressions).generate()

                self.log_debug("Calling OD: Types %s, Operand %s, Caseless %s, %s" %
                    (recordTypes, operand, caseless, complexExpression))

                results.extend(
                    lookupMethod(
                        directory,
                        complexExpression,
                        caseless,
                        recordTypes,
                        attrs,
                    )
                )

            return results


        operand = (dsquery.expression.OR if operand == "or"
            else dsquery.expression.AND)

        if recordType is None:
            # The client is looking for records in any of the four types
            recordTypes = set(self._toODRecordTypes.values())
        else:
            # The client is after only one recordType
            recordTypes = [self._toODRecordTypes[recordType]]

        queries = buildQueries(recordTypes, fields, self._ODFields)

        deferred = deferToThread(
            multiQuery,
            self.directory,
            queries,
            [
                dsattributes.kDS1AttrGeneratedUID,
                dsattributes.kDSNAttrRecordName,
                dsattributes.kDSNAttrAltSecurityIdentities,
                dsattributes.kDSNAttrRecordType,
                dsattributes.kDS1AttrDistinguishedName,
                dsattributes.kDS1AttrFirstName,
                dsattributes.kDS1AttrLastName,
                dsattributes.kDSNAttrEMailAddress,
                dsattributes.kDSNAttrMetaNodeLocation,
            ],
            operand
        )
        deferred.addCallback(collectResults)
        return deferred


    def queryDirectory(self, recordTypes, indexType, indexKey,
        lookupMethod=opendirectory.queryRecordsWithAttribute_list):
        
        attrs = [
            dsattributes.kDS1AttrGeneratedUID,
            dsattributes.kDSNAttrRecordName,
            dsattributes.kDSNAttrAltSecurityIdentities,
            dsattributes.kDSNAttrRecordType,
            dsattributes.kDS1AttrDistinguishedName,
            dsattributes.kDS1AttrFirstName,
            dsattributes.kDS1AttrLastName,
            dsattributes.kDSNAttrEMailAddress,
            dsattributes.kDSNAttrMetaNodeLocation,
        ]

        origIndexKey = indexKey
        if indexType == self.INDEX_TYPE_CUA:
            # The directory doesn't contain CUAs, so we need to convert
            # the CUA to the appropriate field name and value:
            queryattr, indexKey = cuAddressConverter(indexKey)
            # queryattr will be one of:
            # guid, emailAddresses, or recordName
            # ...which will need to be mapped to DS
            queryattr = self._ODFields[queryattr]['odField']

        else:
            queryattr = {
                self.INDEX_TYPE_SHORTNAME : dsattributes.kDSNAttrRecordName,
                self.INDEX_TYPE_GUID      : dsattributes.kDS1AttrGeneratedUID,
                self.INDEX_TYPE_AUTHID    : dsattributes.kDSNAttrAltSecurityIdentities,
            }.get(indexType)
            assert queryattr is not None, "Invalid type for record faulting query"

        query = dsquery.match(queryattr, indexKey, dsattributes.eDSExact)


        listRecordTypes = []
        for recordType in recordTypes:
            if recordType == DirectoryService.recordType_users:
                listRecordTypes.append(dsattributes.kDSStdRecordTypeUsers)
    
            elif recordType == DirectoryService.recordType_groups:
                if queryattr != dsattributes.kDSNAttrEMailAddress:
                    listRecordTypes.append(dsattributes.kDSStdRecordTypeGroups)
                    attrs.append(dsattributes.kDSNAttrGroupMembers)
                    attrs.append(dsattributes.kDSNAttrNestedGroups)
    
            elif recordType == DirectoryService.recordType_locations:
                if queryattr != dsattributes.kDSNAttrEMailAddress:
                    listRecordTypes.append(dsattributes.kDSStdRecordTypePlaces)
                # MOR: possibly can be removed
                attrs.append(dsattributes.kDSNAttrResourceInfo)
            
            elif recordType == DirectoryService.recordType_resources:
                if queryattr != dsattributes.kDSNAttrEMailAddress:
                    listRecordTypes.append(dsattributes.kDSStdRecordTypeResources)
                # MOR: possibly can be removed
                attrs.append(dsattributes.kDSNAttrResourceInfo)
            
            else:
                raise UnknownRecordTypeError("Unknown OpenDirectory record type: %s" % (recordType))


        try:
            self.log_debug("opendirectory.queryRecordsWithAttribute_list(%r,%r,%r,%r,%r,%r,%r)" % (
                self.directory,
                query.attribute,
                query.value,
                query.matchType,
                False,
                listRecordTypes,
                attrs,
            ))
            results = lookupMethod(
                self.directory,
                query.attribute,
                query.value,
                query.matchType,
                False,
                listRecordTypes,
                attrs,
            )
            self.log_debug("opendirectory.queryRecordsWithAttribute_list matched records: %s" % (len(results),))

        except opendirectory.ODError, ex:
            if ex.message[1] == -14140 or ex.message[1] == -14200:
                # Unsupported attribute on record - don't fail
                return
            else:
                self.log_error("OpenDirectory (node=%s) error: %s" % (self.realmName, str(ex)))
                raise


        enabledRecords = []
        disabledRecords = []

        for (recordShortName, value) in results:

            # Now get useful record info.
            recordGUID           = value.get(dsattributes.kDS1AttrGeneratedUID)
            recordShortNames     = self._uniqueTupleFromAttribute(value.get(dsattributes.kDSNAttrRecordName))
            recordType           = value.get(dsattributes.kDSNAttrRecordType)
            if isinstance(recordType, list):
                recordType = recordType[0]
            recordAuthIDs        = self._setFromAttribute(value.get(dsattributes.kDSNAttrAltSecurityIdentities))
            recordFullName       = value.get(dsattributes.kDS1AttrDistinguishedName)
            recordFirstName      = value.get(dsattributes.kDS1AttrFirstName)
            recordLastName       = value.get(dsattributes.kDS1AttrLastName)
            recordEmailAddresses = self._setFromAttribute(value.get(dsattributes.kDSNAttrEMailAddress), lower=True)
            recordNodeName       = value.get(dsattributes.kDSNAttrMetaNodeLocation)

            if recordNodeName == "/Local/Default" and not (config.Scheduling.iMIP.Username in recordShortNames):
                self.log_info("Local record (%s)%s is not eligible for calendaring."
                              % (recordType, recordShortName))
                continue

            if not recordType:
                self.log_debug("Record (unknown)%s in node %s has no recordType; ignoring."
                               % (recordShortName, recordNodeName))
                continue

            recordType = self._fromODRecordTypes[recordType]

            if not recordGUID:
{'fraction_non_alphanumeric': 0.07223601952556308, 'fraction_numerical': 0.0035539950329707974, 'mean_word_length': 1.7002543646664354, 'pattern_counts': {'":': 4, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '405436', 'n_tokens_mistral': 11972, 'n_tokens_neox': 11327, 'n_words': 2502}