<commit_before>/**
 * $Id: MembershipEntityProvider.java 51727 2008-09-03 09:00:03Z <EMAIL> $
 * $URL: https://source.sakaiproject.org/svn/entitybroker/trunk/impl/src/java/org/sakaiproject/entitybroker/providers/MembershipEntityProvider.java $
 * ServerConfigEntityProvider.java - entity-broker - Jul 17, 2008 2:19:03 PM - azeckoski
 **************************************************************************
 * Copyright (c) 2008 Sakai Foundation
 *
 * Licensed under the Educational Community License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.osedu.org/licenses/ECL-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.sakaiproject.entitybroker.providers;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.sakaiproject.authz.api.Member;
import org.sakaiproject.email.api.EmailService;
import org.sakaiproject.entitybroker.EntityReference;
import org.sakaiproject.entitybroker.EntityView;
import org.sakaiproject.entitybroker.entityprovider.CoreEntityProvider;
import org.sakaiproject.entitybroker.entityprovider.annotations.EntityCustomAction;
import org.sakaiproject.entitybroker.entityprovider.capabilities.ActionsExecutable;
import org.sakaiproject.entitybroker.entityprovider.capabilities.CollectionResolvable;
import org.sakaiproject.entitybroker.entityprovider.capabilities.RESTful;
import org.sakaiproject.entitybroker.entityprovider.extension.ActionReturn;
import org.sakaiproject.entitybroker.entityprovider.extension.EntityData;
import org.sakaiproject.entitybroker.entityprovider.extension.Formats;
import org.sakaiproject.entitybroker.entityprovider.search.Order;
import org.sakaiproject.entitybroker.entityprovider.search.Restriction;
import org.sakaiproject.entitybroker.entityprovider.search.Search;
import org.sakaiproject.entitybroker.providers.model.EntityMember;
import org.sakaiproject.entitybroker.providers.model.EntityUser;
import org.sakaiproject.entitybroker.util.AbstractEntityProvider;
import org.sakaiproject.exception.IdUnusedException;
import org.sakaiproject.exception.PermissionException;
import org.sakaiproject.site.api.Group;
import org.sakaiproject.site.api.Site;
import org.sakaiproject.site.api.SiteService;
import org.sakaiproject.site.api.SiteService.SelectionType;


/**
 * This provides access to memberships as entities
 * 
 * @author <NAME> (<EMAIL>)
 */
public class MembershipEntityProvider extends AbstractEntityProvider implements CoreEntityProvider, RESTful, ActionsExecutable {

    private static Log log = LogFactory.getLog(MembershipEntityProvider.class);

    private SiteService siteService;
    public void setSiteService(SiteService siteService) {
        this.siteService = siteService;
    }

    private UserEntityProvider userEntityProvider;
    public void setUserEntityProvider(UserEntityProvider userEntityProvider) {
        this.userEntityProvider = userEntityProvider;
    }
    
    private EmailService emailService;
    public void setEmailService(EmailService emailService) {
        this.emailService = emailService;
    }

    public static String PREFIX = "membership";
    public String getEntityPrefix() {
        return PREFIX;
    }
    
    /**
     * Handle the special needs of UX site membership settings, either getting the current
     * list of site memberships via a GET request, or creating a new batch of site memberships
     * via a POST request. In the case of a POST, special HTTP response headers will be
     * used to communicate success or warning conditions to the client.
     * @param view
     * @param params
     * @return
     */
    @EntityCustomAction(action="site",viewKey="")
    public ActionReturn handleSiteMemberships(EntityView view, Map<String, Object> params) {
        if (log.isDebugEnabled()) log.debug("handleSiteMemberships method=" + view.getMethod() + ", params=" + params);
        String siteId = view.getPathSegment(2);
        if (siteId == null) {
            siteId = (String) params.get("siteId");
            if (siteId == null) {
                throw new IllegalArgumentException("siteId must be set in order to get site memberships, set in params or in the URL /membership/site/siteId");
            }
        }
        String locationReference = "/site/" + siteId;
        
        Map<String, String> extraResponseHeaders = null;
        if (EntityView.Method.POST.name().equals(view.getMethod())) {
            extraResponseHeaders = createBatchMemberships(view, params, locationReference);
        }
        
        List<EntityData> l = getEntities(new EntityReference(PREFIX,""), 
                new Search(CollectionResolvable.SEARCH_LOCATION_REFERENCE, locationReference));
        ActionReturn actionReturn = new ActionReturn(l, Formats.JSON);
        if ((extraResponseHeaders != null) && !extraResponseHeaders.isEmpty()) {
            actionReturn.setHeaders(extraResponseHeaders);
        }
        return actionReturn;
    }

    /**
     * Add members to a site.
     * @param view
     * @param params request parameters including a list of userSearchValues
     * @param locationReference
     * @return headers containing success or warning messages for the client
     */
    public Map<String, String> createBatchMemberships(EntityView view, Map<String, Object> params, String locationReference) {
        SiteGroup sg = findLocationByReference(locationReference);
        String roleId = (String)params.get("memberRole");
        String notificationMessage = (String)params.get("notificationMessage");
        if ((notificationMessage != null) && (notificationMessage.trim().length() == 0)) {
            notificationMessage = null;
        }
        boolean active = true;
        
        Map<String, String> responseHeaders = new HashMap<String, String>();
        Set<EntityUser> users = new HashSet<EntityUser>();
        Set<String> valuesNotFound = new HashSet<String>();
        Set<String> valuesAlreadyMembers = new HashSet<String>();
        List<String> userSearchValues = getListFromValue(params.get("userSearchValues"));
        for (String userSearchValue : userSearchValues) {
            EntityUser user = userEntityProvider.findUserFromSearchValue(userSearchValue);
            if (user != null) {
                if (sg.site.getUserRole(user.getId()) != null) {
                    valuesAlreadyMembers.add(userSearchValue);
                } else {
                    users.add(user);
                }
            } else {
                valuesNotFound.add(userSearchValue);
            }
        }
        
        if (!users.isEmpty()) {
            for (EntityUser user : users) {
                sg.site.addMember(user.getId(), roleId, active, false);
                if (notificationMessage != null) {
                    /**
                     * TODO Should the From address be the site contact or the "setup.request" Sakai property?
                     * TODO We need to retrieve a localized message title and additional body (if any) instead of hard-coding it.
                     * TODO Any special boilerplate to include site URL or title or description?
                     */
                    
                    emailService.send("<EMAIL>", user.getEmail(), 
                        "New Site Membership Notification", notificationMessage, null, null, null);
                }
            }
            saveSiteMembership(sg.site);
            responseHeaders.put("x-success-count", String.valueOf(users.size()));
        }        
        if (!valuesNotFound.isEmpty()) {
            Iterator<String> listIter = valuesNotFound.iterator();
            StringBuilder listString = new StringBuilder(listIter.next());
            while (listIter.hasNext()) {
                listString.append(", ").append(listIter.next());
            }
            responseHeaders.put("x-warning-not-found", listString.toString());
        }
        if (!valuesAlreadyMembers.isEmpty()) {
            Iterator<String> listIter = valuesAlreadyMembers.iterator();
            StringBuilder listString = new StringBuilder(listIter.next());
            while (listIter.hasNext()) {
                listString.append(", ").append(listIter.next());
            }
            responseHeaders.put("x-warning-already-members", listString.toString());
        }
        return responseHeaders;
    }

    @EntityCustomAction(action="group",viewKey=EntityView.VIEW_LIST)
    public List<?> getGroupMemberships(EntityView view, Map<String, Object> params) {
        String groupId = view.getPathSegment(2);
        if (groupId == null) {
            groupId = (String) params.get("groupId");
            if (groupId == null) {
                throw new IllegalArgumentException("groupId must be set in order to get group memberships, set in params or in the URL /membership/group/groupId");
            }
        }
        List<?> l = getEntities(new EntityReference(PREFIX,""), 
                new Search(CollectionResolvable.SEARCH_LOCATION_REFERENCE,"/group/" + groupId));
        return l;
    }

    public boolean entityExists(String id) {
        if (id == null) {
            return false;
        }
        if ("".equals(id)) {
            return true;
        }
        String[] parts = EntityMember.parseId(id);
        if (parts != null) {
            // TODO check it later when there is an efficient way
            return true;
        }
        return false;
    }

    public Object getEntity(EntityReference ref) {
        if (ref.getId() == null) {
            return new EntityMember();
        }
        String mid = ref.getId();
        String[] parts = EntityMember.parseId(mid);
        if (parts == null) {
            throw new IllegalArgumentException("Invalid membership id (" + mid + "), should be formed like so: 'userId::site:siteId' or 'userId::group:groupId");
        }
        EntityMember member = getMember(parts[0], parts[1]);
        if (member == null) {
            throw new IllegalArgumentException("Cannot find membership with id: " + mid);
        }
        return member;
    }

    /**
     * Gets the list of all memberships for the current user if no params provided,
     * otherwise gets memberships in a specified location or for a specified user
     */
    public List<EntityData> getEntities(EntityReference ref, Search search) {
        String currentUserId = developerHelperService.getCurrentUserId();
        String userId = null;
        String locationReference = null;
        String roleId = null;
        boolean includeSites = true;
        boolean includeGroups = false;
        if (search == null) {
            search = new Search();
        }
        if (! search.isEmpty()) {
            // process the search
            roleId = (String) search.getRestrictionValueByProperties(new String[] {"role","roleId"});
            Restriction userRes = search.getRestrictionByProperty(CollectionResolvable.SEARCH_USER_REFERENCE);
            if (userRes != null) {
                String userRef = userRes.getStringValue();
                userId = EntityReference.getIdFromRef(userRef);
            }
            Restriction locRes = search.getRestrictionByProperty(CollectionResolvable.SEARCH_LOCATION_REFERENCE);
            if (locRes != null) {
                locationReference = locRes.getStringValue();
            }
            Restriction incSites = search.getRestrictionByProperty("includeSites");
            if (incSites != null) {
                includeSites = incSites.getBooleanValue();
            }
            Restriction incGroups = search.getRestrictionByProperty("includeGroups");
            if (incGroups != null) {
                includeGroups = incGroups.getBooleanValue();
            }
        }
        if (locationReference == null && userId == null) {
            // if these are both nul then we default to getting memberships for the current user
            if (currentUserId != null) {
                userId = currentUserId;
            }
        }
        if (locationReference == null && userId == null) {
            // fail if there is still nothing to output
            throw new IllegalArgumentException("There must be a current user logged in " +
                    "OR you must provide a search with the following restrictions (getting all is not supported): " +
                    "siteId, locationReference, groupId AND (optionally) roleId OR userReference, userId, user");
        }
        List<EntityMember> members = new ArrayList<EntityMember>();
        boolean findByLocation = false;
        if (locationReference != null) {
            // get membership for a location
            findByLocation = true;
            members = getMembers(locationReference);
        } else {
            // get membership for 
            if (!includeGroups && !includeSites) {
                throw new IllegalArgumentException("includesSites and includesGroups cannot both be false");
            }
            // find memberships by userId
            userId = userEntityProvider.findAndCheckUserId(userId, null);
            boolean userCurrent = userId.equals(currentUserId);
            // Is there a faster way to do this? I really truly hope so -AZ
            try {
                if (!userCurrent) {
                    developerHelperService.setCurrentUser("/user/" + userId);
                }
                List<Site> sites = siteService.getSites(SelectionType.ACCESS, null, null, null, null, null);
                for (Site site : sites) {
                    Member sm = site.getMember(userId);
                    if (sm != null) {
                        if (includeSites) {
                            members.add( new EntityMember(sm, site.getReference(), null) );
                        }
                        // also check the groups
                        if (includeGroups) {
                            Collection<Group> groups = site.getGroups();
                            for (Group group : groups) {
                                Member gm = group.getMember(userId);
                                if (gm != null) {
                                    members.add( new EntityMember(gm, group.getReference(), null) );
                                }
                            }
                        }
                    }
                }<commit_msg>SAK-14882 Need admin permissions for membership fetch<commit_after>/**
 * $Id: MembershipEntityProvider.java 51727 2008-09-03 09:00:03Z <EMAIL> $
 * $URL: https://source.sakaiproject.org/svn/entitybroker/trunk/impl/src/java/org/sakaiproject/entitybroker/providers/MembershipEntityProvider.java $
 * ServerConfigEntityProvider.java - entity-broker - Jul 17, 2008 2:19:03 PM - azeckoski
 **************************************************************************
 * Copyright (c) 2008 Sakai Foundation
 *
 * Licensed under the Educational Community License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.osedu.org/licenses/ECL-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.sakaiproject.entitybroker.providers;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.sakaiproject.authz.api.Member;
import org.sakaiproject.email.api.EmailService;
import org.sakaiproject.entitybroker.EntityReference;
import org.sakaiproject.entitybroker.EntityView;
import org.sakaiproject.entitybroker.entityprovider.CoreEntityProvider;
import org.sakaiproject.entitybroker.entityprovider.annotations.EntityCustomAction;
import org.sakaiproject.entitybroker.entityprovider.capabilities.ActionsExecutable;
import org.sakaiproject.entitybroker.entityprovider.capabilities.CollectionResolvable;
import org.sakaiproject.entitybroker.entityprovider.capabilities.RESTful;
import org.sakaiproject.entitybroker.entityprovider.extension.ActionReturn;
import org.sakaiproject.entitybroker.entityprovider.extension.EntityData;
import org.sakaiproject.entitybroker.entityprovider.extension.Formats;
import org.sakaiproject.entitybroker.entityprovider.search.Order;
import org.sakaiproject.entitybroker.entityprovider.search.Restriction;
import org.sakaiproject.entitybroker.entityprovider.search.Search;
import org.sakaiproject.entitybroker.providers.model.EntityMember;
import org.sakaiproject.entitybroker.providers.model.EntityUser;
import org.sakaiproject.entitybroker.util.AbstractEntityProvider;
import org.sakaiproject.exception.IdUnusedException;
import org.sakaiproject.exception.PermissionException;
import org.sakaiproject.site.api.Group;
import org.sakaiproject.site.api.Site;
import org.sakaiproject.site.api.SiteService;
import org.sakaiproject.site.api.SiteService.SelectionType;


/**
 * This provides access to memberships as entities
 * 
 * @author <NAME> (<EMAIL>)
 */
public class MembershipEntityProvider extends AbstractEntityProvider implements CoreEntityProvider, RESTful, ActionsExecutable {

    private static Log log = LogFactory.getLog(MembershipEntityProvider.class);

    private SiteService siteService;
    public void setSiteService(SiteService siteService) {
        this.siteService = siteService;
    }

    private UserEntityProvider userEntityProvider;
    public void setUserEntityProvider(UserEntityProvider userEntityProvider) {
        this.userEntityProvider = userEntityProvider;
    }
    
    private EmailService emailService;
    public void setEmailService(EmailService emailService) {
        this.emailService = emailService;
    }

    public static String PREFIX = "membership";
    public String getEntityPrefix() {
        return PREFIX;
    }
    
    /**
     * Handle the special needs of UX site membership settings, either getting the current
     * list of site memberships via a GET request, or creating a new batch of site memberships
     * via a POST request. In the case of a POST, special HTTP response headers will be
     * used to communicate success or warning conditions to the client.
     * @param view
     * @param params
     * @return
     */
    @EntityCustomAction(action="site",viewKey="")
    public ActionReturn handleSiteMemberships(EntityView view, Map<String, Object> params) {
        if (log.isDebugEnabled()) log.debug("handleSiteMemberships method=" + view.getMethod() + ", params=" + params);
        String siteId = view.getPathSegment(2);
        if (siteId == null) {
            siteId = (String) params.get("siteId");
            if (siteId == null) {
                throw new IllegalArgumentException("siteId must be set in order to get site memberships, set in params or in the URL /membership/site/siteId");
            }
        }
        String locationReference = "/site/" + siteId;
        
        Map<String, String> extraResponseHeaders = null;
        if (EntityView.Method.POST.name().equals(view.getMethod())) {
            extraResponseHeaders = createBatchMemberships(view, params, locationReference);
        }
        
        List<EntityData> l = getEntities(new EntityReference(PREFIX,""), 
                new Search(CollectionResolvable.SEARCH_LOCATION_REFERENCE, locationReference));
        ActionReturn actionReturn = new ActionReturn(l, Formats.JSON);
        if ((extraResponseHeaders != null) && !extraResponseHeaders.isEmpty()) {
            actionReturn.setHeaders(extraResponseHeaders);
        }
        return actionReturn;
    }

    /**
     * Add members to a site.
     * @param view
     * @param params request parameters including a list of userSearchValues
     * @param locationReference
     * @return headers containing success or warning messages for the client
     */
    public Map<String, String> createBatchMemberships(EntityView view, Map<String, Object> params, String locationReference) {
        SiteGroup sg = findLocationByReference(locationReference);
        String roleId = (String)params.get("memberRole");
        String notificationMessage = (String)params.get("notificationMessage");
        if ((notificationMessage != null) && (notificationMessage.trim().length() == 0)) {
            notificationMessage = null;
        }
        boolean active = true;
        
        Map<String, String> responseHeaders = new HashMap<String, String>();
        Set<EntityUser> users = new HashSet<EntityUser>();
        Set<String> valuesNotFound = new HashSet<String>();
        Set<String> valuesAlreadyMembers = new HashSet<String>();
        List<String> userSearchValues = getListFromValue(params.get("userSearchValues"));
        for (String userSearchValue : userSearchValues) {
            EntityUser user = userEntityProvider.findUserFromSearchValue(userSearchValue);
            if (user != null) {
                if (sg.site.getUserRole(user.getId()) != null) {
                    valuesAlreadyMembers.add(userSearchValue);
                } else {
                    users.add(user);
                }
            } else {
                valuesNotFound.add(userSearchValue);
            }
        }
        
        if (!users.isEmpty()) {
            for (EntityUser user : users) {
                sg.site.addMember(user.getId(), roleId, active, false);
                if (notificationMessage != null) {
                    /**
                     * TODO Should the From address be the site contact or the "setup.request" Sakai property?
                     * TODO We need to retrieve a localized message title and additional body (if any) instead of hard-coding it.
                     * TODO Any special boilerplate to include site URL or title or description?
                     */
                    
                    emailService.send("<EMAIL>", user.getEmail(), 
                        "New Site Membership Notification", notificationMessage, null, null, null);
                }
            }
            saveSiteMembership(sg.site);
            responseHeaders.put("x-success-count", String.valueOf(users.size()));
        }        
        if (!valuesNotFound.isEmpty()) {
            Iterator<String> listIter = valuesNotFound.iterator();
            StringBuilder listString = new StringBuilder(listIter.next());
            while (listIter.hasNext()) {
                listString.append(", ").append(listIter.next());
            }
            responseHeaders.put("x-warning-not-found", listString.toString());
        }
        if (!valuesAlreadyMembers.isEmpty()) {
            Iterator<String> listIter = valuesAlreadyMembers.iterator();
            StringBuilder listString = new StringBuilder(listIter.next());
            while (listIter.hasNext()) {
                listString.append(", ").append(listIter.next());
            }
            responseHeaders.put("x-warning-already-members", listString.toString());
        }
        return responseHeaders;
    }

    @EntityCustomAction(action="group",viewKey=EntityView.VIEW_LIST)
    public List<?> getGroupMemberships(EntityView view, Map<String, Object> params) {
        String groupId = view.getPathSegment(2);
        if (groupId == null) {
            groupId = (String) params.get("groupId");
            if (groupId == null) {
                throw new IllegalArgumentException("groupId must be set in order to get group memberships, set in params or in the URL /membership/group/groupId");
            }
        }
        List<?> l = getEntities(new EntityReference(PREFIX,""), 
                new Search(CollectionResolvable.SEARCH_LOCATION_REFERENCE,"/group/" + groupId));
        return l;
    }

    public boolean entityExists(String id) {
        if (id == null) {
            return false;
        }
        if ("".equals(id)) {
            return true;
        }
        String[] parts = EntityMember.parseId(id);
        if (parts != null) {
            // TODO check it later when there is an efficient way
            return true;
        }
        return false;
    }

    public Object getEntity(EntityReference ref) {
        if (ref.getId() == null) {
            return new EntityMember();
        }
        String mid = ref.getId();
        String[] parts = EntityMember.parseId(mid);
        if (parts == null) {
            throw new IllegalArgumentException("Invalid membership id (" + mid + "), should be formed like so: 'userId::site:siteId' or 'userId::group:groupId");
        }
        EntityMember member = getMember(parts[0], parts[1]);
        if (member == null) {
            throw new IllegalArgumentException("Cannot find membership with id: " + mid);
        }
        return member;
    }

    /**
     * Gets the list of all memberships for the current user if no params provided,
     * otherwise gets memberships in a specified location or for a specified user
     */
    public List<EntityData> getEntities(EntityReference ref, Search search) {
        String currentUserId = developerHelperService.getCurrentUserId();
        String userId = null;
        String locationReference = null;
        String roleId = null;
        boolean includeSites = true;
        boolean includeGroups = false;
        if (search == null) {
            search = new Search();
        }
        if (! search.isEmpty()) {
            // process the search
            roleId = (String) search.getRestrictionValueByProperties(new String[] {"role","roleId"});
            Restriction userRes = search.getRestrictionByProperty(CollectionResolvable.SEARCH_USER_REFERENCE);
            if (userRes != null) {
                String userRef = userRes.getStringValue();
                userId = EntityReference.getIdFromRef(userRef);
            }
            Restriction locRes = search.getRestrictionByProperty(CollectionResolvable.SEARCH_LOCATION_REFERENCE);
            if (locRes != null) {
                locationReference = locRes.getStringValue();
            }
            Restriction incSites = search.getRestrictionByProperty("includeSites");
            if (incSites != null) {
                includeSites = incSites.getBooleanValue();
            }
            Restriction incGroups = search.getRestrictionByProperty("includeGroups");
            if (incGroups != null) {
                includeGroups = incGroups.getBooleanValue();
            }
        }
        if (locationReference == null && userId == null) {
            // if these are both nul then we default to getting memberships for the current user
            if (currentUserId != null) {
                userId = currentUserId;
            }
        }
        if (locationReference == null && userId == null) {
            // fail if there is still nothing to output
            throw new IllegalArgumentException("There must be a current user logged in " +
                    "OR you must provide a search with the following restrictions (getting all is not supported): " +
                    "siteId, locationReference, groupId AND (optionally) roleId OR userReference, userId, user");
        }
        List<EntityMember> members = new ArrayList<EntityMember>();
        boolean findByLocation = false;
        if (locationReference != null) {
            // get membership for a location
            findByLocation = true;
            members = getMembers(locationReference);
        } else {
            // get membership for 
            if (!includeGroups && !includeSites) {
                throw new IllegalArgumentException("includesSites and includesGroups cannot both be false");
            }
            // find memberships by userId
            userId = userEntityProvider.findAndCheckUserId(userId, null);
            boolean userCurrent = userId.equals(currentUserId);
            if (!userCurrent && !developerHelperService.isUserAdmin(currentUserId)) {
                throw new SecurityException("Only admin can access other user memberships, current user ("+currentUserId+") cannot access ref: " + userId);
            }
            
            // Is there a faster way to do this? I really truly hope so -AZ
            try {
                if (!userCurrent) {
                    developerHelperService.setCurrentUser("/user/" + userId);
                }
                List<Site> sites = siteService.getSites(SelectionType.ACCESS, null, null, null, null, null);
                for (Site site : sites) {
                    Member sm = site.getMember(userId);
                    if (sm != null) {
                        if (includeSites) {
                            members.add( new EntityMember(sm, site.getReference(), null) );
                        }
                        // also check the groups
                        if (includeGroups) {
                            Collection<Group> groups = site.getGroups();
                            for (Group group : groups) {
                                Member gm = group.getMember(userId);
                                if (gm != null) {
                                    members.add( new EntityMember(gm, group.getReference(), null) );
                                }
                            }
                        }
                    }
                }
{'fraction_non_alphanumeric': 0.09744687860319552, 'fraction_numerical': 0.0029978586723768737, 'mean_word_length': 2.4107865168539324, 'pattern_counts': {'":': 0, '<': 59, '<?xml version=': 0, '>': 59, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6966833', 'n_tokens_mistral': 7721, 'n_tokens_neox': 7443, 'n_words': 2018}