<commit_before>package org.sagebionetworks.crowd.workers;

import java.io.ByteArrayInputStream;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;

import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathExpressionException;
import javax.xml.xpath.XPathFactory;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.sagebionetworks.authutil.AuthenticationException;
import org.sagebionetworks.authutil.CrowdAuthUtil;
import org.sagebionetworks.repo.model.ConflictingUpdateException;
import org.sagebionetworks.repo.model.DatastoreException;
import org.sagebionetworks.repo.model.GroupMembersDAO;
import org.sagebionetworks.repo.model.InvalidModelException;
import org.sagebionetworks.repo.model.User;
import org.sagebionetworks.repo.model.UserDAO;
import org.sagebionetworks.repo.model.UserGroup;
import org.sagebionetworks.repo.model.UserGroupDAO;
import org.sagebionetworks.repo.model.UserProfile;
import org.sagebionetworks.repo.model.UserProfileDAO;
import org.sagebionetworks.repo.web.NotFoundException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpStatus;
import org.w3c.dom.NodeList;
import org.xml.sax.InputSource;


/**
 * Synchronizes Crowd and RDS on groups and users.
 */
public class CrowdGroupSynchronizer implements Runnable {

	@Autowired
	private GroupMembersDAO crowdGroupMembersDAO;
	
	@Autowired
	private GroupMembersDAO groupMembersDAO;
	
	@Autowired
	private UserGroupDAO userGroupDAO;
	
	@Autowired
	private UserProfileDAO userProfileDAO;
	
	@Autowired
	private UserDAO userDAOImpl; // the CrowdUserDAO
	
	private static Log log = LogFactory.getLog(CrowdGroupSynchronizer.class); 
	
	@Override
	public void run() {
		List<String> crowdGroups = getCrowdGroups("group");
		List<String> crowdUsers = getCrowdGroups("user");
		List<String> principalIds = new ArrayList<String>();
		
		// Make sure all the Crowd groups exist in RDS
		for (int i = 0; i < crowdGroups.size() + crowdUsers.size(); i++) {
			boolean isIndividual = i >= crowdGroups.size();
			String name = isIndividual ? crowdUsers.get(i - crowdGroups.size()) : crowdGroups.get(i);
			String principalId;
			
			// Make sure a UserGroup exists for each user/group 
			if (userGroupDAO.doesPrincipalExist(name)) {
				principalId = userGroupDAO.findGroup(name, isIndividual).getId();
			} else {
				UserGroup ug = new UserGroup();
				ug.setIsIndividual(isIndividual);
				ug.setName(name);
				principalId = userGroupDAO.create(ug);
			}
			
			// Make sure a profile exists for each user
			if (isIndividual) {
				User user;
				try {
					user = userDAOImpl.getUser(URLEncoder.encode(name, "UTF-8"));
				} catch (DatastoreException e) {
					throw new RuntimeException(e);
				} catch (UnsupportedEncodingException e) {
					throw new RuntimeException(e);
				} catch (NotFoundException e) {
					throw new RuntimeException(e);
				}

				UserProfile userProfile;
				try {
					userProfile = userProfileDAO.get(principalId);
					
				} catch (NotFoundException e) {
					// Must make a new profile
					try {
						userProfile = new UserProfile();
						userProfile.setOwnerId(principalId);
						userProfile.setFirstName(user.getFname());
						userProfile.setLastName(user.getLname());
						userProfile.setDisplayName(user.getDisplayName());
						userProfileDAO.create(userProfile);
					} catch (InvalidModelException ime) {
						throw new RuntimeException(ime);
					}
				}
				
				// For profiles that already exist, migrate the boolean for the termsOfUse over
				long termsTimeStamp;
				if (user.isAgreesToTermsOfUse() && user.getCreationDate() != null) {
					termsTimeStamp = user.getCreationDate().getTime() / 1000;
				} else {
					termsTimeStamp = 0L;
				}
				
				// Don't needlessly re-update the profile with the same data
				if (userProfile.getAgreesToTermsOfUse() == null 
						|| userProfile.getAgreesToTermsOfUse().longValue() != termsTimeStamp) {
					userProfile.setAgreesToTermsOfUse(termsTimeStamp);
					try {
						userProfileDAO.update(userProfile);
					} catch (DatastoreException e) {
						throw new RuntimeException(e);
					} catch (InvalidModelException e) {
						throw new RuntimeException(e);
					} catch (ConflictingUpdateException e) {
						throw new RuntimeException(e);
					} catch (NotFoundException e) {
						throw new RuntimeException(e);
					}<commit_msg>Must fetch before update on E-tagged objects<commit_after>package org.sagebionetworks.crowd.workers;

import java.io.ByteArrayInputStream;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;

import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathExpressionException;
import javax.xml.xpath.XPathFactory;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.sagebionetworks.authutil.AuthenticationException;
import org.sagebionetworks.authutil.CrowdAuthUtil;
import org.sagebionetworks.repo.model.ConflictingUpdateException;
import org.sagebionetworks.repo.model.DatastoreException;
import org.sagebionetworks.repo.model.GroupMembersDAO;
import org.sagebionetworks.repo.model.InvalidModelException;
import org.sagebionetworks.repo.model.User;
import org.sagebionetworks.repo.model.UserDAO;
import org.sagebionetworks.repo.model.UserGroup;
import org.sagebionetworks.repo.model.UserGroupDAO;
import org.sagebionetworks.repo.model.UserProfile;
import org.sagebionetworks.repo.model.UserProfileDAO;
import org.sagebionetworks.repo.web.NotFoundException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpStatus;
import org.w3c.dom.NodeList;
import org.xml.sax.InputSource;


/**
 * Synchronizes Crowd and RDS on groups and users.
 */
public class CrowdGroupSynchronizer implements Runnable {

	@Autowired
	private GroupMembersDAO crowdGroupMembersDAO;
	
	@Autowired
	private GroupMembersDAO groupMembersDAO;
	
	@Autowired
	private UserGroupDAO userGroupDAO;
	
	@Autowired
	private UserProfileDAO userProfileDAO;
	
	@Autowired
	private UserDAO userDAOImpl; // the CrowdUserDAO
	
	private static Log log = LogFactory.getLog(CrowdGroupSynchronizer.class); 
	
	@Override
	public void run() {
		List<String> crowdGroups = getCrowdGroups("group");
		List<String> crowdUsers = getCrowdGroups("user");
		List<String> principalIds = new ArrayList<String>();
		
		// Make sure all the Crowd groups exist in RDS
		for (int i = 0; i < crowdGroups.size() + crowdUsers.size(); i++) {
			boolean isIndividual = i >= crowdGroups.size();
			String name = isIndividual ? crowdUsers.get(i - crowdGroups.size()) : crowdGroups.get(i);
			String principalId;
			
			// Make sure a UserGroup exists for each user/group 
			if (userGroupDAO.doesPrincipalExist(name)) {
				principalId = userGroupDAO.findGroup(name, isIndividual).getId();
			} else {
				UserGroup ug = new UserGroup();
				ug.setIsIndividual(isIndividual);
				ug.setName(name);
				principalId = userGroupDAO.create(ug);
			}
			
			// Make sure a profile exists for each user
			if (isIndividual) {
				User user;
				try {
					user = userDAOImpl.getUser(URLEncoder.encode(name, "UTF-8"));
				} catch (DatastoreException e) {
					throw new RuntimeException(e);
				} catch (UnsupportedEncodingException e) {
					throw new RuntimeException(e);
				} catch (NotFoundException e) {
					throw new RuntimeException(e);
				}

				UserProfile userProfile;
				try {
					userProfile = userProfileDAO.get(principalId);
				} catch (NotFoundException e) {
					// Must make a new profile
					try {
						userProfile = new UserProfile();
						userProfile.setOwnerId(principalId);
						userProfile.setFirstName(user.getFname());
						userProfile.setLastName(user.getLname());
						userProfile.setDisplayName(user.getDisplayName());
						userProfileDAO.create(userProfile);
					} catch (InvalidModelException ime) {
						throw new RuntimeException(ime);
					}
					
					// Re-fetch the profile after making it
					try {
						userProfile = userProfileDAO.get(principalId);
					} catch (NotFoundException nfe) {
						throw new RuntimeException(nfe);
					}
				}
				
				// For profiles that already exist, migrate the boolean for the termsOfUse over
				long termsTimeStamp;
				if (user.isAgreesToTermsOfUse() && user.getCreationDate() != null) {
					termsTimeStamp = user.getCreationDate().getTime() / 1000;
				} else {
					termsTimeStamp = 0L;
				}
				
				// Don't needlessly re-update the profile with the same data
				if (userProfile.getAgreesToTermsOfUse() == null 
						|| userProfile.getAgreesToTermsOfUse().longValue() != termsTimeStamp) {
					userProfile.setAgreesToTermsOfUse(termsTimeStamp);
					try {
						userProfileDAO.update(userProfile);
					} catch (DatastoreException e) {
						throw new RuntimeException(e);
					} catch (InvalidModelException e) {
						throw new RuntimeException(e);
					} catch (ConflictingUpdateException e) {
						throw new RuntimeException(e);
					} catch (NotFoundException e) {
						throw new RuntimeException(e);
					}
{'fraction_non_alphanumeric': 0.09924072291733504, 'fraction_numerical': 0.0017110469468506041, 'mean_word_length': 4.107591480065538, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 10}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7155460', 'n_tokens_mistral': 3485, 'n_tokens_neox': 2906, 'n_words': 609}