<commit_before>/**
 * 
 */
package com.thinkbiganalytics.metadata.modeshape;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.regex.Matcher;
import java.util.stream.Collectors;

import javax.inject.Inject;
import javax.jcr.Node;
import javax.jcr.NodeIterator;
import javax.jcr.PropertyType;
import javax.jcr.RepositoryException;
import javax.jcr.Session;
import javax.jcr.Value;
import javax.jcr.nodetype.NodeType;
import javax.jcr.nodetype.NodeTypeIterator;
import javax.jcr.nodetype.NodeTypeTemplate;
import javax.jcr.nodetype.PropertyDefinition;
import javax.jcr.nodetype.PropertyDefinitionTemplate;
import javax.jcr.security.Privilege;
import javax.jcr.version.Version;
import javax.jcr.version.VersionHistory;
import javax.jcr.version.VersionIterator;
import javax.jcr.version.VersionManager;

import org.apache.tika.metadata.Property;
import org.modeshape.jcr.api.nodetype.NodeTypeManager;
import org.modeshape.jcr.security.SimplePrincipal;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.core.annotation.AnnotationAwareOrderComparator;

import com.thinkbiganalytics.metadata.api.MetadataAccess;
import com.thinkbiganalytics.metadata.api.PostMetadataConfigAction;
import com.thinkbiganalytics.metadata.modeshape.common.SecurityPaths;
import com.thinkbiganalytics.metadata.modeshape.extension.ExtensionsConstants;
import com.thinkbiganalytics.metadata.modeshape.security.AdminCredentials;
import com.thinkbiganalytics.metadata.modeshape.security.JcrAccessControlUtil;
import com.thinkbiganalytics.metadata.modeshape.security.ModeShapeAdminPrincipal;
import com.thinkbiganalytics.metadata.modeshape.support.JcrUtil;

/**
 *
 * @author <NAME>
 */
public class MetadataJcrConfigurator {
    
    private static final Logger log = LoggerFactory.getLogger(MetadataJcrConfigurator.class);
    
    @Inject
    private MetadataAccess metadataAccess;
    
    private List<PostMetadataConfigAction> postConfigActions = new ArrayList<>();
    
    public MetadataJcrConfigurator(List<PostMetadataConfigAction> actions) {
        this.postConfigActions.addAll(actions);
        this.postConfigActions.sort(new AnnotationAwareOrderComparator());
    }
    
    private final AtomicBoolean configured = new AtomicBoolean(false);
    
    
    public void configure() {
        this.metadataAccess.commit(() -> {
            try {
                Session session = JcrMetadataAccess.getActiveSession();
                
                ensureLayout(session);
                ensureTypes(session);
                ensureAccessControl(session);
            } catch (RepositoryException e) {
                throw new MetadataRepositoryException("Could not create initial JCR metadata", e);
            }
        }, MetadataAccess.SERVICE);
        
        this.metadataAccess.commit(() -> {
            try {
                Session session = JcrMetadataAccess.getActiveSession();
                
                removeVersionableFeedType(session);
            } catch (RepositoryException e) {
                throw new MetadataRepositoryException("Could remove versioning from feeds", e);
            }
        }, MetadataAccess.SERVICE);
        
        this.configured.set(true);
        firePostConfigActions();
    }
    
    private void removeVersionableFeedType(Session session) throws RepositoryException {
        Node feedsNode = session.getRootNode().getNode("metadata/feeds");
        
        for (Node catNode : JcrUtil.getNodesOfType(feedsNode, "tba:category")) {
            for (Node feedNode : JcrUtil.getNodesOfType(catNode, "tba:feed")) {
                log.info("Removing prior versions of feed: {}.{}", catNode.getName(), feedNode.getName());
                
                VersionManager versionManager = session.getWorkspace().getVersionManager(); 
                VersionHistory versionHistory = versionManager.getVersionHistory(feedNode.getPath()); 
                VersionIterator vIt = versionHistory.getAllVersions(); 
                int count = 0;
                String last = "";
                
                while (vIt.hasNext()) { 
                    Version version = vIt.nextVersion(); 
                    if (!"jcr:rootVersion".equals(version.getName())) { 
                        last = version.getName();
                        // removeVersion writes directly to workspace, no session.save is necessary 
                        versionHistory.removeVersion(version.getName()); 
                        count++;
                    } 
                } 
                
                if (count > 0) {
                    log.info("Removed {} versions through {} of feed {}", count, last, feedNode.getName());
                } else {
                    log.info("Feed {} had no versions", feedNode.getName());
                }
            }
            
            NodeTypeManager typeMgr = (NodeTypeManager) session.getWorkspace().getNodeTypeManager();
            NodeType currentFeedType = typeMgr.getNodeType("tba:feed");
            List<String> currentSupertypes = Arrays.asList(currentFeedType.getDeclaredSupertypeNames());
            
            if (currentSupertypes.contains("mix:versionable")) {
                List<Node> currentFeeds = JcrUtil.getNodesOfType(catNode, "tba:feed");
                NodeTypeTemplate template = typeMgr.createNodeTypeTemplate(currentFeedType);
                
                List<String> newSupertypes = currentSupertypes.stream().filter(type -> ! type.equals("mix:versionable")).collect(Collectors.toList());
                template.setDeclaredSuperTypeNames(newSupertypes.toArray(new String[newSupertypes.size()]));
                @SuppressWarnings("unchecked")
                List<PropertyDefinitionTemplate> propTemplates = template.getPropertyDefinitionTemplates();
                PropertyDefinitionTemplate prop = typeMgr.createPropertyDefinitionTemplate();
                prop.setName("jcr:versionHistory");
                prop.setRequiredType(PropertyType.WEAKREFERENCE);
                propTemplates.add(prop);
                prop = typeMgr.createPropertyDefinitionTemplate();
                prop.setName("jcr:baseVersion");
                prop.setRequiredType(PropertyType.WEAKREFERENCE);
                propTemplates.add(prop);
                prop = typeMgr.createPropertyDefinitionTemplate();
                prop.setName("jcr:predecessors");
                prop.setRequiredType(PropertyType.WEAKREFERENCE);
                prop.setMultiple(true);
                propTemplates.add(prop);
                prop = typeMgr.createPropertyDefinitionTemplate();
                prop.setName("jcr:mergeFailed");
                prop.setRequiredType(PropertyType.WEAKREFERENCE);
                propTemplates.add(prop);
                prop = typeMgr.createPropertyDefinitionTemplate();
                prop.setName("jcr:activity");
                prop.setRequiredType(PropertyType.WEAKREFERENCE);
                propTemplates.add(prop);
                prop = typeMgr.createPropertyDefinitionTemplate();
                prop.setName("jcr:configuration");
                prop.setRequiredType(PropertyType.WEAKREFERENCE);
                propTemplates.add(prop);
                
                log.info("Replacing the versionable feed type '{}' with a non-versionable type", currentFeedType);
                NodeType newType = typeMgr.registerNodeType(template, true);
                log.info("Replaced with new feed type '{}' with a non-versionavble type", newType);
                
                for (Node feedNode : currentFeeds) {
                    feedNode.setPrimaryType(newType.getName());
                    log.info("Replaced type of node {}", feedNode);
                    
                    if (feedNode.hasProperty("jcr:predecessors")) {<commit_msg>Cleaned up feed versioning removal code.<commit_after>/**
 * 
 */
package com.thinkbiganalytics.metadata.modeshape;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.regex.Matcher;
import java.util.stream.Collectors;

import javax.inject.Inject;
import javax.jcr.Node;
import javax.jcr.NodeIterator;
import javax.jcr.PropertyType;
import javax.jcr.RepositoryException;
import javax.jcr.Session;
import javax.jcr.Value;
import javax.jcr.nodetype.NodeType;
import javax.jcr.nodetype.NodeTypeIterator;
import javax.jcr.nodetype.NodeTypeTemplate;
import javax.jcr.nodetype.PropertyDefinition;
import javax.jcr.nodetype.PropertyDefinitionTemplate;
import javax.jcr.security.Privilege;
import javax.jcr.version.Version;
import javax.jcr.version.VersionHistory;
import javax.jcr.version.VersionIterator;
import javax.jcr.version.VersionManager;

import org.apache.tika.metadata.Property;
import org.modeshape.jcr.api.nodetype.NodeTypeManager;
import org.modeshape.jcr.security.SimplePrincipal;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.core.annotation.AnnotationAwareOrderComparator;

import com.thinkbiganalytics.metadata.api.MetadataAccess;
import com.thinkbiganalytics.metadata.api.PostMetadataConfigAction;
import com.thinkbiganalytics.metadata.modeshape.common.SecurityPaths;
import com.thinkbiganalytics.metadata.modeshape.extension.ExtensionsConstants;
import com.thinkbiganalytics.metadata.modeshape.security.AdminCredentials;
import com.thinkbiganalytics.metadata.modeshape.security.JcrAccessControlUtil;
import com.thinkbiganalytics.metadata.modeshape.security.ModeShapeAdminPrincipal;
import com.thinkbiganalytics.metadata.modeshape.support.JcrUtil;

/**
 *
 * @author <NAME>
 */
public class MetadataJcrConfigurator {
    
    private static final Logger log = LoggerFactory.getLogger(MetadataJcrConfigurator.class);
    
    @Inject
    private MetadataAccess metadataAccess;
    
    private List<PostMetadataConfigAction> postConfigActions = new ArrayList<>();
    
    public MetadataJcrConfigurator(List<PostMetadataConfigAction> actions) {
        this.postConfigActions.addAll(actions);
        this.postConfigActions.sort(new AnnotationAwareOrderComparator());
    }
    
    private final AtomicBoolean configured = new AtomicBoolean(false);
    
    
    public void configure() {
        this.metadataAccess.commit(() -> {
            try {
                Session session = JcrMetadataAccess.getActiveSession();
                
                ensureLayout(session);
                ensureTypes(session);
                ensureAccessControl(session);
            } catch (RepositoryException e) {
                throw new MetadataRepositoryException("Could not create initial JCR metadata", e);
            }
        }, MetadataAccess.SERVICE);
        
        this.metadataAccess.commit(() -> {
            try {
                Session session = JcrMetadataAccess.getActiveSession();
                
                removeVersionableFeedType(session);
            } catch (RepositoryException e) {
                throw new MetadataRepositoryException("Could remove versioning from feeds", e);
            }
        }, MetadataAccess.SERVICE);
        
        this.configured.set(true);
        firePostConfigActions();
    }
    
    private void removeVersionableFeedType(Session session) throws RepositoryException {
        Node feedsNode = session.getRootNode().getNode("metadata/feeds");
        
        NodeTypeManager typeMgr = (NodeTypeManager) session.getWorkspace().getNodeTypeManager();
        NodeType currentFeedType = typeMgr.getNodeType("tba:feed");
        List<String> currentSupertypes = Arrays.asList(currentFeedType.getDeclaredSupertypeNames());
        
        if (currentSupertypes.contains("mix:versionable")) {
            // Remove feed version history
            for (Node catNode : JcrUtil.getNodesOfType(feedsNode, "tba:category")) {
                for (Node feedNode : JcrUtil.getNodesOfType(catNode, "tba:feed")) {
                    log.info("Removing prior versions of feed: {}.{}", catNode.getName(), feedNode.getName());
                    
                    VersionManager versionManager = session.getWorkspace().getVersionManager(); 
                    VersionHistory versionHistory = versionManager.getVersionHistory(feedNode.getPath()); 
                    VersionIterator vIt = versionHistory.getAllVersions(); 
                    int count = 0;
                    String last = "";
                    
                    while (vIt.hasNext()) { 
                        Version version = vIt.nextVersion(); 
                        if (!"jcr:rootVersion".equals(version.getName())) { 
                            last = version.getName();
                            // removeVersion writes directly to workspace, no session.save is necessary 
                            versionHistory.removeVersion(version.getName()); 
                            count++;
                        } 
                    } 
                    
                    if (count > 0) {
                        log.info("Removed {} versions through {} of feed {}", count, last, feedNode.getName());
                    } else {
                        log.info("Feed {} had no versions", feedNode.getName());
                    }
                }
            }
            
            // Redefine the NodeType of tba:feed to remove versionable but retain the versionable properties with weaker constraints
            // Retaining the properties seems to override some residual properties on feed nodes that causes a failure later.
            // In particular, jcr:predecessors was accessed later but redefining all mix:versionable properties to be safe.
            NodeTypeTemplate template = typeMgr.createNodeTypeTemplate(currentFeedType);
            List<String> newSupertypes = currentSupertypes.stream().filter(type -> ! type.equals("mix:versionable")).collect(Collectors.toList());
            
            template.setDeclaredSuperTypeNames(newSupertypes.toArray(new String[newSupertypes.size()]));
            
            @SuppressWarnings("unchecked")
            List<PropertyDefinitionTemplate> propTemplates = template.getPropertyDefinitionTemplates();
            PropertyDefinitionTemplate prop = typeMgr.createPropertyDefinitionTemplate();
            prop.setName("jcr:versionHistory");
            prop.setRequiredType(PropertyType.WEAKREFERENCE);
            propTemplates.add(prop);
            prop = typeMgr.createPropertyDefinitionTemplate();
            prop.setName("jcr:baseVersion");
            prop.setRequiredType(PropertyType.WEAKREFERENCE);
            propTemplates.add(prop);
            prop = typeMgr.createPropertyDefinitionTemplate();
            prop.setName("jcr:predecessors");
            prop.setRequiredType(PropertyType.WEAKREFERENCE);
            prop.setMultiple(true);
            propTemplates.add(prop);
            prop = typeMgr.createPropertyDefinitionTemplate();
            prop.setName("jcr:mergeFailed");
            prop.setRequiredType(PropertyType.WEAKREFERENCE);
            propTemplates.add(prop);
            prop = typeMgr.createPropertyDefinitionTemplate();
            prop.setName("jcr:activity");
            prop.setRequiredType(PropertyType.WEAKREFERENCE);
            propTemplates.add(prop);
            prop = typeMgr.createPropertyDefinitionTemplate();
            prop.setName("jcr:configuration");
            prop.setRequiredType(PropertyType.WEAKREFERENCE);
            propTemplates.add(prop);
            
            log.info("Replacing the versionable feed type '{}' with a non-versionable type", currentFeedType);
            NodeType newType = typeMgr.registerNodeType(template, true);
            log.info("Replaced with new feed type '{}' with a non-versionavble type", newType);
            
            // This step may not be necessary.
            for (Node catNode : JcrUtil.getNodesOfType(feedsNode, "tba:category")) {
                for (Node feedNode : JcrUtil.getNodesOfType(catNode, "tba:feed")) {
                    feedNode.setPrimaryType(newType.getName());
                    log.info("Replaced type of node {}", feedNode);
                    
                    if (feedNode.hasProperty("jcr:predecessors")) {
{'fraction_non_alphanumeric': 0.0944804797454412, 'fraction_numerical': 0.0004895361644841513, 'mean_word_length': 2.4801959114139693, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '382752', 'n_tokens_mistral': 4165, 'n_tokens_neox': 4061, 'n_words': 718}