<commit_before>package com.thinkbiganalytics.metadata.upgrade;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;

import javax.inject.Inject;
import javax.jcr.Property;
import javax.jcr.PropertyIterator;
import javax.jcr.RepositoryException;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.core.annotation.Order;
import org.springframework.security.crypto.password.PasswordEncoder;

import com.thinkbiganalytics.feedmgr.security.FeedsAccessControl;
import com.thinkbiganalytics.jobrepo.security.OperationsAccessControl;

/*-
 * #%L
 * thinkbig-operational-metadata-upgrade-service
 * %%
 * Copyright (C) 2017 <NAME>
 * %%
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 *     http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * #L%
 */

import com.thinkbiganalytics.metadata.api.MetadataAccess;
import com.thinkbiganalytics.metadata.api.PostMetadataConfigAction;
import com.thinkbiganalytics.metadata.api.app.KyloVersion;
import com.thinkbiganalytics.metadata.api.app.KyloVersionProvider;
import com.thinkbiganalytics.metadata.api.feed.FeedProvider;
import com.thinkbiganalytics.metadata.api.feed.OpsManagerFeed;
import com.thinkbiganalytics.metadata.api.feed.OpsManagerFeedProvider;
import com.thinkbiganalytics.metadata.api.feedmgr.category.FeedManagerCategory;
import com.thinkbiganalytics.metadata.api.feedmgr.category.FeedManagerCategoryProvider;
import com.thinkbiganalytics.metadata.api.feedmgr.feed.FeedManagerFeed;
import com.thinkbiganalytics.metadata.api.feedmgr.feed.FeedManagerFeedProvider;
import com.thinkbiganalytics.metadata.api.feedmgr.template.FeedManagerTemplate;
import com.thinkbiganalytics.metadata.api.feedmgr.template.FeedManagerTemplateProvider;
import com.thinkbiganalytics.metadata.api.user.User;
import com.thinkbiganalytics.metadata.api.user.UserGroup;
import com.thinkbiganalytics.metadata.api.user.UserProvider;
import com.thinkbiganalytics.metadata.jpa.feed.JpaOpsManagerFeed;
import com.thinkbiganalytics.metadata.modeshape.MetadataRepositoryException;
import com.thinkbiganalytics.metadata.modeshape.feed.JcrFeedManagerFeed;
import com.thinkbiganalytics.security.action.AllowedModuleActionsProvider;
import com.thinkbiganalytics.support.FeedNameUtil;

@Order(PostMetadataConfigAction.DEFAULT_ORDER + 100)
public class UpgradeKyloService implements PostMetadataConfigAction {

    private static final Logger log = LoggerFactory.getLogger(UpgradeKyloService.class);
    @Inject
    OpsManagerFeedProvider opsManagerFeedProvider;
    @Inject
    MetadataAccess metadataAccess;
    @Inject
    private KyloVersionProvider kyloVersionProvider;
    @Inject
    private FeedManagerFeedProvider feedManagerFeedProvider;
    @Inject
    private FeedProvider feedProvider;
    @Inject
    private FeedManagerTemplateProvider feedManagerTemplateProvider;
    @Inject
    private FeedManagerCategoryProvider feedManagerCategoryProvider;
    @Inject
    private UserProvider userProvider;
    @Inject
    private PasswordEncoder passwordEncoder;
    @Inject
    private AllowedModuleActionsProvider actionsProvider;

    
    public void run() {
        upgradeCheck();
    };

    /**
     * checks the upgrade for Kylo and updates the version if needed
     */
    public void upgradeCheck() {
        KyloVersion version = kyloVersionProvider.getKyloVersion();
        
        if (version == null) {
            setupFreshInstall();
        }
        
        if (version == null || version.getMajorVersionNumber() == null || (version.getMajorVersionNumber() != null && version.getMajorVersionNumber()< 0.4f)) {
            version = upgradeTo0_4_0();
        }
        ensureFeedTemplateFeedRelationships();
        // migrateUnusedFeedProperties();
        version = metadataAccess.commit(() -> {
            //ensure/update the version
            KyloVersion kyloVersion = kyloVersionProvider.updateToCurrentVersion();
            return kyloVersion;
        }, MetadataAccess.SERVICE);

        log.info("Upgrade check complete for Kylo {}", version.getVersion());
    }

    /**
     * 
     */
    private void setupFreshInstall() {
        metadataAccess.commit(() -> {
            User dladmin = createDefaultUser("dladmin", "Data Lake Administrator", "thinkbig");
            User analyst = createDefaultUser("analyst", "Analyst", "analyst");
            User designer = createDefaultUser("designer", "Designer", "designer");
            User operator = createDefaultUser("operator", "Operator", "operator");
            
            // Create default groups if they don't exist.
            UserGroup adminsGroup = createDefaultGroup("admin", "Administrators");
            UserGroup opsGroup = createDefaultGroup("operations", "Operations");
            UserGroup designersGroup = createDefaultGroup("designer", "Designers");
            UserGroup analystsGroup = createDefaultGroup("analyst", "Analysts");
            UserGroup usersGroup = createDefaultGroup("user", "Users");
            
            // Add default users to their respective groups
            adminsGroup.addUser(dladmin);
            designersGroup.addUser(designer);
            analystsGroup.addUser(analyst);
            opsGroup.addUser(operator);
            usersGroup.addUser(dladmin);
            usersGroup.addUser(analyst);
            usersGroup.addUser(designer);
            usersGroup.addUser(operator);
            
            // Setup initial group access control.  Administrators group already has all rights.
            actionsProvider.getAllowedActions("services")
                            .ifPresent((allowed) -> {
                                allowed.enable(opsGroup.getRootPrincial(), 
                                               OperationsAccessControl.ADMIN_OPS,
                                               FeedsAccessControl.ACCESS_CATEGORIES,
                                               FeedsAccessControl.ACCESS_FEEDS);
                                allowed.enable(designersGroup.getRootPrincial(), 
                                               OperationsAccessControl.ACCESS_OPS,
                                               FeedsAccessControl.EDIT_FEEDS,
                                               FeedsAccessControl.IMPORT_FEEDS,
                                               FeedsAccessControl.EXPORT_FEEDS,
                                               FeedsAccessControl.EDIT_CATEGORIES,
                                               FeedsAccessControl.EDIT_TEMPLATES);
                                allowed.enable(analystsGroup.getRootPrincial(), 
                                               OperationsAccessControl.ACCESS_OPS,
                                               FeedsAccessControl.EDIT_FEEDS,
                                               FeedsAccessControl.ACCESS_CATEGORIES,
                                               FeedsAccessControl.IMPORT_TEMPLATES,
                                               FeedsAccessControl.EXPORT_TEMPLATES,
                                               FeedsAccessControl.ACCESS_TEMPLATES);
                            });
        }, MetadataAccess.SERVICE);
    }

    /*
     * Ensure the Feed Template has the relationships setup to its related feeds
     */
    private void ensureFeedTemplateFeedRelationships() {
        metadataAccess.commit(() -> {

            //ensure the templates have the feed relationships
            List<FeedManagerFeed> feeds = feedManagerFeedProvider.findAll();
            if (feeds != null) {
                feeds.stream().forEach(feed -> {
                    FeedManagerTemplate template = feed.getTemplate();
                    if (template != null) {
                        //ensure the template has feeds
                        if (template.getFeeds() == null || !template.getFeeds().contains(feed)) {
                            template.addFeed(feed);
                            feedManagerTemplateProvider.update(template);
                        }
                    }
                });

            }

            feedProvider.populateInverseFeedDependencies();
<commit_msg>PC-1207: Re-added mistakenly reverted perms assignments<commit_after>package com.thinkbiganalytics.metadata.upgrade;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;

import javax.inject.Inject;
import javax.jcr.Property;
import javax.jcr.PropertyIterator;
import javax.jcr.RepositoryException;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.core.annotation.Order;
import org.springframework.security.crypto.password.PasswordEncoder;

import com.thinkbiganalytics.feedmgr.security.FeedsAccessControl;
import com.thinkbiganalytics.jobrepo.security.OperationsAccessControl;

/*-
 * #%L
 * thinkbig-operational-metadata-upgrade-service
 * %%
 * Copyright (C) 2017 ThinkBig Analytics
 * %%
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 *     http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * #L%
 */

import com.thinkbiganalytics.metadata.api.MetadataAccess;
import com.thinkbiganalytics.metadata.api.PostMetadataConfigAction;
import com.thinkbiganalytics.metadata.api.app.KyloVersion;
import com.thinkbiganalytics.metadata.api.app.KyloVersionProvider;
import com.thinkbiganalytics.metadata.api.feed.FeedProvider;
import com.thinkbiganalytics.metadata.api.feed.OpsManagerFeed;
import com.thinkbiganalytics.metadata.api.feed.OpsManagerFeedProvider;
import com.thinkbiganalytics.metadata.api.feedmgr.category.FeedManagerCategory;
import com.thinkbiganalytics.metadata.api.feedmgr.category.FeedManagerCategoryProvider;
import com.thinkbiganalytics.metadata.api.feedmgr.feed.FeedManagerFeed;
import com.thinkbiganalytics.metadata.api.feedmgr.feed.FeedManagerFeedProvider;
import com.thinkbiganalytics.metadata.api.feedmgr.template.FeedManagerTemplate;
import com.thinkbiganalytics.metadata.api.feedmgr.template.FeedManagerTemplateProvider;
import com.thinkbiganalytics.metadata.api.user.User;
import com.thinkbiganalytics.metadata.api.user.UserGroup;
import com.thinkbiganalytics.metadata.api.user.UserProvider;
import com.thinkbiganalytics.metadata.jpa.feed.JpaOpsManagerFeed;
import com.thinkbiganalytics.metadata.modeshape.MetadataRepositoryException;
import com.thinkbiganalytics.metadata.modeshape.feed.JcrFeedManagerFeed;
import com.thinkbiganalytics.security.action.AllowedModuleActionsProvider;
import com.thinkbiganalytics.support.FeedNameUtil;

@Order(PostMetadataConfigAction.DEFAULT_ORDER + 100)
public class UpgradeKyloService implements PostMetadataConfigAction {

    private static final Logger log = LoggerFactory.getLogger(UpgradeKyloService.class);
    @Inject
    OpsManagerFeedProvider opsManagerFeedProvider;
    @Inject
    MetadataAccess metadataAccess;
    @Inject
    private KyloVersionProvider kyloVersionProvider;
    @Inject
    private FeedManagerFeedProvider feedManagerFeedProvider;
    @Inject
    private FeedProvider feedProvider;
    @Inject
    private FeedManagerTemplateProvider feedManagerTemplateProvider;
    @Inject
    private FeedManagerCategoryProvider feedManagerCategoryProvider;
    @Inject
    private UserProvider userProvider;
    @Inject
    private PasswordEncoder passwordEncoder;
    @Inject
    private AllowedModuleActionsProvider actionsProvider;

    
    public void run() {
        upgradeCheck();
    };

    /**
     * checks the upgrade for Kylo and updates the version if needed
     */
    public void upgradeCheck() {
        KyloVersion version = kyloVersionProvider.getKyloVersion();
        
        if (version == null) {
            setupFreshInstall();
        }
        
        if (version == null || version.getMajorVersionNumber() == null || (version.getMajorVersionNumber() != null && version.getMajorVersionNumber()< 0.4f)) {
            version = upgradeTo0_4_0();
        }
        ensureFeedTemplateFeedRelationships();
        // migrateUnusedFeedProperties();
        version = metadataAccess.commit(() -> {
            //ensure/update the version
            KyloVersion kyloVersion = kyloVersionProvider.updateToCurrentVersion();
            return kyloVersion;
        }, MetadataAccess.SERVICE);

        log.info("Upgrade check complete for Kylo {}", version.getVersion());
    }

    /**
     * 
     */
    private void setupFreshInstall() {
        metadataAccess.commit(() -> {
            User dladmin = createDefaultUser("dladmin", "Data Lake Administrator", "thinkbig");
            User analyst = createDefaultUser("analyst", "Analyst", "analyst");
            User designer = createDefaultUser("designer", "Designer", "designer");
            User operator = createDefaultUser("operator", "Operator", "operator");
            
            // Create default groups if they don't exist.
            UserGroup adminsGroup = createDefaultGroup("admin", "Administrators");
            UserGroup opsGroup = createDefaultGroup("operations", "Operations");
            UserGroup designersGroup = createDefaultGroup("designer", "Designers");
            UserGroup analystsGroup = createDefaultGroup("analyst", "Analysts");
            UserGroup usersGroup = createDefaultGroup("user", "Users");
            
            // Add default users to their respective groups
            adminsGroup.addUser(dladmin);
            designersGroup.addUser(designer);
            analystsGroup.addUser(analyst);
            opsGroup.addUser(operator);
            usersGroup.addUser(dladmin);
            usersGroup.addUser(analyst);
            usersGroup.addUser(designer);
            usersGroup.addUser(operator);
            
            // Setup initial group access control.  Administrators group already has all rights.
            actionsProvider.getAllowedActions("services")
                            .ifPresent((allowed) -> {
                                allowed.enable(opsGroup.getRootPrincial(), 
                                               OperationsAccessControl.ADMIN_OPS,
                                               FeedsAccessControl.ACCESS_CATEGORIES,
                                               FeedsAccessControl.ACCESS_FEEDS);
                                allowed.enable(designersGroup.getRootPrincial(), 
                                               OperationsAccessControl.ACCESS_OPS,
                                               FeedsAccessControl.EDIT_FEEDS,
                                               FeedsAccessControl.IMPORT_FEEDS,
                                               FeedsAccessControl.EXPORT_FEEDS,
                                               FeedsAccessControl.EDIT_CATEGORIES,
                                               FeedsAccessControl.EDIT_TEMPLATES,
                                               FeedsAccessControl.IMPORT_TEMPLATES,
                                               FeedsAccessControl.EXPORT_TEMPLATES);
                                allowed.enable(analystsGroup.getRootPrincial(), 
                                               OperationsAccessControl.ACCESS_OPS,
                                               FeedsAccessControl.EDIT_FEEDS,
                                               FeedsAccessControl.IMPORT_FEEDS,
                                               FeedsAccessControl.EXPORT_FEEDS,
                                               FeedsAccessControl.EDIT_CATEGORIES,
                                               FeedsAccessControl.ACCESS_TEMPLATES);
                            });
        }, MetadataAccess.SERVICE);
    }

    /*
     * Ensure the Feed Template has the relationships setup to its related feeds
     */
    private void ensureFeedTemplateFeedRelationships() {
        metadataAccess.commit(() -> {

            //ensure the templates have the feed relationships
            List<FeedManagerFeed> feeds = feedManagerFeedProvider.findAll();
            if (feeds != null) {
                feeds.stream().forEach(feed -> {
                    FeedManagerTemplate template = feed.getTemplate();
                    if (template != null) {
                        //ensure the template has feeds
                        if (template.getFeeds() == null || !template.getFeeds().contains(feed)) {
                            template.addFeed(feed);
                            feedManagerTemplateProvider.update(template);
                        }
                    }
                });

            }

            feedProvider.populateInverseFeedDependencies();

{'fraction_non_alphanumeric': 0.08007056681083542, 'fraction_numerical': 0.0022763487366264513, 'mean_word_length': 2.3238131265367885, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 64, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1925694', 'n_tokens_mistral': 4233, 'n_tokens_neox': 4142, 'n_words': 842}