<commit_before>/*
* OpenSpeedMonitor (OSM)
* Copyright 2014 iteratec GmbH
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* 	http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/


import de.iteratec.osm.OsmConfiguration
import de.iteratec.osm.api.MicroServiceApiKey
import de.iteratec.osm.batch.BatchActivity
import de.iteratec.osm.batch.Status
import de.iteratec.osm.csi.*
import de.iteratec.osm.measurement.environment.Browser
import de.iteratec.osm.measurement.environment.BrowserAlias
import de.iteratec.osm.measurement.environment.wptserverproxy.AssetRequestPersisterService
import de.iteratec.osm.measurement.environment.wptserverproxy.LocationPersisterService
import de.iteratec.osm.measurement.environment.wptserverproxy.ResultPersisterService
import de.iteratec.osm.measurement.environment.wptserverproxy.ProxyService
import de.iteratec.osm.measurement.schedule.ConnectivityProfile
import de.iteratec.osm.measurement.schedule.JobGroup

import de.iteratec.osm.measurement.schedule.JobProcessingService
import de.iteratec.osm.report.chart.AggregatorType
import de.iteratec.osm.report.chart.CsiAggregationInterval
import de.iteratec.osm.report.chart.CsiAggregationUtilService
import de.iteratec.osm.report.chart.MeasurandGroup
import de.iteratec.osm.report.external.GraphiteServer
import de.iteratec.osm.report.external.HealthReportService
import de.iteratec.osm.result.JobResultDaoService
import de.iteratec.osm.security.Role
import de.iteratec.osm.security.User
import de.iteratec.osm.security.UserRole
import de.iteratec.osm.util.I18nService
import grails.util.BuildSettings
import grails.util.Environment
import org.apache.commons.validator.routines.UrlValidator
import org.joda.time.DateTime
import org.springframework.core.io.DefaultResourceLoader
import org.springframework.core.io.Resource
import org.springframework.core.io.ResourceLoader

class BootStrap {

    ResourceLoader defaultResourceLoader = new DefaultResourceLoader()
    EventCsiAggregationService eventCsiAggregationService
    CsiAggregationUtilService csiAggregationUtilService
    JobProcessingService jobProcessingService
    I18nService i18nService
    ResultPersisterService resultPersisterService
    LocationPersisterService locationPersisterService
    AssetRequestPersisterService assetRequestPersisterService
    ProxyService proxyService
    HealthReportService healthReportService
    def grailsApplication

    def init = { servletContext ->

        switch (Environment.getCurrent()) {
            case Environment.DEVELOPMENT:
                initApplicationData(true)
                registerProxyListener()

                break
            case Environment.TEST:
                // no creation of test-data, cause each test will create its own data
                registerProxyListener()
                break;
            case Environment.PRODUCTION:
                initApplicationData(false)
                registerProxyListener()

                break
        }

    }

    def destroy = {
    }

    def initApplicationData = { boolean createDefaultUsers ->
        log.info "initApplicationData() OSM starts"

        initConfig()
        initUserData(createDefaultUsers)
        initChartData()
        initCsiData()
        initMeasurementInfrastructure()
        initJobScheduling()
        cancelActiveBatchActivity()
        excludePropertiesInJsonRepresentationsofDomainObjects()
        initHealthReporting()

        log.info "initApplicationData() OSM ends"
    }


    void initHealthReporting() {
        log.info("initHealthReporting() OSM starts")
        GraphiteServer.findAllByReportHealthMetrics(true).each {
            healthReportService.handleGraphiteServer(it)
        }
        log.info("initHealthReporting() OSM ends")
    }

    void initConfig() {
        log.info "initConfig() OSM starts"

        List<OsmConfiguration> configs = OsmConfiguration.list()

        if (configs.size() != 1) {
            deleteAllInvalidAndCreateNewOsmConfig(configs)
        }

        log.info "initConfig() OSM ends"
    }

    void deleteAllInvalidAndCreateNewOsmConfig(List<OsmConfiguration> configs) {
        configs.each {
            it.delete()
        }
        new OsmConfiguration(
                detailDataStorageTimeInWeeks: 12,
                defaultMaxDownloadTimeInMinutes: 60,
                minDocCompleteTimeInMillisecs: 250,
                maxDocCompleteTimeInMillisecs: 180000,
                maxDataStorageTimeInMonths: 13,
                csiTransformation: CsiTransformation.BY_MAPPING
        ).save(failOnError: true)
    }

    void initJobScheduling() {
        log.info "initJobScheduling() OSM starts"

        createConnectivityProfileIfMissing(6000, 512, 50, 'DSL 6.000', 0)
        createConnectivityProfileIfMissing(384, 384, 140, 'UMTS', 0)
        createConnectivityProfileIfMissing(3600, 1500, 40, 'UMTS - HSDPA', 0)

        jobProcessingService.scheduleAllActiveJobs()

        log.info "initJobScheduling() OSM ends"
    }

    void initUserData(boolean createDefaultUsers) {
        log.info "initUserData() OSM starts"

        // Roles ////////////////////////////////////////////////////////////////////////
        Role adminRole = Role.findByAuthority('ROLE_ADMIN') ?: new Role(authority: 'ROLE_ADMIN').save(failOnError: true)
        Role rootRole = Role.findByAuthority('ROLE_SUPER_ADMIN') ?: new Role(authority: 'ROLE_SUPER_ADMIN').save(failOnError: true)

        // Users ////////////////////////////////////////////////////////////////////////

        //read config entries
        String appAdminUserName = grailsApplication.config.grails.de.iteratec.osm.security.initialOsmAdminUser.username.isEmpty() ?
                null : grailsApplication.config.grails.de.iteratec.osm.security.initialOsmAdminUser.username
        String appAdminPassword = grailsApplication.config.grails.de.iteratec.osm.security.initialOsmAdminUser.password.isEmpty() ?
                null : grailsApplication.config.grails.de.iteratec.osm.security.initialOsmAdminUser.password
        String appRootUserName = grailsApplication.config.grails.de.iteratec.osm.security.initialOsmRootUser.username.isEmpty() ?
                null : grailsApplication.config.grails.de.iteratec.osm.security.initialOsmRootUser.username
        String appRootPassword = grailsApplication.config.grails.de.iteratec.osm.security.initialOsmRootUser.password.isEmpty() ?
                null : grailsApplication.config.grails.de.iteratec.osm.security.initialOsmRootUser.password
        String warnMessage = createDefaultUsers ? 'A default user will be created if no one existed.' : 'No such user will be created.'

        // admin user
        if (appAdminUserName == null || appAdminPassword == null) {
            log.warn("You haven't set environment variables to create an admin user. ${warnMessage}")
            if (createDefaultUsers) createUser('admin', 'admin', adminRole)
        } else {
            createUser(appAdminUserName, appAdminPassword, adminRole)
        }
        //root user
        if (appRootUserName == null || appRootPassword == null) {
            log.warn("You haven't set environment variables to create a root user. ${warnMessage}")
            if (createDefaultUsers) createUser('root', 'root', rootRole)
        } else {
            createUser(appRootUserName, appRootPassword, rootRole)
        }

//        //apiKey for microService
        if(MicroServiceApiKey.list().isEmpty()){
            String initialApiKey = grailsApplication.config.grails.de.iteratec.osm.security.initialApiKey.isEmpty() ?
                    null : grailsApplication.config.grails.de.iteratec.osm.security.initialApiKey
            String initialMicroServiceName = grailsApplication.config.grails.de.iteratec.osm.security.initialMicroServiceName.isEmpty() ?
                    null : grailsApplication.config.grails.de.iteratec.osm.security.initialMicroServiceName
            if(!initialApiKey || !initialMicroServiceName) log.warn("initial ApiKey configuration missing")
            else new MicroServiceApiKey([secretKey:initialApiKey,microService:initialMicroServiceName, valid: true]).save(failOnError:true)

        }

        log.info "initUserData() OSM ends"
    }

    void createUser(String username, String password, Role role) {
        User user = User.findByUsername(username) ?: new User(
                username: username,
                password: password,
                enabled: true,
                accountExpired: false,
                accountLocked: false,
                passwordExpired: false).save(failOnError: true)
        UserRole.findByUser(user) ?: new UserRole(user: user, role: role).save(failOnError: true)
    }

    void initChartData() {
        log.info "initChartData starts"

        createAggregatorTypeIfMissing(AggregatorType.RESULT_UNCACHED_DOC_COMPLETE_TIME, MeasurandGroup.LOAD_TIMES);
        createAggregatorTypeIfMissing(AggregatorType.RESULT_UNCACHED_DOM_TIME, MeasurandGroup.LOAD_TIMES);
        createAggregatorTypeIfMissing(AggregatorType.RESULT_UNCACHED_FIRST_BYTE, MeasurandGroup.LOAD_TIMES);
        createAggregatorTypeIfMissing(AggregatorType.RESULT_UNCACHED_FULLY_LOADED_REQUEST_COUNT, MeasurandGroup.REQUEST_COUNTS);
        createAggregatorTypeIfMissing(AggregatorType.RESULT_UNCACHED_FULLY_LOADED_TIME, MeasurandGroup.LOAD_TIMES);
        createAggregatorTypeIfMissing(AggregatorType.RESULT_UNCACHED_LOAD_TIME, MeasurandGroup.LOAD_TIMES);<commit_msg>make sure that microservice initialization only happens if enablePersistenceOfAssetRequests is active<commit_after>/*
* OpenSpeedMonitor (OSM)
* Copyright 2014 iteratec GmbH
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* 	http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/


import de.iteratec.osm.OsmConfiguration
import de.iteratec.osm.api.MicroServiceApiKey
import de.iteratec.osm.batch.BatchActivity
import de.iteratec.osm.batch.Status
import de.iteratec.osm.csi.*
import de.iteratec.osm.measurement.environment.Browser
import de.iteratec.osm.measurement.environment.BrowserAlias
import de.iteratec.osm.measurement.environment.wptserverproxy.AssetRequestPersisterService
import de.iteratec.osm.measurement.environment.wptserverproxy.LocationPersisterService
import de.iteratec.osm.measurement.environment.wptserverproxy.ResultPersisterService
import de.iteratec.osm.measurement.environment.wptserverproxy.ProxyService
import de.iteratec.osm.measurement.schedule.ConnectivityProfile
import de.iteratec.osm.measurement.schedule.JobGroup

import de.iteratec.osm.measurement.schedule.JobProcessingService
import de.iteratec.osm.report.chart.AggregatorType
import de.iteratec.osm.report.chart.CsiAggregationInterval
import de.iteratec.osm.report.chart.CsiAggregationUtilService
import de.iteratec.osm.report.chart.MeasurandGroup
import de.iteratec.osm.report.external.GraphiteServer
import de.iteratec.osm.report.external.HealthReportService
import de.iteratec.osm.result.JobResultDaoService
import de.iteratec.osm.security.Role
import de.iteratec.osm.security.User
import de.iteratec.osm.security.UserRole
import de.iteratec.osm.util.I18nService
import grails.util.BuildSettings
import grails.util.Environment
import org.apache.commons.validator.routines.UrlValidator
import org.joda.time.DateTime
import org.springframework.core.io.DefaultResourceLoader
import org.springframework.core.io.Resource
import org.springframework.core.io.ResourceLoader

class BootStrap {

    ResourceLoader defaultResourceLoader = new DefaultResourceLoader()
    EventCsiAggregationService eventCsiAggregationService
    CsiAggregationUtilService csiAggregationUtilService
    JobProcessingService jobProcessingService
    I18nService i18nService
    ResultPersisterService resultPersisterService
    LocationPersisterService locationPersisterService
    AssetRequestPersisterService assetRequestPersisterService
    ProxyService proxyService
    HealthReportService healthReportService
    def grailsApplication

    def init = { servletContext ->

        switch (Environment.getCurrent()) {
            case Environment.DEVELOPMENT:
                initApplicationData(true)
                registerProxyListener()

                break
            case Environment.TEST:
                // no creation of test-data, cause each test will create its own data
                registerProxyListener()
                break;
            case Environment.PRODUCTION:
                initApplicationData(false)
                registerProxyListener()

                break
        }

    }

    def destroy = {
    }

    def initApplicationData = { boolean createDefaultUsers ->
        log.info "initApplicationData() OSM starts"

        initConfig()
        initUserData(createDefaultUsers)
        initChartData()
        initCsiData()
        initMeasurementInfrastructure()
        initJobScheduling()
        cancelActiveBatchActivity()
        excludePropertiesInJsonRepresentationsofDomainObjects()
        initHealthReporting()

        log.info "initApplicationData() OSM ends"
    }


    void initHealthReporting() {
        log.info("initHealthReporting() OSM starts")
        GraphiteServer.findAllByReportHealthMetrics(true).each {
            healthReportService.handleGraphiteServer(it)
        }
        log.info("initHealthReporting() OSM ends")
    }

    void initConfig() {
        log.info "initConfig() OSM starts"

        List<OsmConfiguration> configs = OsmConfiguration.list()

        if (configs.size() != 1) {
            deleteAllInvalidAndCreateNewOsmConfig(configs)
        }

        log.info "initConfig() OSM ends"
    }

    void deleteAllInvalidAndCreateNewOsmConfig(List<OsmConfiguration> configs) {
        configs.each {
            it.delete()
        }
        new OsmConfiguration(
                detailDataStorageTimeInWeeks: 12,
                defaultMaxDownloadTimeInMinutes: 60,
                minDocCompleteTimeInMillisecs: 250,
                maxDocCompleteTimeInMillisecs: 180000,
                maxDataStorageTimeInMonths: 13,
                csiTransformation: CsiTransformation.BY_MAPPING
        ).save(failOnError: true)
    }

    void initJobScheduling() {
        log.info "initJobScheduling() OSM starts"

        createConnectivityProfileIfMissing(6000, 512, 50, 'DSL 6.000', 0)
        createConnectivityProfileIfMissing(384, 384, 140, 'UMTS', 0)
        createConnectivityProfileIfMissing(3600, 1500, 40, 'UMTS - HSDPA', 0)

        jobProcessingService.scheduleAllActiveJobs()

        log.info "initJobScheduling() OSM ends"
    }

    void initUserData(boolean createDefaultUsers) {
        log.info "initUserData() OSM starts"

        // Roles ////////////////////////////////////////////////////////////////////////
        Role adminRole = Role.findByAuthority('ROLE_ADMIN') ?: new Role(authority: 'ROLE_ADMIN').save(failOnError: true)
        Role rootRole = Role.findByAuthority('ROLE_SUPER_ADMIN') ?: new Role(authority: 'ROLE_SUPER_ADMIN').save(failOnError: true)

        // Users ////////////////////////////////////////////////////////////////////////

        //read config entries
        String appAdminUserName = grailsApplication.config.grails.de.iteratec.osm.security.initialOsmAdminUser.username.isEmpty() ?
                null : grailsApplication.config.grails.de.iteratec.osm.security.initialOsmAdminUser.username
        String appAdminPassword = grailsApplication.config.grails.de.iteratec.osm.security.initialOsmAdminUser.password.isEmpty() ?
                null : grailsApplication.config.grails.de.iteratec.osm.security.initialOsmAdminUser.password
        String appRootUserName = grailsApplication.config.grails.de.iteratec.osm.security.initialOsmRootUser.username.isEmpty() ?
                null : grailsApplication.config.grails.de.iteratec.osm.security.initialOsmRootUser.username
        String appRootPassword = grailsApplication.config.grails.de.iteratec.osm.security.initialOsmRootUser.password.isEmpty() ?
                null : grailsApplication.config.grails.de.iteratec.osm.security.initialOsmRootUser.password
        String warnMessage = createDefaultUsers ? 'A default user will be created if no one existed.' : 'No such user will be created.'

        // admin user
        if (appAdminUserName == null || appAdminPassword == null) {
            log.warn("You haven't set environment variables to create an admin user. ${warnMessage}")
            if (createDefaultUsers) createUser('admin', 'admin', adminRole)
        } else {
            createUser(appAdminUserName, appAdminPassword, adminRole)
        }
        //root user
        if (appRootUserName == null || appRootPassword == null) {
            log.warn("You haven't set environment variables to create a root user. ${warnMessage}")
            if (createDefaultUsers) createUser('root', 'root', rootRole)
        } else {
            createUser(appRootUserName, appRootPassword, rootRole)
        }


        //apiKey for microService
        Boolean enablePersistenceOfAssetRequests = grailsApplication.config.grails.de.iteratec.osm.assetRequests.enablePersistenceOfAssetRequests == null ?
                null :  grailsApplication.config.grails.de.iteratec.osm.assetRequests.enablePersistenceOfAssetRequests
        if(enablePersistenceOfAssetRequests) {
            if (MicroServiceApiKey.list().isEmpty()) {
                String initialApiKey = grailsApplication.config.grails.de.iteratec.osm.security.initialApiKey.isEmpty() ?
                        null : grailsApplication.config.grails.de.iteratec.osm.security.initialApiKey
                String initialMicroServiceName = grailsApplication.config.grails.de.iteratec.osm.security.initialMicroServiceName.isEmpty() ?
                        null : grailsApplication.config.grails.de.iteratec.osm.security.initialMicroServiceName
                if (!initialApiKey || !initialMicroServiceName) log.warn("initial MicroserviceApiKey configuration missing")
                else new MicroServiceApiKey([secretKey: initialApiKey, microService: initialMicroServiceName, valid: true]).save(failOnError: true)
            }
        }

        log.info "initUserData() OSM ends"
    }

    void createUser(String username, String password, Role role) {
        User user = User.findByUsername(username) ?: new User(
                username: username,
                password: password,
                enabled: true,
                accountExpired: false,
                accountLocked: false,
                passwordExpired: false).save(failOnError: true)
        UserRole.findByUser(user) ?: new UserRole(user: user, role: role).save(failOnError: true)
    }

    void initChartData() {
        log.info "initChartData starts"

        createAggregatorTypeIfMissing(AggregatorType.RESULT_UNCACHED_DOC_COMPLETE_TIME, MeasurandGroup.LOAD_TIMES);
        createAggregatorTypeIfMissing(AggregatorType.RESULT_UNCACHED_DOM_TIME, MeasurandGroup.LOAD_TIMES);
        createAggregatorTypeIfMissing(AggregatorType.RESULT_UNCACHED_FIRST_BYTE, MeasurandGroup.LOAD_TIMES);
        createAggregatorTypeIfMissing(AggregatorType.RESULT_UNCACHED_FULLY_LOADED_REQUEST_COUNT, MeasurandGroup.REQUEST_COUNTS);
        createAggregatorTypeIfMissing(AggregatorType.RESULT_UNCACHED_FULLY_LOADED_TIME, MeasurandGroup.LOAD_TIMES);
        createAggregatorTypeIfMissing(AggregatorType.RESULT_UNCACHED_LOAD_TIME, MeasurandGroup.LOAD_TIMES);
{'fraction_non_alphanumeric': 0.09252230492433579, 'fraction_numerical': 0.006408044560556021, 'mean_word_length': 3.1642036124794743, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3929828', 'n_tokens_mistral': 5741, 'n_tokens_neox': 5425, 'n_words': 1026}