<commit_before>package de.iteratec.osm.result

import de.iteratec.osm.OsmConfigCacheService
import de.iteratec.osm.annotations.RestAction
import de.iteratec.osm.barchart.GetBarchartCommand
import de.iteratec.osm.csi.Page
import de.iteratec.osm.measurement.schedule.Job
import de.iteratec.osm.measurement.schedule.JobDaoService
import de.iteratec.osm.measurement.schedule.JobGroup
import de.iteratec.osm.measurement.schedule.dao.JobGroupDaoService
import de.iteratec.osm.measurement.script.PlaceholdersUtility
import de.iteratec.osm.measurement.script.Script
import de.iteratec.osm.measurement.script.ScriptParser
import de.iteratec.osm.result.dto.PageAggregationChartDTO
import de.iteratec.osm.result.dto.PageAggregationChartSeriesDTO
import de.iteratec.osm.util.ControllerUtils
import de.iteratec.osm.util.ExceptionHandlerController
import de.iteratec.osm.util.I18nService

import org.springframework.http.HttpStatus

class PageAggregationController extends ExceptionHandlerController {

    public final static String DATE_FORMAT_STRING_FOR_HIGH_CHART = 'dd.mm.yyyy';
    public final static int MONDAY_WEEKSTART = 1

    JobGroupDaoService jobGroupDaoService
    JobDaoService jobDaoService
    EventResultDashboardService eventResultDashboardService
    I18nService i18nService
    PageService pageService
    OsmConfigCacheService osmConfigCacheService


    def index() {
        redirect(action: 'show')
    }

    def show() {
        Map<String, Object> modelToRender = [:]

        // AggregatorTypes
        modelToRender.put('aggrGroupValuesUnCached', Measurand.values().groupBy { it.measurandGroup })

        // JobGroups
        List<JobGroup> jobGroups = eventResultDashboardService.getAllJobGroups()
        modelToRender.put('folders', jobGroups)
        modelToRender.put('selectedFolder', params.selectedFolder)

        // Pages
        List<Page> pages = eventResultDashboardService.getAllPages()
        modelToRender.put('pages', pages)
        modelToRender.put('selectedPages', params.selectedPages)

        // JavaScript-Utility-Stuff:
        modelToRender.put("dateFormat", DATE_FORMAT_STRING_FOR_HIGH_CHART)
        modelToRender.put("weekStart", MONDAY_WEEKSTART)

        modelToRender.put('selectedAggrGroupValuesUnCached', [])

        modelToRender.put("tagToJobGroupNameMap", jobGroupDaoService.getTagToJobGroupNameMap())

        // Done! :)
        return modelToRender
    }

    def entryAndFollow(){
        return [:]
    }

    /**
     * Rest Method for ajax call.
     * @param cmd The requested data.
     * @return PageAggregationChartDTO as JSON or string message if an error occurred
     */
    @RestAction
    def getBarchartData(GetBarchartCommand cmd) {
        String errorMessages = getErrorMessages(cmd)
        if (errorMessages) {
            ControllerUtils.sendSimpleResponseAsStream(response, HttpStatus.BAD_REQUEST, errorMessages)
            return
        }

        List<JobGroup> allJobGroups = JobGroup.findAllByNameInList(cmd.selectedJobGroups)
        List<Page> allPages = Page.findAllByNameInList(cmd.selectedPages)
        List<String> allMeasurands = cmd.selectedSeries*.measurands.flatten()
        List<String> measurandFieldName = allMeasurands.collect {(it as Measurand).eventResultField}

        List eventResultAverages = EventResult.createCriteria().list {
            'in'('page', allPages)
            'in'('jobGroup', allJobGroups)
            'between'('jobResultDate', cmd.from.toDate(), cmd.to.toDate())
            'between'(
                'fullyLoadedTimeInMillisecs',
                osmConfigCacheService.getMinValidLoadtime(),
                osmConfigCacheService.getMaxValidLoadtime()
            )
            projections {
                groupProperty('page')
                groupProperty('jobGroup')
                measurandFieldName.each { m ->
                    avg(m)
                }
            }
        }

        Map comparativeEventResultAverages = null
        boolean hasComparativeData = cmd.fromComparative && cmd.toComparative;
        if (hasComparativeData) {
            comparativeEventResultAverages = EventResult.createCriteria().list {
                'in'('page', allPages)
                'in'('jobGroup', allJobGroups)
                'between'('jobResultDate', cmd.fromComparative.toDate(), cmd.toComparative.toDate())
                'between'(
                        'fullyLoadedTimeInMillisecs',
                        osmConfigCacheService.getMinValidLoadtime(),
                        osmConfigCacheService.getMaxValidLoadtime()
                )
                projections {
                    groupProperty('page')
                    groupProperty('jobGroup')
                    measurandFieldName.each { m ->
                        avg(m)
                    }
                }
            }?.collectEntries({ it ->
                [("${it[0].name} | ${it[1].name}".toString()): it.takeRight(it.size()-2)]
            })
        }

        // return if no data is available
        if (!eventResultAverages && !comparativeEventResultAverages) {
            ControllerUtils.sendObjectAsJSON(response, [:])
            return
        }

        List allSeries = cmd.selectedSeries
        PageAggregationChartDTO chartDto = new PageAggregationChartDTO(hasComparativeData: hasComparativeData)
        chartDto.i18nMap.put("measurand", i18nService.msg("de.iteratec.result.measurand.label", "Measurand"))
        chartDto.i18nMap.put("jobGroup", i18nService.msg("de.iteratec.isr.wptrd.labels.filterFolder", "JobGroup"))
        chartDto.i18nMap.put("page", i18nService.msg("de.iteratec.isr.wptrd.labels.filterPage", "Page"))
        chartDto.i18nMap.put("comparativeImprovement", i18nService.msg("de.iteratec.osm.chart.comparative.improvement", "Improvement"))
        chartDto.i18nMap.put("comparativeDeterioration", i18nService.msg("de.iteratec.osm.chart.comparative.deterioration", "Deterioration"))

        allSeries.each { series ->
            BarchartSeries barchartSeries = new BarchartSeries(
                    dimensionalUnit: (series.measurands[0] as Measurand).measurandGroup.unit.label,
                    yAxisLabel:  (series.measurands[0] as Measurand).measurandGroup.unit.label,
                    stacked: series.stacked)
            series.measurands.each { currentMeasurand ->
                eventResultAverages.each { datum ->
                    def measurandIndex = allMeasurands.indexOf(currentMeasurand)
                    def key = "${datum[0]} | ${datum[1]?.name}".toString()
                    def value = Measurand.valueOf(currentMeasurand).normalizeValue(datum[measurandIndex + 2])
                    if (value) {
                        PageAggregationChartSeriesDTO seriesDto = new PageAggregationChartSeriesDTO(
                                unit: (series.measurands[0] as Measurand).measurandGroup.unit.label,
                                measurandLabel: i18nService.msg("de.iteratec.isr.measurand.${currentMeasurand}", currentMeasurand),
                                measurand: currentMeasurand,
                                measurandGroup: Measurand.valueOf(currentMeasurand).measurandGroup,
                                value: value,
                                valueComparative: Measurand.valueOf(currentMeasurand).normalizeValue(comparativeEventResultAverages?.get(key)?.getAt(measurandIndex)),
                                page: datum[0],
                                jobGroup: datum[1]?.name
                        )
                        chartDto.series.add(seriesDto)
                    }
                }
            }
            barchartDTO.series.add(barchartSeries)
        }

//      TODO: see ticket [IT-1614]
        chartDto.filterRules = createFilterRules(allPages, allJobGroups)
//        barchartDTO.filterRules = filteringAndSortingDataService.createFilterRules(allPages, allJobGroups)
<commit_msg>fixup of problem introduced by last revert commit<commit_after>package de.iteratec.osm.result

import de.iteratec.osm.OsmConfigCacheService
import de.iteratec.osm.annotations.RestAction
import de.iteratec.osm.barchart.GetBarchartCommand
import de.iteratec.osm.csi.Page
import de.iteratec.osm.measurement.schedule.Job
import de.iteratec.osm.measurement.schedule.JobDaoService
import de.iteratec.osm.measurement.schedule.JobGroup
import de.iteratec.osm.measurement.schedule.dao.JobGroupDaoService
import de.iteratec.osm.measurement.script.PlaceholdersUtility
import de.iteratec.osm.measurement.script.Script
import de.iteratec.osm.measurement.script.ScriptParser
import de.iteratec.osm.result.dto.PageAggregationChartDTO
import de.iteratec.osm.result.dto.PageAggregationChartSeriesDTO
import de.iteratec.osm.util.ControllerUtils
import de.iteratec.osm.util.ExceptionHandlerController
import de.iteratec.osm.util.I18nService

import org.springframework.http.HttpStatus

class PageAggregationController extends ExceptionHandlerController {

    public final static String DATE_FORMAT_STRING_FOR_HIGH_CHART = 'dd.mm.yyyy';
    public final static int MONDAY_WEEKSTART = 1

    JobGroupDaoService jobGroupDaoService
    JobDaoService jobDaoService
    EventResultDashboardService eventResultDashboardService
    I18nService i18nService
    PageService pageService
    OsmConfigCacheService osmConfigCacheService


    def index() {
        redirect(action: 'show')
    }

    def show() {
        Map<String, Object> modelToRender = [:]

        // AggregatorTypes
        modelToRender.put('aggrGroupValuesUnCached', Measurand.values().groupBy { it.measurandGroup })

        // JobGroups
        List<JobGroup> jobGroups = eventResultDashboardService.getAllJobGroups()
        modelToRender.put('folders', jobGroups)
        modelToRender.put('selectedFolder', params.selectedFolder)

        // Pages
        List<Page> pages = eventResultDashboardService.getAllPages()
        modelToRender.put('pages', pages)
        modelToRender.put('selectedPages', params.selectedPages)

        // JavaScript-Utility-Stuff:
        modelToRender.put("dateFormat", DATE_FORMAT_STRING_FOR_HIGH_CHART)
        modelToRender.put("weekStart", MONDAY_WEEKSTART)

        modelToRender.put('selectedAggrGroupValuesUnCached', [])

        modelToRender.put("tagToJobGroupNameMap", jobGroupDaoService.getTagToJobGroupNameMap())

        // Done! :)
        return modelToRender
    }

    def entryAndFollow(){
        return [:]
    }

    /**
     * Rest Method for ajax call.
     * @param cmd The requested data.
     * @return PageAggregationChartDTO as JSON or string message if an error occurred
     */
    @RestAction
    def getBarchartData(GetBarchartCommand cmd) {
        String errorMessages = getErrorMessages(cmd)
        if (errorMessages) {
            ControllerUtils.sendSimpleResponseAsStream(response, HttpStatus.BAD_REQUEST, errorMessages)
            return
        }

        List<JobGroup> allJobGroups = JobGroup.findAllByNameInList(cmd.selectedJobGroups)
        List<Page> allPages = Page.findAllByNameInList(cmd.selectedPages)
        List<String> allMeasurands = cmd.selectedSeries*.measurands.flatten()
        List<String> measurandFieldName = allMeasurands.collect {(it as Measurand).eventResultField}

        List eventResultAverages = EventResult.createCriteria().list {
            'in'('page', allPages)
            'in'('jobGroup', allJobGroups)
            'between'('jobResultDate', cmd.from.toDate(), cmd.to.toDate())
            'between'(
                'fullyLoadedTimeInMillisecs',
                osmConfigCacheService.getMinValidLoadtime(),
                osmConfigCacheService.getMaxValidLoadtime()
            )
            projections {
                groupProperty('page')
                groupProperty('jobGroup')
                measurandFieldName.each { m ->
                    avg(m)
                }
            }
        }

        Map comparativeEventResultAverages = null
        boolean hasComparativeData = cmd.fromComparative && cmd.toComparative;
        if (hasComparativeData) {
            comparativeEventResultAverages = EventResult.createCriteria().list {
                'in'('page', allPages)
                'in'('jobGroup', allJobGroups)
                'between'('jobResultDate', cmd.fromComparative.toDate(), cmd.toComparative.toDate())
                'between'(
                        'fullyLoadedTimeInMillisecs',
                        osmConfigCacheService.getMinValidLoadtime(),
                        osmConfigCacheService.getMaxValidLoadtime()
                )
                projections {
                    groupProperty('page')
                    groupProperty('jobGroup')
                    measurandFieldName.each { m ->
                        avg(m)
                    }
                }
            }?.collectEntries({ it ->
                [("${it[0].name} | ${it[1].name}".toString()): it.takeRight(it.size()-2)]
            })
        }

        // return if no data is available
        if (!eventResultAverages && !comparativeEventResultAverages) {
            ControllerUtils.sendObjectAsJSON(response, [:])
            return
        }

        List allSeries = cmd.selectedSeries
        PageAggregationChartDTO chartDto = new PageAggregationChartDTO(hasComparativeData: hasComparativeData)
        chartDto.i18nMap.put("measurand", i18nService.msg("de.iteratec.result.measurand.label", "Measurand"))
        chartDto.i18nMap.put("jobGroup", i18nService.msg("de.iteratec.isr.wptrd.labels.filterFolder", "JobGroup"))
        chartDto.i18nMap.put("page", i18nService.msg("de.iteratec.isr.wptrd.labels.filterPage", "Page"))
        chartDto.i18nMap.put("comparativeImprovement", i18nService.msg("de.iteratec.osm.chart.comparative.improvement", "Improvement"))
        chartDto.i18nMap.put("comparativeDeterioration", i18nService.msg("de.iteratec.osm.chart.comparative.deterioration", "Deterioration"))

        allSeries.each { series ->
            series.measurands.each { currentMeasurand ->
                eventResultAverages.each { datum ->
                    def measurandIndex = allMeasurands.indexOf(currentMeasurand)
                    def key = "${datum[0]} | ${datum[1]?.name}".toString()
                    def value = Measurand.valueOf(currentMeasurand).normalizeValue(datum[measurandIndex + 2])
                    if (value) {
                        PageAggregationChartSeriesDTO seriesDto = new PageAggregationChartSeriesDTO(
                                unit: (series.measurands[0] as Measurand).measurandGroup.unit.label,
                                measurandLabel: i18nService.msg("de.iteratec.isr.measurand.${currentMeasurand}", currentMeasurand),
                                measurand: currentMeasurand,
                                measurandGroup: Measurand.valueOf(currentMeasurand).measurandGroup,
                                value: value,
                                valueComparative: Measurand.valueOf(currentMeasurand).normalizeValue(comparativeEventResultAverages?.get(key)?.getAt(measurandIndex)),
                                page: datum[0],
                                jobGroup: datum[1]?.name
                        )
                        chartDto.series.add(seriesDto)
                    }
                }
            }
        }

//      TODO: see ticket [IT-1614]
        chartDto.filterRules = createFilterRules(allPages, allJobGroups)
//        barchartDTO.filterRules = filteringAndSortingDataService.createFilterRules(allPages, allJobGroups)

{'fraction_non_alphanumeric': 0.09448160535117058, 'fraction_numerical': 0.0055312580396192435, 'mean_word_length': 2.4385227775320653, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2373470', 'n_tokens_mistral': 4530, 'n_tokens_neox': 4334, 'n_words': 614}