<commit_before>/* 
* OpenSpeedMonitor (OSM)
* Copyright 2014 iteratec GmbH
* 
* Licensed under the Apache License, Version 2.0 (the "License"); 
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* 
* 	http://www.apache.org/licenses/LICENSE-2.0
* 
* Unless required by applicable law or agreed to in writing, software 
* distributed under the License is distributed on an "AS IS" BASIS, 
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 
* See the License for the specific language governing permissions and 
* limitations under the License.
*/

package de.iteratec.osm.report.external

import de.iteratec.osm.batch.Activity
import de.iteratec.osm.batch.BatchActivity
import de.iteratec.osm.batch.BatchActivityService
import de.iteratec.osm.measurement.environment.wptserverproxy.HttpRequestService
import de.iteratec.osm.report.chart.Event
import de.iteratec.osm.report.chart.EventDaoService
import de.iteratec.osm.report.chart.MeasuredValueUtilService
import org.joda.time.DateTime
import org.joda.time.format.DateTimeFormat
import org.joda.time.format.DateTimeFormatter

/**
 * GraphiteEventService
 * A service to fetch Events from GraphiteServers.
 */
class GraphiteEventService {

    EventDaoService eventDaoService
    BatchActivityService batchActivityService
	HttpRequestService httpRequestService
    MeasuredValueUtilService measuredValueUtilService

    static final DateTimeFormatter GRAPHITE_RENDERING_ENGINES_DATETIME_FORMAT =  DateTimeFormat.forPattern("HH:mm_yyyyMMdd")

    static transactional = true

    /**
     * Iterates over all existing GraphiteServers. If they got any GraphiteEventSourcePath,
     * these will be used to fetch new Events from.
     * @param createBatchActivity defines if a BatchActivity should be created and updated
     * @param lastNMinutes Defines oldest Events that get fetched from EventSources.
     * @see de.iteratec.osm.report.external.GraphiteEventCollectorJob
     */
    public void fetchGraphiteEvents(boolean createBatchActivity, int lastNMinutes) {

        DateTime until = measuredValueUtilService.getNowInUtc()
        DateTime from = until.minusMinutes(lastNMinutes)
        String untilFormatted = GRAPHITE_RENDERING_ENGINES_DATETIME_FORMAT.print(until)
        String fromFormatted = GRAPHITE_RENDERING_ENGINES_DATETIME_FORMAT.print(from)

        def graphiteServers = GraphiteServer.list()
        int size = graphiteServers.size()
        BatchActivity activity = batchActivityService.getActiveBatchActivity(this.class,new Date().getTime(),Activity.CREATE,"Fetch Graphite Events",createBatchActivity)

        graphiteServers.eachWithIndex {server,index ->
            activity.updateStatus(["progress": batchActivityService.calculateProgress(size,index+1), "stage": "Delete JobResults"])
            server.graphiteEventSourcePaths.each {eventSourcePath->
                createEvents(eventSourcePath, server, fromFormatted, untilFormatted)
            }
        }

    }
    /**
     * Fetches the Events from the given GraphiteServer within the path.
     * @param eventSourcePath
     * @param server
     * @param from
     * @param until
     * @return List of fetched Events, they are already saved
     */
    private List<Event> createEvents(GraphiteEventSourcePath eventSourcePath, GraphiteServer server, String from, String until){
        def events = []
        def json = getEventJSON(eventSourcePath.targetMetricName, server, from, until)
        json.each{
            String shortName = it.target
            it.datapoints.each{point ->
                long unixTimeStamp = Long.parseLong(point[1] as String) * 1000L
                events << eventDaoService.createEvent(
                        "${eventSourcePath.staticPrefix}${shortName}",
                        new DateTime(unixTimeStamp),
                        "Read from Graphite: ${shortName} [${eventSourcePath.targetMetricName}]",
                        false,
                        eventSourcePath.jobGroups)
            }
        }
        return events
    }
    /**
     * Calls the path from a GraphiteServer and converts it into a JSONObject
     * @param eventSourceMetricName Name of the metric to fetch.
     * @param server GraphiteServer
     * @param from
     * @param until
     * @return
     */
    private Object getEventJSON(String eventSourceMetricName, GraphiteServer server, String from, String until){

        String webappUrl = createWebappUrlOf(server)
        LinkedHashMap<String, String> queryParams = ['format': 'json', 'from': from, 'until': until, 'target': eventSourceMetricName]

        return httpRequestService.getJsonResponse(webappUrl, server.webappPathToRenderingEngine, queryParams)
    }

    String createWebappUrlOf(GraphiteServer server){<commit_msg>just take graphite points with values not null as events<commit_after>/* 
* OpenSpeedMonitor (OSM)
* Copyright 2014 iteratec GmbH
* 
* Licensed under the Apache License, Version 2.0 (the "License"); 
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* 
* 	http://www.apache.org/licenses/LICENSE-2.0
* 
* Unless required by applicable law or agreed to in writing, software 
* distributed under the License is distributed on an "AS IS" BASIS, 
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 
* See the License for the specific language governing permissions and 
* limitations under the License.
*/

package de.iteratec.osm.report.external

import de.iteratec.osm.batch.Activity
import de.iteratec.osm.batch.BatchActivity
import de.iteratec.osm.batch.BatchActivityService
import de.iteratec.osm.measurement.environment.wptserverproxy.HttpRequestService
import de.iteratec.osm.report.chart.Event
import de.iteratec.osm.report.chart.EventDaoService
import de.iteratec.osm.report.chart.MeasuredValueUtilService
import org.joda.time.DateTime
import org.joda.time.format.DateTimeFormat
import org.joda.time.format.DateTimeFormatter

/**
 * GraphiteEventService
 * A service to fetch Events from GraphiteServers.
 */
class GraphiteEventService {

    EventDaoService eventDaoService
    BatchActivityService batchActivityService
	HttpRequestService httpRequestService
    MeasuredValueUtilService measuredValueUtilService

    static final DateTimeFormatter GRAPHITE_RENDERING_ENGINES_DATETIME_FORMAT =  DateTimeFormat.forPattern("HH:mm_yyyyMMdd")

    static transactional = true

    /**
     * Iterates over all existing GraphiteServers. If they got any GraphiteEventSourcePath,
     * these will be used to fetch new Events from.
     * @param createBatchActivity defines if a BatchActivity should be created and updated
     * @param lastNMinutes Defines oldest Events that get fetched from EventSources.
     * @see de.iteratec.osm.report.external.GraphiteEventCollectorJob
     */
    public void fetchGraphiteEvents(boolean createBatchActivity, int lastNMinutes) {

        DateTime until = measuredValueUtilService.getNowInUtc()
        DateTime from = until.minusMinutes(lastNMinutes)
        String untilFormatted = GRAPHITE_RENDERING_ENGINES_DATETIME_FORMAT.print(until)
        String fromFormatted = GRAPHITE_RENDERING_ENGINES_DATETIME_FORMAT.print(from)

        def graphiteServers = GraphiteServer.list()
        int size = graphiteServers.size()
        BatchActivity activity = batchActivityService.getActiveBatchActivity(this.class,new Date().getTime(),Activity.CREATE,"Fetch Graphite Events",createBatchActivity)

        graphiteServers.eachWithIndex {server,index ->
            activity.updateStatus(["progress": batchActivityService.calculateProgress(size,index+1), "stage": "Delete JobResults"])
            server.graphiteEventSourcePaths.each {eventSourcePath->
                createEvents(eventSourcePath, server, fromFormatted, untilFormatted)
            }
        }

    }
    /**
     * Fetches the Events from the given GraphiteServer within the path.
     * @param eventSourcePath
     * @param server
     * @param from
     * @param until
     * @return List of fetched Events, they are already saved
     */
    private List<Event> createEvents(GraphiteEventSourcePath eventSourcePath, GraphiteServer server, String from, String until){
        def events = []
        def json = getEventJSON(eventSourcePath.targetMetricName, server, from, until)
        json.each{
            String shortName = it.target
            it.datapoints.findAll{it[0]}.each{point ->
                long unixTimeStamp = Long.parseLong(point[1] as String) * 1000L
                events << eventDaoService.createEvent(
                        "${eventSourcePath.staticPrefix}${shortName}",
                        new DateTime(unixTimeStamp),
                        "Read from Graphite: ${shortName} [${eventSourcePath.targetMetricName}]",
                        false,
                        eventSourcePath.jobGroups)
            }
        }
        return events
    }
    /**
     * Calls the path from a GraphiteServer and converts it into a JSONObject
     * @param eventSourceMetricName Name of the metric to fetch.
     * @param server GraphiteServer
     * @param from
     * @param until
     * @return
     */
    private Object getEventJSON(String eventSourceMetricName, GraphiteServer server, String from, String until){

        String webappUrl = createWebappUrlOf(server)
        LinkedHashMap<String, String> queryParams = ['format': 'json', 'from': from, 'until': until, 'target': eventSourceMetricName]

        return httpRequestService.getJsonResponse(webappUrl, server.webappPathToRenderingEngine, queryParams)
    }

    String createWebappUrlOf(GraphiteServer server){
{'fraction_non_alphanumeric': 0.07515936664610323, 'fraction_numerical': 0.0029816985399958875, 'mean_word_length': 3.42337426102774, 'pattern_counts': {'":': 4, '<': 11, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1568732', 'n_tokens_mistral': 2655, 'n_tokens_neox': 2565, 'n_words': 702}