<commit_before>package de.qaware.chronix.server.benchmark.queryrunner;


import com.fasterxml.jackson.databind.ObjectMapper;
import de.qaware.chronix.database.BenchmarkDataSource;
import de.qaware.chronix.database.BenchmarkQuery;
import de.qaware.chronix.database.TimeSeries;
import de.qaware.chronix.server.benchmark.collector.StatsCollector;
import de.qaware.chronix.server.util.DockerCommandLineUtil;
import de.qaware.chronix.server.util.DockerStatsRecord;
import de.qaware.chronix.server.util.DockerStatsUtil;
import de.qaware.chronix.server.util.ServerSystemUtil;
import de.qaware.chronix.shared.DataModels.ImportRecordWrapper;
import de.qaware.chronix.shared.DataModels.Tuple;
import de.qaware.chronix.shared.QueryUtil.*;
import de.qaware.chronix.shared.ServerConfig.TSDBInterfaceHandler;
import org.glassfish.jersey.media.multipart.FormDataContentDisposition;
import org.glassfish.jersey.media.multipart.FormDataParam;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import org.apache.commons.fileupload.FileItem;
import org.apache.commons.fileupload.FileItemFactory;
import org.apache.commons.fileupload.FileUploadException;
import org.apache.commons.fileupload.disk.DiskFileItemFactory;
import org.apache.commons.fileupload.servlet.ServletFileUpload;

import javax.servlet.http.HttpServletRequest;
import javax.ws.rs.*;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import java.io.File;
import java.io.InputStream;
import java.util.*;

/**
 * Created by mcqueen666 on 15.06.16.
 */
@Path("/queryrunner")
@Produces(MediaType.APPLICATION_JSON)
public class QueryRunnerResource {

    private final Logger logger = LoggerFactory.getLogger(QueryRunnerResource.class);
    private final StatsCollector statsCollector = StatsCollector.getInstance();

    @POST
    @Path("performQuery")
    @Consumes(MediaType.APPLICATION_JSON)
    public Response performQuery(QueryRecord queryRecord){
        if(queryRecord != null) {
            TSDBInterfaceHandler tsdbInterfaceHandler = TSDBInterfaceHandler.getInstance();
            BenchmarkDataSource<Object> tsdb = tsdbInterfaceHandler.getTSDBInstance(queryRecord.getTsdbName());
            if (tsdb == null) {
                logger.error("No TSDB implementation with name " + queryRecord.getTsdbName() + " found on.");
                return Response.serverError().entity(new String[]{"No TSDB implementation with name " + queryRecord.getTsdbName() + " found on server!"}).build();
            }

            if(tsdb.setup(queryRecord.getIpAddress(),Integer.valueOf(queryRecord.getPortNumber()))) {
                logger.info("Performing query on {}",tsdb.getClass().getName());
                List<BenchmarkQuery> queryList = queryRecord.getQueryList();
                List<String> queryResults = new LinkedList<>();

                //get the query strings
                Map<Object, BenchmarkQuery> queryBenchmarkQueryMap = new HashMap<>();
                for (BenchmarkQuery benchmarkQuery : queryList) {
                    if (benchmarkQuery != null) {
                        queryBenchmarkQueryMap.put(tsdb.getQueryObject(benchmarkQuery), benchmarkQuery);
                    }
                }

                //start threaded background measurement
                DockerStatsUtil dockerStatsUtil = DockerStatsUtil.getInstance();
                Long startDiskUsage = dockerStatsUtil.estimateStorageSize(queryRecord.getTsdbName(), tsdb.getMappedStorageDirectoryPath());
                dockerStatsUtil.startDockerContainerMeasurement(DockerCommandLineUtil.getRunningContainerId(queryRecord.getTsdbName()));
                long startMilliseconds = System.currentTimeMillis();

                //perform the query mix
                for (Map.Entry<Object, BenchmarkQuery> entry : queryBenchmarkQueryMap.entrySet()) {

                    List<String> results = tsdb.performQuery(entry.getValue(), entry.getKey());
                    if (results != null && !results.isEmpty()) {
                        queryResults.addAll(results);
                    }
                }

                // end measurement
                long endMilliseconds = System.currentTimeMillis();
                tsdb.writeCachesToDisk();
                List<DockerStatsRecord> dockerMeasurement = dockerStatsUtil.stopDockerContainerMeasurement();
                Long endDiskUsage = dockerStatsUtil.estimateStorageSize(queryRecord.getTsdbName(), tsdb.getMappedStorageDirectoryPath());


                // edit and write the queryrecord to json file (threaded)
                queryRecord.setQueryTimeMilliseconds(endMilliseconds - startMilliseconds);
                if (startDiskUsage != -1 && endDiskUsage != -1) {
                    queryRecord.setDiskUsage(String.valueOf((endDiskUsage - startDiskUsage)));
                    queryRecord.setDiskUsageTotal(String.valueOf(endDiskUsage));
                }
                statsCollector.addQueryRecordEditJob(queryRecord, dockerMeasurement);

                tsdb.shutdown();
                if (!queryResults.isEmpty()) {
                    return Response.ok().entity(queryResults.toArray(new String[]{})).build();
                }
            }

        }
        logger.error("Error performing queries!");
        return Response.serverError().entity(new String[]{"Error performing queries!"}).build();<commit_msg>if diskUsage couldn't be measured, set value to -1 instead of null<commit_after>package de.qaware.chronix.server.benchmark.queryrunner;


import com.fasterxml.jackson.databind.ObjectMapper;
import de.qaware.chronix.database.BenchmarkDataSource;
import de.qaware.chronix.database.BenchmarkQuery;
import de.qaware.chronix.database.TimeSeries;
import de.qaware.chronix.server.benchmark.collector.StatsCollector;
import de.qaware.chronix.server.util.DockerCommandLineUtil;
import de.qaware.chronix.server.util.DockerStatsRecord;
import de.qaware.chronix.server.util.DockerStatsUtil;
import de.qaware.chronix.server.util.ServerSystemUtil;
import de.qaware.chronix.shared.DataModels.ImportRecordWrapper;
import de.qaware.chronix.shared.DataModels.Tuple;
import de.qaware.chronix.shared.QueryUtil.*;
import de.qaware.chronix.shared.ServerConfig.TSDBInterfaceHandler;
import org.glassfish.jersey.media.multipart.FormDataContentDisposition;
import org.glassfish.jersey.media.multipart.FormDataParam;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import org.apache.commons.fileupload.FileItem;
import org.apache.commons.fileupload.FileItemFactory;
import org.apache.commons.fileupload.FileUploadException;
import org.apache.commons.fileupload.disk.DiskFileItemFactory;
import org.apache.commons.fileupload.servlet.ServletFileUpload;

import javax.servlet.http.HttpServletRequest;
import javax.ws.rs.*;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import java.io.File;
import java.io.InputStream;
import java.util.*;

/**
 * Created by mcqueen666 on 15.06.16.
 */
@Path("/queryrunner")
@Produces(MediaType.APPLICATION_JSON)
public class QueryRunnerResource {

    private final Logger logger = LoggerFactory.getLogger(QueryRunnerResource.class);
    private final StatsCollector statsCollector = StatsCollector.getInstance();

    @POST
    @Path("performQuery")
    @Consumes(MediaType.APPLICATION_JSON)
    public Response performQuery(QueryRecord queryRecord){
        if(queryRecord != null) {
            TSDBInterfaceHandler tsdbInterfaceHandler = TSDBInterfaceHandler.getInstance();
            BenchmarkDataSource<Object> tsdb = tsdbInterfaceHandler.getTSDBInstance(queryRecord.getTsdbName());
            if (tsdb == null) {
                logger.error("No TSDB implementation with name " + queryRecord.getTsdbName() + " found on.");
                return Response.serverError().entity(new String[]{"No TSDB implementation with name " + queryRecord.getTsdbName() + " found on server!"}).build();
            }

            if(tsdb.setup(queryRecord.getIpAddress(),Integer.valueOf(queryRecord.getPortNumber()))) {
                logger.info("Performing query on {}",tsdb.getClass().getName());
                List<BenchmarkQuery> queryList = queryRecord.getQueryList();
                List<String> queryResults = new LinkedList<>();

                //get the query strings
                Map<Object, BenchmarkQuery> queryBenchmarkQueryMap = new HashMap<>();
                for (BenchmarkQuery benchmarkQuery : queryList) {
                    if (benchmarkQuery != null) {
                        queryBenchmarkQueryMap.put(tsdb.getQueryObject(benchmarkQuery), benchmarkQuery);
                    }
                }

                //start threaded background measurement
                DockerStatsUtil dockerStatsUtil = DockerStatsUtil.getInstance();
                Long startDiskUsage = dockerStatsUtil.estimateStorageSize(queryRecord.getTsdbName(), tsdb.getMappedStorageDirectoryPath());
                dockerStatsUtil.startDockerContainerMeasurement(DockerCommandLineUtil.getRunningContainerId(queryRecord.getTsdbName()));
                long startMilliseconds = System.currentTimeMillis();

                //perform the query mix
                for (Map.Entry<Object, BenchmarkQuery> entry : queryBenchmarkQueryMap.entrySet()) {

                    List<String> results = tsdb.performQuery(entry.getValue(), entry.getKey());
                    if (results != null && !results.isEmpty()) {
                        queryResults.addAll(results);
                    }
                }

                // end measurement
                long endMilliseconds = System.currentTimeMillis();
                tsdb.writeCachesToDisk();
                List<DockerStatsRecord> dockerMeasurement = dockerStatsUtil.stopDockerContainerMeasurement();
                Long endDiskUsage = dockerStatsUtil.estimateStorageSize(queryRecord.getTsdbName(), tsdb.getMappedStorageDirectoryPath());


                // edit and write the queryrecord to json file (threaded)
                queryRecord.setQueryTimeMilliseconds(endMilliseconds - startMilliseconds);
                if (startDiskUsage != -1 && endDiskUsage != -1) {
                    queryRecord.setDiskUsage(String.valueOf((endDiskUsage - startDiskUsage)));
                    queryRecord.setDiskUsageTotal(String.valueOf(endDiskUsage));
                } else {
                    queryRecord.setDiskUsage("-1");
                    queryRecord.setDiskUsageTotal("-1");
                }
                statsCollector.addQueryRecordEditJob(queryRecord, dockerMeasurement);

                tsdb.shutdown();
                if (!queryResults.isEmpty()) {
                    return Response.ok().entity(queryResults.toArray(new String[]{})).build();
                }
            }

        }
        logger.error("Error performing queries!");
        return Response.serverError().entity(new String[]{"Error performing queries!"}).build();
{'fraction_non_alphanumeric': 0.10272536687631027, 'fraction_numerical': 0.0026433324218393946, 'mean_word_length': 3.0043795620437956, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5111626', 'n_tokens_mistral': 2972, 'n_tokens_neox': 2813, 'n_words': 473}