<commit_before>package com.groupon.lex.metrics.api.endpoints;

import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import com.google.gson.annotations.SerializedName;
import static com.groupon.lex.metrics.ConfigSupport.quotedString;
import com.groupon.lex.metrics.MetricValue;
import com.groupon.lex.metrics.Tags;
import com.groupon.lex.metrics.config.ConfigurationException;
import com.groupon.lex.metrics.config.ParserSupport;
import com.groupon.lex.metrics.history.CollectHistory;
import com.groupon.lex.metrics.lib.BufferedIterator;
import com.groupon.lex.metrics.lib.SimpleMapEntry;
import com.groupon.lex.metrics.timeseries.TimeSeriesMetricDeltaSet;
import com.groupon.lex.metrics.timeseries.TimeSeriesMetricExpression;
import java.io.IOException;
import java.security.SecureRandom;
import java.util.Arrays;
import java.util.Collection;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import static java.util.Objects.requireNonNull;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import javax.servlet.AsyncContext;
import javax.servlet.ServletOutputStream;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import lombok.Getter;
import lombok.NonNull;
import lombok.RequiredArgsConstructor;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
import org.joda.time.Duration;

/**
 *
 * @author ariane
 */
public class ExprEval extends HttpServlet {
    private static final Logger LOG = Logger.getLogger(ExprEval.class.getName());
    private static final Cache<String, IteratorAndCookie> ITERATORS = CacheBuilder.newBuilder()
                .expireAfterAccess(5, TimeUnit.MINUTES)
                .softValues()
                .build();
    private static final int ITER_BUFCOUNT = 4;
    private static final AtomicLong ITERATORS_SEQUENCE = new AtomicLong();
    private final static String EXPR_PREFIX = "expr:";
    private final CollectHistory history_;

    public ExprEval(CollectHistory history) {
        history_ = requireNonNull(history);
    }

    private Stream<Collection<CollectHistory.NamedEvaluation>> eval_(Map<String, ? extends TimeSeriesMetricExpression> expr, Optional<DateTime> begin, Optional<DateTime> end, Optional<Duration> stepsize) {
        if (end.isPresent())
            return history_.evaluate(expr, begin.get(), end.get(), stepsize.orElse(Duration.ZERO));
        else if (begin.isPresent())
            return history_.evaluate(expr, begin.get(), stepsize.orElse(Duration.ZERO));
        else
            return history_.evaluate(expr, stepsize.orElse(Duration.ZERO));
    }

    private static Map<String, String> expressions_(HttpServletRequest hsr) {
        return hsr.getParameterMap().entrySet().stream()
                .flatMap(entry -> Arrays.stream(entry.getValue()).map(v -> SimpleMapEntry.create(entry.getKey(), v)))
                .filter(entry -> entry.getKey().startsWith(EXPR_PREFIX))
                .collect(Collectors.toMap(entry -> entry.getKey().substring(EXPR_PREFIX.length()), entry -> entry.getValue()));
    }

    private static Stream<TSV> encode_evaluation_result_(Stream<Collection<CollectHistory.NamedEvaluation>> evaluated) {
        return evaluated
                .map(c -> {
                    final DateTime dt = c.stream().findAny().map(CollectHistory.NamedEvaluation::getDatetime).orElseThrow(() -> new IllegalStateException("no expression result"));
                    return new TSV(dt, c);
                });
    }

    @Override
    protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOException {
        final IteratorAndCookie iterator;
        final String iteratorName;

        final Optional<Map.Entry<String, IteratorAndCookie>> cachedIterator = getCachedIterator(req);
        if (cachedIterator.isPresent()) {
            // Use cached iterator, ignore other arguments.
            iterator = cachedIterator.get().getValue();
            iteratorName = cachedIterator.get().getKey();
        } else {
            // Build new cached iterator from arguments.
            try {
                Map.Entry<String, IteratorAndCookie> newIterator = newIterator(req);
                iterator = newIterator.getValue();
                iteratorName = newIterator.getKey();
            } catch (Exception ex) {
                resp.sendError(HttpServletResponse.SC_BAD_REQUEST, ex.getMessage());
                return;
            }
        }

        final Optional<Long> deadline;
        final DateTime begin;
        try {
            deadline = Optional.ofNullable(req.getParameter("delay"))
                    .map(Long::parseLong)
                    .map(delay -> System.currentTimeMillis() + delay);
            begin = Optional.ofNullable(req.getParameter("begin"))
                    .map(Long::valueOf)
                    .map(msecSinceEpoch -> new DateTime(msecSinceEpoch, DateTimeZone.UTC))
                    .orElseGet(() -> new DateTime(0, DateTimeZone.UTC));
        } catch (Exception ex) {
            resp.sendError(HttpServletResponse.SC_BAD_REQUEST, ex.getMessage());
            return;
        }

        resp.setStatus(HttpServletResponse.SC_OK);
        resp.setContentType("application/json");
        resp.setCharacterEncoding("UTF-8");

        final AsyncContext ctx = req.startAsync();
        final ServletOutputStream out = resp.getOutputStream();

        final StreamingJsonListEntity<TSV> expr_result = new StreamingJsonListEntity<>(
                ctx,
                out,
                iterator.getIterator(),
                iteratorName,
                iterator.getCookie(),
                begin,
                iterator.getStepSize(),<commit_msg>Calculate deadling before potentially expensive setup of iterator.<commit_after>package com.groupon.lex.metrics.api.endpoints;

import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import com.google.gson.annotations.SerializedName;
import static com.groupon.lex.metrics.ConfigSupport.quotedString;
import com.groupon.lex.metrics.MetricValue;
import com.groupon.lex.metrics.Tags;
import com.groupon.lex.metrics.config.ConfigurationException;
import com.groupon.lex.metrics.config.ParserSupport;
import com.groupon.lex.metrics.history.CollectHistory;
import com.groupon.lex.metrics.lib.BufferedIterator;
import com.groupon.lex.metrics.lib.SimpleMapEntry;
import com.groupon.lex.metrics.timeseries.TimeSeriesMetricDeltaSet;
import com.groupon.lex.metrics.timeseries.TimeSeriesMetricExpression;
import java.io.IOException;
import java.security.SecureRandom;
import java.util.Arrays;
import java.util.Collection;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import static java.util.Objects.requireNonNull;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import javax.servlet.AsyncContext;
import javax.servlet.ServletOutputStream;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import lombok.Getter;
import lombok.NonNull;
import lombok.RequiredArgsConstructor;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
import org.joda.time.Duration;

/**
 *
 * @author ariane
 */
public class ExprEval extends HttpServlet {
    private static final Logger LOG = Logger.getLogger(ExprEval.class.getName());
    private static final Cache<String, IteratorAndCookie> ITERATORS = CacheBuilder.newBuilder()
                .expireAfterAccess(5, TimeUnit.MINUTES)
                .softValues()
                .build();
    private static final int ITER_BUFCOUNT = 4;
    private static final AtomicLong ITERATORS_SEQUENCE = new AtomicLong();
    private final static String EXPR_PREFIX = "expr:";
    private final CollectHistory history_;

    public ExprEval(CollectHistory history) {
        history_ = requireNonNull(history);
    }

    private Stream<Collection<CollectHistory.NamedEvaluation>> eval_(Map<String, ? extends TimeSeriesMetricExpression> expr, Optional<DateTime> begin, Optional<DateTime> end, Optional<Duration> stepsize) {
        if (end.isPresent())
            return history_.evaluate(expr, begin.get(), end.get(), stepsize.orElse(Duration.ZERO));
        else if (begin.isPresent())
            return history_.evaluate(expr, begin.get(), stepsize.orElse(Duration.ZERO));
        else
            return history_.evaluate(expr, stepsize.orElse(Duration.ZERO));
    }

    private static Map<String, String> expressions_(HttpServletRequest hsr) {
        return hsr.getParameterMap().entrySet().stream()
                .flatMap(entry -> Arrays.stream(entry.getValue()).map(v -> SimpleMapEntry.create(entry.getKey(), v)))
                .filter(entry -> entry.getKey().startsWith(EXPR_PREFIX))
                .collect(Collectors.toMap(entry -> entry.getKey().substring(EXPR_PREFIX.length()), entry -> entry.getValue()));
    }

    private static Stream<TSV> encode_evaluation_result_(Stream<Collection<CollectHistory.NamedEvaluation>> evaluated) {
        return evaluated
                .map(c -> {
                    final DateTime dt = c.stream().findAny().map(CollectHistory.NamedEvaluation::getDatetime).orElseThrow(() -> new IllegalStateException("no expression result"));
                    return new TSV(dt, c);
                });
    }

    @Override
    protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOException {
        final Optional<Long> deadline;
        final DateTime begin;
        try {
            deadline = Optional.ofNullable(req.getParameter("delay"))
                    .map(Long::parseLong)
                    .map(delay -> System.currentTimeMillis() + delay);
            begin = Optional.ofNullable(req.getParameter("begin"))
                    .map(Long::valueOf)
                    .map(msecSinceEpoch -> new DateTime(msecSinceEpoch, DateTimeZone.UTC))
                    .orElseGet(() -> new DateTime(0, DateTimeZone.UTC));
        } catch (Exception ex) {
            resp.sendError(HttpServletResponse.SC_BAD_REQUEST, ex.getMessage());
            return;
        }

        final IteratorAndCookie iterator;
        final String iteratorName;

        final Optional<Map.Entry<String, IteratorAndCookie>> cachedIterator = getCachedIterator(req);
        if (cachedIterator.isPresent()) {
            // Use cached iterator, ignore other arguments.
            iterator = cachedIterator.get().getValue();
            iteratorName = cachedIterator.get().getKey();
        } else {
            // Build new cached iterator from arguments.
            try {
                Map.Entry<String, IteratorAndCookie> newIterator = newIterator(req);
                iterator = newIterator.getValue();
                iteratorName = newIterator.getKey();
            } catch (Exception ex) {
                resp.sendError(HttpServletResponse.SC_BAD_REQUEST, ex.getMessage());
                return;
            }
        }

        resp.setStatus(HttpServletResponse.SC_OK);
        resp.setContentType("application/json");
        resp.setCharacterEncoding("UTF-8");

        final AsyncContext ctx = req.startAsync();
        final ServletOutputStream out = resp.getOutputStream();

        final StreamingJsonListEntity<TSV> expr_result = new StreamingJsonListEntity<>(
                ctx,
                out,
                iterator.getIterator(),
                iteratorName,
                iterator.getCookie(),
                begin,
                iterator.getStepSize(),
{'fraction_non_alphanumeric': 0.10586468674397263, 'fraction_numerical': 0.0006673896721448235, 'mean_word_length': 3.00802407221665, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 57, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2144663', 'n_tokens_mistral': 3174, 'n_tokens_neox': 3106, 'n_words': 624}