<commit_before>/*
 * Copyright 2011-2014 the original author or authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.glowroot.local.ui;

import java.io.IOException;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Locale;

import checkers.igj.quals.ReadOnly;
import checkers.nullness.quals.MonotonicNonNull;
import checkers.nullness.quals.Nullable;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Function;
import com.google.common.base.Strings;
import com.google.common.base.Ticker;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import com.google.common.collect.Ordering;
import com.google.common.io.CharStreams;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import org.glowroot.collector.TraceCollectorImpl;
import org.glowroot.common.Clock;
import org.glowroot.common.ObjectMappers;
import org.glowroot.local.store.SnapshotDao;
import org.glowroot.local.store.TracePoint;
import org.glowroot.local.store.TracePointQuery;
import org.glowroot.local.store.TracePointQuery.StringComparator;
import org.glowroot.markers.Singleton;
import org.glowroot.trace.TraceRegistry;
import org.glowroot.trace.model.Trace;
import org.glowroot.trace.model.Trace.TraceAttribute;

import static org.glowroot.common.Nullness.castNonNull;

/**
 * Json service to read trace point data, bound under /backend/trace/points.
 * 
 * @author <NAME>
 * @since 0.5
 */
@Singleton
@JsonService
class TracePointJsonService {

    private static final Logger logger = LoggerFactory.getLogger(TracePointJsonService.class);
    @ReadOnly
    private static final ObjectMapper mapper = ObjectMappers.create();
    private static final int NANOSECONDS_PER_SECOND = 1000000000;

    private final SnapshotDao snapshotDao;
    private final TraceRegistry traceRegistry;
    private final TraceCollectorImpl traceCollector;
    private final Ticker ticker;
    private final Clock clock;

    TracePointJsonService(SnapshotDao snapshotDao, TraceRegistry traceRegistry,
            TraceCollectorImpl traceCollector, Ticker ticker, Clock clock) {
        this.snapshotDao = snapshotDao;
        this.traceRegistry = traceRegistry;
        this.traceCollector = traceCollector;
        this.ticker = ticker;
        this.clock = clock;
    }

    @GET("/backend/trace/points")
    String getPoints(String content) throws IOException {
        logger.debug("getPoints(): content={}", content);
        TracePointRequest request =
                ObjectMappers.readRequiredValue(mapper, content, TracePointRequest.class);
        return new Handler(request).handle();
    }

    private class Handler {

        private final TracePointRequest request;
        private long requestAt;
        private long low;
        private long high;
        @MonotonicNonNull
        private StringComparator transactionNameComparator;
        @MonotonicNonNull
        private StringComparator headlineComparator;
        @MonotonicNonNull
        private StringComparator errorComparator;
        @MonotonicNonNull
        private StringComparator userComparator;
        @MonotonicNonNull
        private StringComparator attributeValueComparator;

        public Handler(TracePointRequest request) {
            this.request = request;
        }

        private String handle() throws IOException {
            requestAt = clock.currentTimeMillis();
            if (request.getFrom() < 0) {
                request.setFrom(requestAt + request.getFrom());
            }
            low = (long) Math.ceil(request.getLow() * NANOSECONDS_PER_SECOND);
            high = request.getHigh() == 0 ? Long.MAX_VALUE : (long) Math.floor(request.getHigh()
                    * NANOSECONDS_PER_SECOND);
            String transactionNameComparator = request.getTransactionNameComparator();
            if (transactionNameComparator != null) {
                this.transactionNameComparator =
                        StringComparator.valueOf(transactionNameComparator
                                .toUpperCase(Locale.ENGLISH));
            }
            String headlineComparator = request.getHeadlineComparator();
            if (headlineComparator != null) {
                this.headlineComparator =
                        StringComparator.valueOf(headlineComparator.toUpperCase(Locale.ENGLISH));
            }
            String errorComparator = request.getErrorComparator();
            if (errorComparator != null) {
                this.errorComparator =
                        StringComparator.valueOf(errorComparator.toUpperCase(Locale.ENGLISH));
            }
            String userComparator = request.getUserComparator();
            if (userComparator != null) {
                this.userComparator =
                        StringComparator.valueOf(userComparator.toUpperCase(Locale.ENGLISH));
            }
            String attributeValueComparator = request.getAttributeValueComparator();
            if (attributeValueComparator != null) {
                this.attributeValueComparator = StringComparator.valueOf(
                        attributeValueComparator.toUpperCase(Locale.ENGLISH));
            }
            boolean captureActiveTraces = shouldCaptureActiveTraces();
            List<Trace> activeTraces = Lists.newArrayList();
            long captureTime = 0;
            long captureTick = 0;
            if (captureActiveTraces) {
                // capture active traces first to make sure that none are missed in the transition
                // between active and pending/stored (possible duplicates are removed below)
                activeTraces = getMatchingActiveTraces();
                // take capture timings after the capture to make sure there are no traces captured
                // that start after the recorded capture time (resulting in negative duration)
                captureTime = clock.currentTimeMillis();
                captureTick = ticker.read();
            }
            if (request.getTo() == 0) {
                request.setTo(requestAt);
            }
            List<TracePoint> points = getStoredAndPendingPoints(captureActiveTraces);
            removeDuplicatesBetweenActiveTracesAndPoints(activeTraces, points);
            boolean limitExceeded = (points.size() + activeTraces.size() > request.getLimit());
            if (points.size() + activeTraces.size() > request.getLimit()) {
                // points is already ordered, so just drop the last few items
                // always include all active traces
                points = points.subList(0, request.getLimit() - activeTraces.size());
            }
            return writeResponse(points, activeTraces, captureTime, captureTick, limitExceeded);
        }

        private boolean shouldCaptureActiveTraces() {
            return (request.getTo() == 0 || request.getTo() > requestAt)
                    && request.getFrom() < requestAt;
        }

        private List<TracePoint> getStoredAndPendingPoints(boolean captureActiveTraces) {
            List<TracePoint> matchingPendingPoints;
            // it only seems worth looking at pending traces if request asks for active traces
            if (captureActiveTraces) {
                // important to grab pending traces before stored points to ensure none are
                // missed in the transition between pending and stored
                matchingPendingPoints = getMatchingPendingPoints();
            } else {
                matchingPendingPoints = ImmutableList.of();
            }
            TracePointQuery query = new TracePointQuery(request.getFrom(), request.getTo(), low,
                    high, request.isBackground(), request.isErrorOnly(), request.isFineOnly(),
                    transactionNameComparator, request.getTransactionName(), headlineComparator,
                    request.getHeadline(), errorComparator, request.getError(), userComparator,
                    request.getUser(), request.getAttributeName(), attributeValueComparator,
                    request.getAttributeValue(), request.getLimit() + 1);
            List<TracePoint> points = snapshotDao.readPoints(query);
            // create single merged and limited list of points
            List<TracePoint> orderedPoints = Lists.newArrayList(points);
            for (TracePoint pendingPoint : matchingPendingPoints) {
                insertIntoOrderedPoints(pendingPoint, orderedPoints);
            }
            return orderedPoints;
        }

        private List<Trace> getMatchingActiveTraces() {
            List<Trace> activeTraces = Lists.newArrayList();
            for (Trace trace : traceRegistry.getTraces()) {
                if (traceCollector.shouldStore(trace)
                        && matchesDuration(trace)
                        && matchesBackground(trace)
                        && matchesErrorOnly(trace)
                        && matchesFineOnly(trace)
                        && matchesTransactionName(trace)
                        && matchesHeadline(trace)
                        && matchesUser(trace)
                        && matchesAttribute(trace)) {
                    activeTraces.add(trace);
                }
            }
            Collections.sort(activeTraces,
                    Ordering.natural().onResultOf(new Function<Trace, Long>() {
                        @Override
                        public Long apply(@Nullable Trace trace) {
                            // sorting activeTraces which is List<@NonNull Trace>
                            castNonNull(trace);
                            return trace.getStartTick();
                        }
                    }));
            if (activeTraces.size() > request.getLimit()) {
                activeTraces = activeTraces.subList(0, request.getLimit());
            }
            return activeTraces;
        }

        private List<TracePoint> getMatchingPendingPoints() {
            List<TracePoint> points = Lists.newArrayList();
            for (Trace trace : traceCollector.getPendingCompleteTraces()) {
                if (matchesDuration(trace)
                        && matchesBackground(trace)
                        && matchesErrorOnly(trace)
                        && matchesFineOnly(trace)
                        && matchesTransactionName(trace)
                        && matchesHeadline(trace)
                        && matchesUser(trace)
                        && matchesAttribute(trace)) {
                    points.add(TracePoint.from(trace.getId(), clock.currentTimeMillis(),
                            trace.getDuration(), trace.getError() != null));
                }
            }
            return points;
        }

        private boolean matchesDuration(Trace trace) {
            long duration = trace.getDuration();
            return duration >= low && duration <= high;
        }

        private boolean matchesBackground(Trace trace) {
            Boolean background = request.isBackground();
            return background == null || background == trace.isBackground();
        }

        private boolean matchesErrorOnly(Trace trace) {
            return !request.isErrorOnly() || trace.getError() != null;
        }

        private boolean matchesFineOnly(Trace trace) {
            return !request.isFineOnly() || trace.isFine();
        }

        private boolean matchesTransactionName(Trace trace) {
            return matchesUsingStringComparator(transactionNameComparator,
                    request.getTransactionName(), trace.getTransactionName());
        }

        private boolean matchesHeadline(Trace trace) {
            return matchesUsingStringComparator(headlineComparator, request.getHeadline(),
                    trace.getHeadline());
        }

        private boolean matchesUser(Trace trace) {
            return matchesUsingStringComparator(userComparator, request.getUser(), trace.getUser());
        }

        private boolean matchesAttribute(Trace trace) {
            if (Strings.isNullOrEmpty(request.getAttributeName())
                    && (attributeValueComparator == null
                    || Strings.isNullOrEmpty(request.getAttributeValue()))) {
                // no attribute filter<commit_msg>Fix search matching on active traces<commit_after>/*
 * Copyright 2011-2014 the original author or authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.glowroot.local.ui;

import java.io.IOException;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Locale;

import checkers.igj.quals.ReadOnly;
import checkers.nullness.quals.MonotonicNonNull;
import checkers.nullness.quals.Nullable;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Function;
import com.google.common.base.Strings;
import com.google.common.base.Ticker;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import com.google.common.collect.Ordering;
import com.google.common.io.CharStreams;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import org.glowroot.collector.TraceCollectorImpl;
import org.glowroot.common.Clock;
import org.glowroot.common.ObjectMappers;
import org.glowroot.local.store.SnapshotDao;
import org.glowroot.local.store.TracePoint;
import org.glowroot.local.store.TracePointQuery;
import org.glowroot.local.store.TracePointQuery.StringComparator;
import org.glowroot.markers.Singleton;
import org.glowroot.trace.TraceRegistry;
import org.glowroot.trace.model.Trace;
import org.glowroot.trace.model.Trace.TraceAttribute;

import static org.glowroot.common.Nullness.castNonNull;

/**
 * Json service to read trace point data, bound under /backend/trace/points.
 * 
 * @author <NAME>
 * @since 0.5
 */
@Singleton
@JsonService
class TracePointJsonService {

    private static final Logger logger = LoggerFactory.getLogger(TracePointJsonService.class);
    @ReadOnly
    private static final ObjectMapper mapper = ObjectMappers.create();
    private static final int NANOSECONDS_PER_SECOND = 1000000000;

    private final SnapshotDao snapshotDao;
    private final TraceRegistry traceRegistry;
    private final TraceCollectorImpl traceCollector;
    private final Ticker ticker;
    private final Clock clock;

    TracePointJsonService(SnapshotDao snapshotDao, TraceRegistry traceRegistry,
            TraceCollectorImpl traceCollector, Ticker ticker, Clock clock) {
        this.snapshotDao = snapshotDao;
        this.traceRegistry = traceRegistry;
        this.traceCollector = traceCollector;
        this.ticker = ticker;
        this.clock = clock;
    }

    @GET("/backend/trace/points")
    String getPoints(String content) throws IOException {
        logger.debug("getPoints(): content={}", content);
        TracePointRequest request =
                ObjectMappers.readRequiredValue(mapper, content, TracePointRequest.class);
        return new Handler(request).handle();
    }

    private class Handler {

        private final TracePointRequest request;
        private long requestAt;
        private long low;
        private long high;
        @MonotonicNonNull
        private StringComparator transactionNameComparator;
        @MonotonicNonNull
        private StringComparator headlineComparator;
        @MonotonicNonNull
        private StringComparator errorComparator;
        @MonotonicNonNull
        private StringComparator userComparator;
        @MonotonicNonNull
        private StringComparator attributeValueComparator;

        public Handler(TracePointRequest request) {
            this.request = request;
        }

        private String handle() throws IOException {
            requestAt = clock.currentTimeMillis();
            if (request.getFrom() < 0) {
                request.setFrom(requestAt + request.getFrom());
            }
            low = (long) Math.ceil(request.getLow() * NANOSECONDS_PER_SECOND);
            high = request.getHigh() == 0 ? Long.MAX_VALUE : (long) Math.floor(request.getHigh()
                    * NANOSECONDS_PER_SECOND);
            String transactionNameComparator = request.getTransactionNameComparator();
            if (transactionNameComparator != null) {
                this.transactionNameComparator =
                        StringComparator.valueOf(transactionNameComparator
                                .toUpperCase(Locale.ENGLISH));
            }
            String headlineComparator = request.getHeadlineComparator();
            if (headlineComparator != null) {
                this.headlineComparator =
                        StringComparator.valueOf(headlineComparator.toUpperCase(Locale.ENGLISH));
            }
            String errorComparator = request.getErrorComparator();
            if (errorComparator != null) {
                this.errorComparator =
                        StringComparator.valueOf(errorComparator.toUpperCase(Locale.ENGLISH));
            }
            String userComparator = request.getUserComparator();
            if (userComparator != null) {
                this.userComparator =
                        StringComparator.valueOf(userComparator.toUpperCase(Locale.ENGLISH));
            }
            String attributeValueComparator = request.getAttributeValueComparator();
            if (attributeValueComparator != null) {
                this.attributeValueComparator = StringComparator.valueOf(
                        attributeValueComparator.toUpperCase(Locale.ENGLISH));
            }
            boolean captureActiveTraces = shouldCaptureActiveTraces();
            List<Trace> activeTraces = Lists.newArrayList();
            long captureTime = 0;
            long captureTick = 0;
            if (captureActiveTraces) {
                // capture active traces first to make sure that none are missed in the transition
                // between active and pending/stored (possible duplicates are removed below)
                activeTraces = getMatchingActiveTraces();
                // take capture timings after the capture to make sure there are no traces captured
                // that start after the recorded capture time (resulting in negative duration)
                captureTime = clock.currentTimeMillis();
                captureTick = ticker.read();
            }
            if (request.getTo() == 0) {
                request.setTo(requestAt);
            }
            List<TracePoint> points = getStoredAndPendingPoints(captureActiveTraces);
            removeDuplicatesBetweenActiveTracesAndPoints(activeTraces, points);
            boolean limitExceeded = (points.size() + activeTraces.size() > request.getLimit());
            if (points.size() + activeTraces.size() > request.getLimit()) {
                // points is already ordered, so just drop the last few items
                // always include all active traces
                points = points.subList(0, request.getLimit() - activeTraces.size());
            }
            return writeResponse(points, activeTraces, captureTime, captureTick, limitExceeded);
        }

        private boolean shouldCaptureActiveTraces() {
            return (request.getTo() == 0 || request.getTo() > requestAt)
                    && request.getFrom() < requestAt;
        }

        private List<TracePoint> getStoredAndPendingPoints(boolean captureActiveTraces) {
            List<TracePoint> matchingPendingPoints;
            // it only seems worth looking at pending traces if request asks for active traces
            if (captureActiveTraces) {
                // important to grab pending traces before stored points to ensure none are
                // missed in the transition between pending and stored
                matchingPendingPoints = getMatchingPendingPoints();
            } else {
                matchingPendingPoints = ImmutableList.of();
            }
            TracePointQuery query = new TracePointQuery(request.getFrom(), request.getTo(), low,
                    high, request.isBackground(), request.isErrorOnly(), request.isFineOnly(),
                    transactionNameComparator, request.getTransactionName(), headlineComparator,
                    request.getHeadline(), errorComparator, request.getError(), userComparator,
                    request.getUser(), request.getAttributeName(), attributeValueComparator,
                    request.getAttributeValue(), request.getLimit() + 1);
            List<TracePoint> points = snapshotDao.readPoints(query);
            // create single merged and limited list of points
            List<TracePoint> orderedPoints = Lists.newArrayList(points);
            for (TracePoint pendingPoint : matchingPendingPoints) {
                insertIntoOrderedPoints(pendingPoint, orderedPoints);
            }
            return orderedPoints;
        }

        private List<Trace> getMatchingActiveTraces() {
            List<Trace> activeTraces = Lists.newArrayList();
            for (Trace trace : traceRegistry.getTraces()) {
                if (traceCollector.shouldStore(trace)
                        && matchesDuration(trace)
                        && matchesBackground(trace)
                        && matchesErrorOnly(trace)
                        && matchesFineOnly(trace)
                        && matchesHeadline(trace)
                        && matchesTransactionName(trace)
                        && matchesError(trace)
                        && matchesUser(trace)
                        && matchesAttribute(trace)) {
                    activeTraces.add(trace);
                }
            }
            Collections.sort(activeTraces,
                    Ordering.natural().onResultOf(new Function<Trace, Long>() {
                        @Override
                        public Long apply(@Nullable Trace trace) {
                            // sorting activeTraces which is List<@NonNull Trace>
                            castNonNull(trace);
                            return trace.getStartTick();
                        }
                    }));
            if (activeTraces.size() > request.getLimit()) {
                activeTraces = activeTraces.subList(0, request.getLimit());
            }
            return activeTraces;
        }

        private List<TracePoint> getMatchingPendingPoints() {
            List<TracePoint> points = Lists.newArrayList();
            for (Trace trace : traceCollector.getPendingCompleteTraces()) {
                if (matchesDuration(trace)
                        && matchesBackground(trace)
                        && matchesErrorOnly(trace)
                        && matchesFineOnly(trace)
                        && matchesHeadline(trace)
                        && matchesTransactionName(trace)
                        && matchesError(trace)
                        && matchesUser(trace)
                        && matchesAttribute(trace)) {
                    points.add(TracePoint.from(trace.getId(), clock.currentTimeMillis(),
                            trace.getDuration(), trace.getError() != null));
                }
            }
            return points;
        }

        private boolean matchesDuration(Trace trace) {
            long duration = trace.getDuration();
            return duration >= low && duration <= high;
        }

        private boolean matchesBackground(Trace trace) {
            Boolean background = request.isBackground();
            return background == null || background == trace.isBackground();
        }

        private boolean matchesErrorOnly(Trace trace) {
            return !request.isErrorOnly() || trace.getError() != null;
        }

        private boolean matchesFineOnly(Trace trace) {
            return !request.isFineOnly() || trace.isFine();
        }

        private boolean matchesHeadline(Trace trace) {
            return matchesUsingStringComparator(headlineComparator, request.getHeadline(),
                    trace.getHeadline());
        }

        private boolean matchesTransactionName(Trace trace) {
            return matchesUsingStringComparator(transactionNameComparator,
                    request.getTransactionName(), trace.getTransactionName());
        }

        private boolean matchesError(Trace trace) {
            return matchesUsingStringComparator(errorComparator, request.getError(),
                    trace.getError());
        }

        private boolean matchesUser(Trace trace) {
            return matchesUsingStringComparator(userComparator, request.getUser(), trace.getUser());
        }

        private boolean matchesAttribute(Trace trace) {
            if (Strings.isNullOrEmpty(request.getAttributeName())
                    && (attributeValueComparator == null
                    || Strings.isNullOrEmpty(request.getAttributeValue()))) {
                // no attribute filter
{'fraction_non_alphanumeric': 0.08334615384615385, 'fraction_numerical': 0.002692307692307692, 'mean_word_length': 2.211189329381252, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3835386', 'n_tokens_mistral': 6167, 'n_tokens_neox': 6068, 'n_words': 1442}