<commit_before>package ua.com.fielden.platform.gis.gps.actors;

import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;

import ua.com.fielden.platform.gis.gps.AbstractAvlMessage;

/**
 * A holder of "blackout" messages.
 *
 * @author TG Team
 *
 * @param <T>
 */
public class Blackout<T extends AbstractAvlMessage> {
    private T start;
    private T finish;
    private final List<T> messages = new ArrayList<>();
    private final List<Packet<T>> packets = new ArrayList<>();
    private final MessagesComparator<T> messagesComparator;

    public Blackout(final MessagesComparator<T> messagesComparator) {
	this.messagesComparator = messagesComparator;
    }

    public void add(final Packet<T> packet) {
	packets.add(packet);
	messages.addAll(packet.getMessages());
	if (start == null || (start != null && packet.getStart().getGpsTime().getTime() < start.getGpsTime().getTime())) {
	    start = packet.getStart();
	}

	if (finish == null || (finish != null && packet.getFinish().getGpsTime().getTime() > finish.getGpsTime().getTime())) {
	    finish = packet.getFinish();
	}
    }

    public Collection<T> reset() {
	if (messages.size() > 0) {
	    final Set<Long> gpsTimes = new HashSet<>();
	    final SortedSet<T> result = new TreeSet<T>(messagesComparator);
	    for (final T message : messages) {
		if (!gpsTimes.contains(message.getGpsTime().getTime())) {
		    result.add(message);
		    gpsTimes.add(message.getGpsTime().getTime());
		}

	    }
	    start = null;
	    finish = null;
	    messages.clear();
	    packets.clear();
	    return result;
	} else {
	    return Collections.emptySet();
	}
    }

    public List<T> getMessages() {
	return messages;
    }

    public T getStart() {
	return start;
    }

    public T getFinish() {
	return finish;
    }
}<commit_msg>Implemented additional logic to avoid recurring messages from historical data.<commit_after>package ua.com.fielden.platform.gis.gps.actors;

import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;

import ua.com.fielden.platform.gis.gps.AbstractAvlMessage;

/**
 * A holder of "blackout" messages.
 *
 * @author T<NAME>
 *
 * @param <T>
 */
public class Blackout<T extends AbstractAvlMessage> {
    private T start;
    private T finish;
    private final SortedSet<T> messages;
    private final Set<Long> gpsTimes = new HashSet<>();

    public Blackout(final MessagesComparator<T> messagesComparator) {
	this.messages = new TreeSet<>(messagesComparator);
    }

    public void add(final Packet<T> packet) {
	for (final T message : packet.getMessages()) {
	    if (!gpsTimes.contains(message.getGpsTime().getTime())) {
		messages.add(message);
		gpsTimes.add(message.getGpsTime().getTime());

		if (start == null || (start != null && message.getGpsTime().getTime() < start.getGpsTime().getTime())) {
		    start = message;
		}

		if (finish == null || (finish != null && message.getGpsTime().getTime() > finish.getGpsTime().getTime())) {
		    finish = message;
		}
	    }
	}
    }

    public Collection<T> reset() {
	if (messages.size() > 0) {
	    final List<T> result = new ArrayList<>();
	    for (final T message : messages) {
		result.add(message);
	    }
	    start = null;
	    finish = null;
	    messages.clear();
	    gpsTimes.clear();
	    return result;
	} else {
	    return Collections.emptySet();
	}
    }

    public SortedSet<T> getMessages() {
	return messages;
    }

    public T getStart() {
	return start;
    }

    public T getFinish() {
	return finish;
    }
}
{'fraction_non_alphanumeric': 0.14184023200632745, 'fraction_numerical': 0.0005272871078302136, 'mean_word_length': 3.489940828402367, 'pattern_counts': {'":': 0, '<': 34, '<?xml version=': 0, '>': 36, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1093537', 'n_tokens_mistral': 1264, 'n_tokens_neox': 1206, 'n_words': 280}