<commit_before>package org.vertexium.inmemory;

import com.google.common.collect.Maps;
import org.vertexium.*;
import org.vertexium.HistoricalPropertyValue;
import org.vertexium.HistoricalPropertyValue.HistoricalPropertyValueBuilder;
import org.vertexium.inmemory.mutations.*;
import org.vertexium.property.MutablePropertyImpl;
import org.vertexium.property.StreamingPropertyValue;
import org.vertexium.property.StreamingPropertyValueRef;
import org.vertexium.util.ConvertingIterable;
import org.vertexium.util.FilterIterable;
import org.vertexium.util.IncreasingTime;
import org.vertexium.util.LookAheadIterable;

import java.io.Serializable;
import java.util.*;

public abstract class InMemoryTableElement<TElement extends InMemoryElement> implements Serializable {
    private final String id;
    private TreeSet<Mutation> mutations = new TreeSet<>();

    protected InMemoryTableElement(String id) {
        this.id = id;
    }

    public String getId() {
        return id;
    }

    public void addAll(Mutation... newMutations) {
        Collections.addAll(mutations, newMutations);
    }

    public long getFirstTimestamp() {
        return findFirstMutation(ElementTimestampMutation.class).getTimestamp();
    }

    protected <T extends Mutation> T findLastMutation(Class<T> clazz) {
        T last = null;
        for (Mutation m : this.mutations) {
            if (clazz.isAssignableFrom(m.getClass())) {
                //noinspection unchecked
                last = (T) m;
            }
        }
        return last;
    }

    protected <T extends Mutation> T findFirstMutation(Class<T> clazz) {
        for (Mutation m : this.mutations) {
            if (clazz.isAssignableFrom(m.getClass())) {
                //noinspection unchecked
                return (T) m;
            }
        }
        return null;
    }

    protected <T extends Mutation> Iterable<T> findMutations(final Class<T> clazz) {
        return new ConvertingIterable<Mutation, T>(new FilterIterable<Mutation>(this.mutations) {
            @Override
            protected boolean isIncluded(Mutation m) {
                return clazz.isAssignableFrom(m.getClass());
            }
        }) {
            @Override
            protected T convert(Mutation o) {
                //noinspection unchecked
                return (T) o;
            }
        };
    }

    public Visibility getVisibility() {
        return findLastMutation(AlterVisibilityMutation.class).getNewVisibility();
    }

    public long getTimestamp() {
        return findLastMutation(ElementTimestampMutation.class).getTimestamp();
    }

    private List<PropertyMutation> findPropertyMutations(Property p) {
        return findPropertyMutations(p.getKey(), p.getName(), p.getVisibility());
    }

    public Property deleteProperty(String key, String name, Authorizations authorizations) {
        return deleteProperty(key, name, null, authorizations);
    }

    public Property getProperty(String key, String name, Visibility visibility, Authorizations authorizations) {
        List<PropertyMutation> propertyMutations = findPropertyMutations(key, name, visibility);
        if (propertyMutations == null || propertyMutations.size() == 0) {
            return null;
        }
        return toProperty(propertyMutations, true, authorizations);
    }

    public Property deleteProperty(String key, String name, Visibility visibility, Authorizations authorizations) {
        Property p = getProperty(key, name, visibility, authorizations);
        if (p != null) {
            deleteProperty(p);
        }
        return p;
    }

    protected void deleteProperty(Property p) {
        List<PropertyMutation> propertyMutations = findPropertyMutations(p);
        this.mutations.removeAll(propertyMutations);
    }

    private List<PropertyMutation> findPropertyMutations(String key, String name, Visibility visibility) {
        List<PropertyMutation> results = new ArrayList<>();
        for (Mutation m : this.mutations) {
            if (!(m instanceof PropertyMutation)) {
                continue;
            }
            PropertyMutation pm = (PropertyMutation) m;
            if ((key == null || pm.getPropertyKey().equals(key))
                    && (name == null || pm.getPropertyName().equals(name))
                    && (visibility == null || pm.getPropertyVisibility().equals(visibility))) {
                results.add(pm);
            }
        }
        return results;
    }

    public Iterable<HistoricalPropertyValue> getHistoricalPropertyValues(
            String key,
            String name,
            Visibility visibility,
            Long startTime,
            Long endTime,
            Authorizations authorizations
    ) {
        List<PropertyMutation> propertyMutations = findPropertyMutations(key, name, visibility);
        List<HistoricalPropertyValue> historicalPropertyValues = new ArrayList<>();

        /*
         * There is the expectation that historical property values are a snapshot of the property in
         * time. This method attempts to reconstruct the property current state from mutations.
         */
        Map<String , HistoricalPropertyValueBuilder> currentPropertyBuilders = Maps.newHashMap();
        Set<Visibility> hiddenVisibilities = new HashSet<>();

        for (PropertyMutation m : propertyMutations) {
            String propertyIdentifier = m.getPropertyKey() + m.getPropertyName();
            HistoricalPropertyValueBuilder builder = currentPropertyBuilders.get(propertyIdentifier);
            if(builder == null) {
                builder = new HistoricalPropertyValueBuilder(m.getPropertyKey(), m.getPropertyName(), m.getTimestamp());
                currentPropertyBuilders.put(propertyIdentifier, builder);
            }

            if (startTime != null && m.getTimestamp() < startTime) {
                continue;
            }
            if (endTime != null && m.getTimestamp() > endTime) {
                continue;
            }
            if (!canRead(m.getVisibility(), authorizations)) {
                continue;
            }

            if (m instanceof SoftDeletePropertyMutation) {
                builder.isDeleted(true);
                historicalPropertyValues.add(builder.build());
            } else if (m instanceof AddPropertyMetadataMutation) {
                builder.metadata(((AddPropertyMetadataMutation) m).getMetadata());
            } else if (m instanceof MarkPropertyHiddenMutation) {
                // Ignore
            } else if (m instanceof MarkPropertyVisibleMutation) {
                // Ignore
            } else if (m instanceof AddPropertyValueMutation) {
                AddPropertyValueMutation apvm = (AddPropertyValueMutation) m;
                Object value = apvm.getValue();
                value = loadIfStreamingPropertyValue(value);

                builder.propertyVisibility(m.getVisibility())
                        .timestamp(m.getTimestamp())
                        .value(value)
                        .metadata(apvm.getMetadata())
                        .hiddenVisibilities(hiddenVisibilities)
                        .isDeleted(false);

                // Property modifications use a soft delete immediately followed by an AddPropertyValueMutation.
                // If the condition occurs, remove the delete event from the set.
                if(historicalPropertyValues.size() > 0) {
                    HistoricalPropertyValue last = historicalPropertyValues.get(historicalPropertyValues.size() - 1);
                    if (propertyIdentifier.equals(last.getPropertyKey() + last.getPropertyName()) && last.isDeleted()) {
                        historicalPropertyValues.remove(historicalPropertyValues.size() - 1);
                    }
                }

                historicalPropertyValues.add(builder.build());
            } else {
                throw new VertexiumException("Unhandled PropertyMutation: " + m.getClass().getName());
            }
        }

        Collections.reverse(historicalPropertyValues);
        return historicalPropertyValues;
    }

    public Iterable<Property> getProperties(final boolean includeHidden, Long endTime, final Authorizations authorizations) {
        final TreeMap<String, List<PropertyMutation>> propertiesMutations = new TreeMap<>();
        for (PropertyMutation m : findMutations(PropertyMutation.class)) {
            if (endTime != null && m.getTimestamp() > endTime) {
                continue;<commit_msg>History enhancement patch: Ignore workspace visibility changes<commit_after>package org.vertexium.inmemory;

import com.google.common.collect.Maps;
import org.vertexium.*;
import org.vertexium.HistoricalPropertyValue;
import org.vertexium.HistoricalPropertyValue.HistoricalPropertyValueBuilder;
import org.vertexium.inmemory.mutations.*;
import org.vertexium.property.MutablePropertyImpl;
import org.vertexium.property.StreamingPropertyValue;
import org.vertexium.property.StreamingPropertyValueRef;
import org.vertexium.util.ConvertingIterable;
import org.vertexium.util.FilterIterable;
import org.vertexium.util.IncreasingTime;
import org.vertexium.util.LookAheadIterable;

import java.io.Serializable;
import java.util.*;

public abstract class InMemoryTableElement<TElement extends InMemoryElement> implements Serializable {
    private final String id;
    private TreeSet<Mutation> mutations = new TreeSet<>();

    protected InMemoryTableElement(String id) {
        this.id = id;
    }

    public String getId() {
        return id;
    }

    public void addAll(Mutation... newMutations) {
        Collections.addAll(mutations, newMutations);
    }

    public long getFirstTimestamp() {
        return findFirstMutation(ElementTimestampMutation.class).getTimestamp();
    }

    protected <T extends Mutation> T findLastMutation(Class<T> clazz) {
        T last = null;
        for (Mutation m : this.mutations) {
            if (clazz.isAssignableFrom(m.getClass())) {
                //noinspection unchecked
                last = (T) m;
            }
        }
        return last;
    }

    protected <T extends Mutation> T findFirstMutation(Class<T> clazz) {
        for (Mutation m : this.mutations) {
            if (clazz.isAssignableFrom(m.getClass())) {
                //noinspection unchecked
                return (T) m;
            }
        }
        return null;
    }

    protected <T extends Mutation> Iterable<T> findMutations(final Class<T> clazz) {
        return new ConvertingIterable<Mutation, T>(new FilterIterable<Mutation>(this.mutations) {
            @Override
            protected boolean isIncluded(Mutation m) {
                return clazz.isAssignableFrom(m.getClass());
            }
        }) {
            @Override
            protected T convert(Mutation o) {
                //noinspection unchecked
                return (T) o;
            }
        };
    }

    public Visibility getVisibility() {
        return findLastMutation(AlterVisibilityMutation.class).getNewVisibility();
    }

    public long getTimestamp() {
        return findLastMutation(ElementTimestampMutation.class).getTimestamp();
    }

    private List<PropertyMutation> findPropertyMutations(Property p) {
        return findPropertyMutations(p.getKey(), p.getName(), p.getVisibility());
    }

    public Property deleteProperty(String key, String name, Authorizations authorizations) {
        return deleteProperty(key, name, null, authorizations);
    }

    public Property getProperty(String key, String name, Visibility visibility, Authorizations authorizations) {
        List<PropertyMutation> propertyMutations = findPropertyMutations(key, name, visibility);
        if (propertyMutations == null || propertyMutations.size() == 0) {
            return null;
        }
        return toProperty(propertyMutations, true, authorizations);
    }

    public Property deleteProperty(String key, String name, Visibility visibility, Authorizations authorizations) {
        Property p = getProperty(key, name, visibility, authorizations);
        if (p != null) {
            deleteProperty(p);
        }
        return p;
    }

    protected void deleteProperty(Property p) {
        List<PropertyMutation> propertyMutations = findPropertyMutations(p);
        this.mutations.removeAll(propertyMutations);
    }

    private List<PropertyMutation> findPropertyMutations(String key, String name, Visibility visibility) {
        List<PropertyMutation> results = new ArrayList<>();
        for (Mutation m : this.mutations) {
            if (!(m instanceof PropertyMutation)) {
                continue;
            }
            PropertyMutation pm = (PropertyMutation) m;
            if ((key == null || pm.getPropertyKey().equals(key))
                    && (name == null || pm.getPropertyName().equals(name))
                    && (visibility == null || pm.getPropertyVisibility().equals(visibility))) {
                results.add(pm);
            }
        }
        return results;
    }

    public Iterable<HistoricalPropertyValue> getHistoricalPropertyValues(
            String key,
            String name,
            Visibility visibility,
            Long startTime,
            Long endTime,
            Authorizations authorizations
    ) {
        List<PropertyMutation> propertyMutations = findPropertyMutations(key, name, visibility);
        List<HistoricalPropertyValue> historicalPropertyValues = new ArrayList<>();

        /*
         * There is the expectation that historical property values are a snapshot of the property in
         * time. This method attempts to reconstruct the property current state from mutations.
         */
        Map<String , HistoricalPropertyValueBuilder> currentPropertyBuilders = Maps.newHashMap();
        Set<Visibility> hiddenVisibilities = new HashSet<>();

        for (PropertyMutation m : propertyMutations) {
            String propertyIdentifier = m.getPropertyKey() + m.getPropertyName();
            HistoricalPropertyValueBuilder builder = currentPropertyBuilders.get(propertyIdentifier);
            if(builder == null) {
                builder = new HistoricalPropertyValueBuilder(m.getPropertyKey(), m.getPropertyName(), m.getTimestamp());
                currentPropertyBuilders.put(propertyIdentifier, builder);
            }

            if (startTime != null && m.getTimestamp() < startTime) {
                continue;
            }
            if (endTime != null && m.getTimestamp() > endTime) {
                continue;
            }
            if (!canRead(m.getVisibility(), authorizations)) {
                continue;
            }
            // Ignore workspace interactions to avoid duplicated entries
            if(m.getVisibility() != null && m.getPropertyVisibility().getVisibilityString().matches("(.*)WORKSPACE(.*)")) {
                continue;
            }

            if (m instanceof SoftDeletePropertyMutation) {
                builder.isDeleted(true);
                builder.timestamp(m.getTimestamp());
                historicalPropertyValues.add(builder.build());
            } else if (m instanceof AddPropertyMetadataMutation) {
                builder.metadata(((AddPropertyMetadataMutation) m).getMetadata());
                builder.timestamp(m.getTimestamp());
            } else if (m instanceof MarkPropertyHiddenMutation) {
                // Ignore
            } else if (m instanceof MarkPropertyVisibleMutation) {
                // Ignore
            } else if (m instanceof AddPropertyValueMutation) {
                AddPropertyValueMutation apvm = (AddPropertyValueMutation) m;
                Object value = apvm.getValue();
                value = loadIfStreamingPropertyValue(value);

                builder.propertyVisibility(m.getPropertyVisibility())
                        .timestamp(m.getTimestamp())
                        .value(value)
                        .metadata(apvm.getMetadata())
                        .hiddenVisibilities(hiddenVisibilities)
                        .isDeleted(false);

                // Property modifications use a soft delete immediately followed by an AddPropertyValueMutation.
                // If the condition occurs, remove the delete event from the set.
                if(historicalPropertyValues.size() > 0) {
                    HistoricalPropertyValue last = historicalPropertyValues.get(historicalPropertyValues.size() - 1);
                    if (propertyIdentifier.equals(last.getPropertyKey() + last.getPropertyName()) && last.isDeleted()) {
                        historicalPropertyValues.remove(historicalPropertyValues.size() - 1);
                    }
                }

                historicalPropertyValues.add(builder.build());
            } else {
                throw new VertexiumException("Unhandled PropertyMutation: " + m.getClass().getName());
            }
        }

        Collections.reverse(historicalPropertyValues);
        return historicalPropertyValues;
    }

    public Iterable<Property> getProperties(final boolean includeHidden, Long endTime, final Authorizations authorizations) {
        final TreeMap<String, List<PropertyMutation>> propertiesMutations = new TreeMap<>();
        for (PropertyMutation m : findMutations(PropertyMutation.class)) {
            if (endTime != null && m.getTimestamp() > endTime) {
                continue;
{'fraction_non_alphanumeric': 0.09076286764705882, 'fraction_numerical': 0.00045955882352941176, 'mean_word_length': 2.3627583542592236, 'pattern_counts': {'":': 0, '<': 63, '<?xml version=': 0, '>': 67, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6962381', 'n_tokens_mistral': 4272, 'n_tokens_neox': 4131, 'n_words': 970}