<commit_before>package uk.co.omegaprime;

import au.com.bytecode.opencsv.CSVReader;
import uk.co.omegaprime.thunder.*;
import uk.co.omegaprime.thunder.schema.*;

import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.time.LocalDate;
import java.util.*;

public class BitemporalSparseLoader {
    private static final Schema<String> ID_BB_GLOBAL_SCHEMA = new Latin1StringSchema(20);
    private static final Schema<Integer> SOURCE_ID_SCHEMA = IntegerSchema.INSTANCE;

    public static class SourceTemporalFieldKey {
        public static Schema<SourceTemporalFieldKey> SCHEMA = Schema.zipWith(ID_BB_GLOBAL_SCHEMA, SourceTemporalFieldKey::getIDBBGlobal,
                                                                             SOURCE_ID_SCHEMA,    SourceTemporalFieldKey::getSourceID,
                                                                             SourceTemporalFieldKey::new);

        private final String idBBGlobal;
        private final int sourceID;

        public SourceTemporalFieldKey(String idBBGlobal, int sourceID) {
            this.idBBGlobal = idBBGlobal;
            this.sourceID = sourceID;
        }

        public String getIDBBGlobal() { return idBBGlobal; }
        public int getSourceID() { return sourceID; }
    }

    public static class SparseSourceTemporalFieldValue<T> {
        public static <T> Schema<SparseSourceTemporalFieldValue<T>> schema(Schema<T> tSchema) {
            return Schema.zipWith(Schema.nullable(IntegerSchema.INSTANCE), SparseSourceTemporalFieldValue::getToSourceID,
                                  tSchema,                                 SparseSourceTemporalFieldValue::getValue,
                                  SparseSourceTemporalFieldValue::new);
        }

        private final Integer toSourceID;
        private final T value;

        public SparseSourceTemporalFieldValue(Integer toSourceID, T value) {
            this.toSourceID = toSourceID;
            this.value = value;
        }

        public Integer getToSourceID() { return toSourceID; }
        public T getValue() { return value; }
    }

    // TODO: the interface of this is inconsistent with SparseTemporalCursor. With STC you setPosition
    // but don't learn about whether you are positioned or not until you actually do an operation,
    // with this one you moveTo and then immediately learn whether you are positioned or not.
    public static class SparseSourceTemporalCursor<V> {
        private final Cursor<SourceTemporalFieldKey, SparseSourceTemporalFieldValue<V>> cursor;
        private final SubcursorView<String, Integer, SparseSourceTemporalFieldValue<V>> subcursor;
        private final int currentSourceID;

        private boolean positioned;

        public SparseSourceTemporalCursor(Cursor<SourceTemporalFieldKey, SparseSourceTemporalFieldValue<V>> cursor, int currentSourceID) {
            this.cursor = cursor;
            this.subcursor = new SubcursorView<>(cursor, ID_BB_GLOBAL_SCHEMA, SOURCE_ID_SCHEMA);
            this.currentSourceID = currentSourceID;
        }

        public boolean moveTo(String idBBGlobal) {
            positioned = false;
            subcursor.setPosition(idBBGlobal);

            if (subcursor.moveFloor(currentSourceID)) {
                final SparseSourceTemporalFieldValue<V> value = subcursor.getValue();
                if (value.getToSourceID() == null || value.getToSourceID() > currentSourceID) {
                    positioned = true;
                }
            }

            return positioned;
        }

        public String getKey() {
            return cursor.getKey().getIDBBGlobal();
        }

        public V getValue() {
            if (!positioned) throw new IllegalStateException("May not getValue() if we aren't positioned");
            return subcursor.getValue().getValue();
        }

        public boolean delete() {
            if (positioned) {
                subcursor.put(new SparseSourceTemporalFieldValue<V>(currentSourceID, getValue()));
                positioned = false;
                return true;
            } else {
                return false;
            }
        }

        public V put(V value) {
            final V existingValue;
            if (positioned) {
                existingValue = getValue();
                if (Objects.equals(existingValue, value)) {
                    return value;
                }
                subcursor.put(new SparseSourceTemporalFieldValue<V>(currentSourceID, existingValue));
            } else {
                existingValue = null;
            }

            subcursor.put(currentSourceID, new SparseSourceTemporalFieldValue<>(null, value));
            return existingValue;
        }

        private boolean isAlive() {
            SparseSourceTemporalFieldValue<V> temporalValue = cursor.getValue();
            return (cursor.getKey().getSourceID() <= currentSourceID && (temporalValue.getToSourceID() == null || temporalValue.getToSourceID() > currentSourceID));
        }

        private boolean scanForwardForAlive() {
            do {
                if (isAlive()) {
                    subcursor.setPosition(cursor.getKey().getIDBBGlobal());
                    return true;<commit_msg>Fix a small bug in the source temporal cursor that could be triggered if you tried to write twice to the same place in the same source<commit_after>package uk.co.omegaprime;

import au.com.bytecode.opencsv.CSVReader;
import uk.co.omegaprime.thunder.*;
import uk.co.omegaprime.thunder.schema.*;

import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.time.LocalDate;
import java.util.*;

public class BitemporalSparseLoader {
    private static final Schema<String> ID_BB_GLOBAL_SCHEMA = new Latin1StringSchema(20);
    private static final Schema<Integer> SOURCE_ID_SCHEMA = IntegerSchema.INSTANCE;

    public static class SourceTemporalFieldKey {
        public static Schema<SourceTemporalFieldKey> SCHEMA = Schema.zipWith(ID_BB_GLOBAL_SCHEMA, SourceTemporalFieldKey::getIDBBGlobal,
                                                                             SOURCE_ID_SCHEMA,    SourceTemporalFieldKey::getSourceID,
                                                                             SourceTemporalFieldKey::new);

        private final String idBBGlobal;
        private final int sourceID;

        public SourceTemporalFieldKey(String idBBGlobal, int sourceID) {
            this.idBBGlobal = idBBGlobal;
            this.sourceID = sourceID;
        }

        public String getIDBBGlobal() { return idBBGlobal; }
        public int getSourceID() { return sourceID; }
    }

    public static class SparseSourceTemporalFieldValue<T> {
        public static <T> Schema<SparseSourceTemporalFieldValue<T>> schema(Schema<T> tSchema) {
            return Schema.zipWith(Schema.nullable(IntegerSchema.INSTANCE), SparseSourceTemporalFieldValue::getToSourceID,
                                  tSchema,                                 SparseSourceTemporalFieldValue::getValue,
                                  SparseSourceTemporalFieldValue::new);
        }

        private final Integer toSourceID;
        private final T value;

        public SparseSourceTemporalFieldValue(Integer toSourceID, T value) {
            this.toSourceID = toSourceID;
            this.value = value;
        }

        public Integer getToSourceID() { return toSourceID; }
        public T getValue() { return value; }
    }

    // TODO: the interface of this is inconsistent with SparseTemporalCursor. With STC you setPosition
    // but don't learn about whether you are positioned or not until you actually do an operation,
    // with this one you moveTo and then immediately learn whether you are positioned or not.
    public static class SparseSourceTemporalCursor<V> {
        private final Cursor<SourceTemporalFieldKey, SparseSourceTemporalFieldValue<V>> cursor;
        private final SubcursorView<String, Integer, SparseSourceTemporalFieldValue<V>> subcursor;
        private final int currentSourceID;

        private boolean positioned;

        public SparseSourceTemporalCursor(Cursor<SourceTemporalFieldKey, SparseSourceTemporalFieldValue<V>> cursor, int currentSourceID) {
            this.cursor = cursor;
            this.subcursor = new SubcursorView<>(cursor, ID_BB_GLOBAL_SCHEMA, SOURCE_ID_SCHEMA);
            this.currentSourceID = currentSourceID;
        }

        public boolean moveTo(String idBBGlobal) {
            positioned = false;
            subcursor.setPosition(idBBGlobal);

            if (subcursor.moveFloor(currentSourceID)) {
                final SparseSourceTemporalFieldValue<V> value = subcursor.getValue();
                if (value.getToSourceID() == null || value.getToSourceID() > currentSourceID) {
                    positioned = true;
                }
            }

            return positioned;
        }

        public String getKey() {
            return cursor.getKey().getIDBBGlobal();
        }

        public V getValue() {
            if (!positioned) throw new IllegalStateException("May not getValue() if we aren't positioned");
            return subcursor.getValue().getValue();
        }

        public boolean delete() {
            if (positioned) {
                truncateExistingSubcursorRange(getValue());
                positioned = false;
                return true;
            } else {
                return false;
            }
        }

        public V put(V value) {
            final V existingValue;
            if (positioned) {
                existingValue = getValue();
                if (Objects.equals(existingValue, value)) {
                    return value;
                }
                truncateExistingSubcursorRange(existingValue);
            } else {
                existingValue = null;
            }

            subcursor.put(currentSourceID, new SparseSourceTemporalFieldValue<>(null, value));
            return existingValue;
        }

        private void truncateExistingSubcursorRange(V existingValue) {
            if (subcursor.getKey() == currentSourceID) {
                subcursor.delete();
            } else {
                subcursor.put(new SparseSourceTemporalFieldValue<V>(currentSourceID, existingValue));
            }
        }

        private boolean isAlive() {
            SparseSourceTemporalFieldValue<V> temporalValue = cursor.getValue();
            return (cursor.getKey().getSourceID() <= currentSourceID && (temporalValue.getToSourceID() == null || temporalValue.getToSourceID() > currentSourceID));
        }

        private boolean scanForwardForAlive() {
            do {
                if (isAlive()) {
                    subcursor.setPosition(cursor.getKey().getIDBBGlobal());
                    return true;
{'fraction_non_alphanumeric': 0.07649167969109129, 'fraction_numerical': 0.0005516226900799853, 'mean_word_length': 2.1421143847487003, 'pattern_counts': {'":': 0, '<': 46, '<?xml version=': 0, '>': 48, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2005974', 'n_tokens_mistral': 2783, 'n_tokens_neox': 2658, 'n_words': 645}