<commit_before>package org.umlg.sqlg.step;

import com.google.common.base.Preconditions;
import com.google.common.collect.LinkedListMultimap;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
import org.apache.tinkerpop.gremlin.process.traversal.Traverser;
import org.apache.tinkerpop.gremlin.process.traversal.step.util.AbstractStep;
import org.apache.tinkerpop.gremlin.process.traversal.traverser.TraverserRequirement;
import org.apache.tinkerpop.gremlin.process.traversal.util.FastNoSuchElementException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.umlg.sqlg.sql.parse.ReplacedStep;
import org.umlg.sqlg.sql.parse.ReplacedStepTree;
import org.umlg.sqlg.sql.parse.SchemaTableTree;
import org.umlg.sqlg.strategy.Emit;
import org.umlg.sqlg.strategy.SqlgComparatorHolder;
import org.umlg.sqlg.structure.*;

import java.util.*;

/**
 * Date: 2014/08/15
 * Time: 8:10 PM
 */
public class SqlgVertexStep<E extends SqlgElement> extends AbstractStep implements SqlgStep {

    private static Logger logger = LoggerFactory.getLogger(SqlgVertexStep.class.getName());
    private SqlgGraph sqlgGraph;

    //This holds the head/start traversers per SchemaTable.
    //A query is executed per SchemaTable
    private Map<SchemaTable, List<Traverser.Admin<E>>> heads = new LinkedHashMap<>();

    //Holds all the incoming head/start elements start index. i.e. the index in which it was added to starts.
    private LinkedHashMap<Long, Traverser.Admin<E>> startIndexTraverserAdminMap = new LinkedHashMap<>();
    //important to start with 1 here to distinguish is from the default long being 0
    private long startIndex = 1;

    /**
     * Needs to be a multi map as TinkerPop will add new starts mid traversal.
     * look at TestRepeatStepVertexOut#testUntilRepeat
     */
    private LinkedListMultimap<SchemaTable, ListIterator<List<Emit<E>>>> schemaTableElements = LinkedListMultimap.create();

    //This holds, for each SchemaTable, a list of RecordId's ids and the start elements' index.
    //It is used to generate the select statements, 'VALUES' and ORDER BY 'index' sql
    private Map<SchemaTable, List<Pair<Long, Long>>> schemaTableParentIds = new LinkedHashMap<>();
    private List<SqlgElement> startElements = new ArrayList<>();

    private List<ReplacedStep<?, ?>> replacedSteps = new ArrayList<>();
    private ReplacedStepTree replacedStepTree;

    private Emit<E> toEmit = null;
    private ListIterator<List<Emit<E>>> elementIterator;

    private List<Emit<E>> traversers = new ArrayList<>();
    private ListIterator<Emit<E>> traversersLstIterator;

    private ReplacedStep<?, ?> lastReplacedStep;
    private long rangeCount = 0;
    private boolean eagerLoad = false;
    private boolean isForMultipleQueries = false;

    public SqlgVertexStep(final Traversal.Admin traversal) {
        super(traversal);
        this.sqlgGraph = (SqlgGraph) traversal.getGraph().get();
    }

    @Override
    protected Traverser.Admin<E> processNextStart() {
        if (this.starts.hasNext()) {
            barrierTheHeads();
            constructQueryPerSchemaTable();
        }
        while (true) {
            if (this.traversersLstIterator != null && this.traversersLstIterator.hasNext()) {
                Emit<E> emit = this.traversersLstIterator.next();
                this.labels = emit.getLabels();
                if (applyRange(emit)) {
                    continue;
                }
                return emit.getTraverser();
            }
            if (!this.eagerLoad && (this.elementIterator != null)) {
                if (this.elementIterator.hasNext()) {
                    this.traversers.clear();
                    this.traversersLstIterator = internalLoad();
                }
            }
            if (this.traversersLstIterator != null && this.traversersLstIterator.hasNext()) {
                Emit<E> emit = this.traversersLstIterator.next();
                this.labels = emit.getLabels();
                if (applyRange(emit)) {
                    continue;
                }
                return emit.getTraverser();
            } else {
                Iterator<Map.Entry<SchemaTable, ListIterator<List<Emit<E>>>>> schemaTableIteratorEntry = this.schemaTableElements.entries().iterator();
                if (schemaTableIteratorEntry.hasNext()) {
                    this.elementIterator = schemaTableIteratorEntry.next().getValue();
                    schemaTableIteratorEntry.remove();
                    if (this.eagerLoad) {
                        if (logger.isDebugEnabled()) {
                            logger.debug("eager load is true");
                        }
                        eagerLoad();
                        Collections.sort(this.traversers);
                        this.traversersLstIterator = this.traversers.listIterator();
                    }
                    this.lastReplacedStep = this.replacedSteps.get(this.replacedSteps.size() - 1);
                } else {
                    if (!this.starts.hasNext()) {
                        throw FastNoSuchElementException.instance();
                    } else {
                        throw new IllegalStateException("BUG: this should never happen.");
                    }
                }
            }
        }
    }

    private void barrierTheHeads() {
        //these collections are only used for the current starts.
        this.heads.clear();
        this.schemaTableParentIds.clear();
        while (this.starts.hasNext()) {
            Traverser.Admin<E> h = this.starts.next();
            E value = h.get();
            this.startElements.add(value);
            SchemaTable schemaTable = value.getSchemaTablePrefixed();
            List<Traverser.Admin<E>> traverserList = this.heads.computeIfAbsent(schemaTable, k -> new ArrayList<>());
            traverserList.add(h);
            List<Pair<Long, Long>> parentIdList = this.schemaTableParentIds.computeIfAbsent(schemaTable, k -> new ArrayList<>());
            parentIdList.add(Pair.of(((RecordId) value.id()).getId(), this.startIndex));
            this.startIndexTraverserAdminMap.put(this.startIndex++, h);<commit_msg>cosmetic<commit_after>package org.umlg.sqlg.step;

import com.google.common.base.Preconditions;
import com.google.common.collect.LinkedListMultimap;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
import org.apache.tinkerpop.gremlin.process.traversal.Traverser;
import org.apache.tinkerpop.gremlin.process.traversal.step.util.AbstractStep;
import org.apache.tinkerpop.gremlin.process.traversal.traverser.TraverserRequirement;
import org.apache.tinkerpop.gremlin.process.traversal.util.FastNoSuchElementException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.umlg.sqlg.sql.parse.ReplacedStep;
import org.umlg.sqlg.sql.parse.ReplacedStepTree;
import org.umlg.sqlg.sql.parse.SchemaTableTree;
import org.umlg.sqlg.strategy.Emit;
import org.umlg.sqlg.strategy.SqlgComparatorHolder;
import org.umlg.sqlg.structure.*;

import java.util.*;

/**
 * Date: 2014/08/15
 * Time: 8:10 PM
 */
public class SqlgVertexStep<E extends SqlgElement> extends AbstractStep implements SqlgStep {

    private static Logger logger = LoggerFactory.getLogger(SqlgVertexStep.class.getName());
    private SqlgGraph sqlgGraph;

    //This holds the head/start traversers per SchemaTable.
    //A query is executed per SchemaTable
    private Map<SchemaTable, List<Traverser.Admin<E>>> heads = new LinkedHashMap<>();

    //Holds all the incoming head/start elements start index. i.e. the index in which it was added to starts.
    private LinkedHashMap<Long, Traverser.Admin<E>> startIndexTraverserAdminMap = new LinkedHashMap<>();
    //important to start with 1 here to distinguish is from the default long being 0
    private long startIndex = 1;

    /**
     * Needs to be a multi map as TinkerPop will add new starts mid traversal.
     * look at TestRepeatStepVertexOut#testUntilRepeat
     */
    private LinkedListMultimap<SchemaTable, ListIterator<List<Emit<E>>>> schemaTableElements = LinkedListMultimap.create();

    //This holds, for each SchemaTable, a list of RecordId's ids and the start elements' index.
    //It is used to generate the select statements, 'VALUES' and ORDER BY 'index' sql
    private Map<SchemaTable, List<Pair<Long, Long>>> schemaTableParentIds = new LinkedHashMap<>();

    private List<ReplacedStep<?, ?>> replacedSteps = new ArrayList<>();
    private ReplacedStepTree replacedStepTree;

    private Emit<E> toEmit = null;
    private ListIterator<List<Emit<E>>> elementIterator;

    private List<Emit<E>> traversers = new ArrayList<>();
    private ListIterator<Emit<E>> traversersLstIterator;

    private ReplacedStep<?, ?> lastReplacedStep;
    private long rangeCount = 0;
    private boolean eagerLoad = false;
    private boolean isForMultipleQueries = false;

    public SqlgVertexStep(final Traversal.Admin traversal) {
        super(traversal);
        this.sqlgGraph = (SqlgGraph) traversal.getGraph().get();
    }

    @Override
    protected Traverser.Admin<E> processNextStart() {
        if (this.starts.hasNext()) {
            barrierTheHeads();
            constructQueryPerSchemaTable();
        }
        while (true) {
            if (this.traversersLstIterator != null && this.traversersLstIterator.hasNext()) {
                Emit<E> emit = this.traversersLstIterator.next();
                this.labels = emit.getLabels();
                if (applyRange(emit)) {
                    continue;
                }
                return emit.getTraverser();
            }
            if (!this.eagerLoad && (this.elementIterator != null)) {
                if (this.elementIterator.hasNext()) {
                    this.traversers.clear();
                    this.traversersLstIterator = internalLoad();
                }
            }
            if (this.traversersLstIterator != null && this.traversersLstIterator.hasNext()) {
                Emit<E> emit = this.traversersLstIterator.next();
                this.labels = emit.getLabels();
                if (applyRange(emit)) {
                    continue;
                }
                return emit.getTraverser();
            } else {
                Iterator<Map.Entry<SchemaTable, ListIterator<List<Emit<E>>>>> schemaTableIteratorEntry = this.schemaTableElements.entries().iterator();
                if (schemaTableIteratorEntry.hasNext()) {
                    this.elementIterator = schemaTableIteratorEntry.next().getValue();
                    schemaTableIteratorEntry.remove();
                    if (this.eagerLoad) {
                        if (logger.isDebugEnabled()) {
                            logger.debug("eager load is true");
                        }
                        eagerLoad();
                        Collections.sort(this.traversers);
                        this.traversersLstIterator = this.traversers.listIterator();
                    }
                    this.lastReplacedStep = this.replacedSteps.get(this.replacedSteps.size() - 1);
                } else {
                    if (!this.starts.hasNext()) {
                        throw FastNoSuchElementException.instance();
                    } else {
                        throw new IllegalStateException("BUG: this should never happen.");
                    }
                }
            }
        }
    }

    private void barrierTheHeads() {
        //these collections are only used for the current starts.
        this.heads.clear();
        this.schemaTableParentIds.clear();
        while (this.starts.hasNext()) {
            Traverser.Admin<E> h = this.starts.next();
            E value = h.get();
            SchemaTable schemaTable = value.getSchemaTablePrefixed();
            List<Traverser.Admin<E>> traverserList = this.heads.computeIfAbsent(schemaTable, k -> new ArrayList<>());
            traverserList.add(h);
            List<Pair<Long, Long>> parentIdList = this.schemaTableParentIds.computeIfAbsent(schemaTable, k -> new ArrayList<>());
            parentIdList.add(Pair.of(((RecordId) value.id()).getId(), this.startIndex));
            this.startIndexTraverserAdminMap.put(this.startIndex++, h);
{'fraction_non_alphanumeric': 0.10672834997159321, 'fraction_numerical': 0.0030841652463274084, 'mean_word_length': 2.4612359550561798, 'pattern_counts': {'":': 0, '<': 93, '<?xml version=': 0, '>': 97, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3955051', 'n_tokens_mistral': 3472, 'n_tokens_neox': 3312, 'n_words': 711}