<commit_before>package com.tinkerpop.gremlin.olap.util;

import com.tinkerpop.blueprints.Graph;
import com.tinkerpop.blueprints.computer.ComputeResult;
import com.tinkerpop.blueprints.computer.GraphComputer;
import com.tinkerpop.blueprints.computer.GraphMemory;
import com.tinkerpop.blueprints.util.StreamFactory;
import com.tinkerpop.gremlin.MicroPath;
import com.tinkerpop.gremlin.Pipeline;
import com.tinkerpop.gremlin.olap.GremlinCounters;
import com.tinkerpop.gremlin.olap.GremlinPaths;
import com.tinkerpop.gremlin.olap.GremlinVertexProgram;
import com.tinkerpop.gremlin.util.optimizers.HolderOptimizer;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import java.util.function.BiFunction;
import java.util.function.Supplier;

/**
 * @author <NAME> (http://markorodriguez.com)
 */
public class GremlinResult<T> implements Iterator<T> {

    private Iterator<T> itty;
    private final Supplier<Pipeline> gremlinSupplier;
    private final Graph graph;
    private final ComputeResult computeResult;

    public <K, V, R> GremlinResult(final Graph graph, final Supplier<Pipeline> gremlinSupplier, final BiFunction<K, Iterator<V>, R> reduction) {
        this.gremlinSupplier = gremlinSupplier;
        this.graph = graph;
        final GraphComputer computer = graph.compute();
        computer.program(GremlinVertexProgram.create().gremlin((Supplier) gremlinSupplier).build());
        if (null != reduction) computer.reduction(reduction);

        try {
            this.computeResult = computer.submit().get();
        } catch (Exception e) {
            throw new RuntimeException(e.getMessage(), e);
        }
        buildIterator();
    }

    public GremlinResult(final Graph graph, final Supplier<Pipeline> gremlinSupplier) {
        this(graph, gremlinSupplier, null);
    }

    public boolean hasNext() {
        return this.itty.hasNext();
    }

    public T next() {
        return this.itty.next();
    }

    public String toString() {
        return this.gremlinSupplier.get().toString();
    }

    private void buildIterator() {
        if (null != this.computeResult.getGraphMemory().getReductionMemory()) {
            final GraphMemory.ReductionMemory reductionMemory = this.computeResult.getGraphMemory().getReductionMemory();
            this.itty = StreamFactory.stream(reductionMemory.getKeys()).map(key -> Arrays.asList(key, reductionMemory.get(key).get())).iterator();
        } else if (HolderOptimizer.trackPaths(this.gremlinSupplier.get())) {
            final List list = new ArrayList();
            this.graph.query().vertices().forEach(vertex -> {
                StreamFactory.stream(vertex)
                        .map(v -> this.computeResult.getVertexMemory().<GremlinPaths>getProperty(v, GremlinVertexProgram.GREMLIN_TRACKER).orElse(null))
                        .filter(tracker -> null != tracker)
                        .forEach(tracker -> {
                            tracker.getDoneObjectTracks().entrySet().stream().forEach(entry -> {
                                entry.getValue().forEach(holder -> {
                                    if (entry.getKey() instanceof MicroPath) {
                                        list.add(((MicroPath) entry.getKey()).inflate(this.graph));
                                    } else {
                                        list.add(entry.getKey());
                                    }
                                });
                            });
                            tracker.getDoneGraphTracks().entrySet().stream().forEach(entry -> {
                                entry.getValue().forEach(holder -> list.add(holder.inflate(vertex).get()));
                            });
                        });
            });
            this.itty = list.iterator();
        } else {
            final List list = new ArrayList();
            this.graph.query().vertices().forEach(vertex -> {
                StreamFactory.stream(vertex)
                        .map(v -> this.computeResult.getVertexMemory().<GremlinCounters>getProperty(v, GremlinVertexProgram.GREMLIN_TRACKER).orElse(null))
                        .filter(tracker -> null != tracker)
                        .forEach(tracker -> {
                            tracker.getDoneObjectTracks().entrySet().stream().forEach(entry -> {
                                for (int i = 0; i < entry.getValue(); i++) {
                                    list.add(entry.getKey().get());
                                }
                            });
                            tracker.getDoneGraphTracks().entrySet().stream().forEach(entry -> {
                                for (int i = 0; i < entry.getValue(); i++) {
                                    list.add(entry.getKey().inflate(vertex).get());
                                }
                            });
                        });
            });
            this.itty = list.iterator();
        }
    }
}<commit_msg>GremlinResult is now a full iterator and not just a List.iterator().<commit_after>package com.tinkerpop.gremlin.olap.util;

import com.tinkerpop.blueprints.Graph;
import com.tinkerpop.blueprints.computer.ComputeResult;
import com.tinkerpop.blueprints.computer.GraphComputer;
import com.tinkerpop.blueprints.computer.GraphMemory;
import com.tinkerpop.blueprints.util.StreamFactory;
import com.tinkerpop.gremlin.MicroPath;
import com.tinkerpop.gremlin.Pipeline;
import com.tinkerpop.gremlin.olap.GremlinCounters;
import com.tinkerpop.gremlin.olap.GremlinPaths;
import com.tinkerpop.gremlin.olap.GremlinVertexProgram;
import com.tinkerpop.gremlin.util.optimizers.HolderOptimizer;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import java.util.function.BiFunction;
import java.util.function.Supplier;

/**
 * @author <NAME> (http://markorodriguez.com)
 */
public class GremlinResult<T> implements Iterator<T> {

    private Iterator<T> itty;
    private final Supplier<Pipeline> gremlinSupplier;
    private final Graph graph;
    private final ComputeResult computeResult;

    public <K, V, R> GremlinResult(final Graph graph, final Supplier<Pipeline> gremlinSupplier, final BiFunction<K, Iterator<V>, R> reduction) {
        this.gremlinSupplier = gremlinSupplier;
        this.graph = graph;
        final GraphComputer computer = graph.compute();
        computer.program(GremlinVertexProgram.create().gremlin((Supplier) gremlinSupplier).build());
        if (null != reduction) computer.reduction(reduction);

        try {
            this.computeResult = computer.submit().get();
        } catch (Exception e) {
            throw new RuntimeException(e.getMessage(), e);
        }
        buildIterator();
    }

    public GremlinResult(final Graph graph, final Supplier<Pipeline> gremlinSupplier) {
        this(graph, gremlinSupplier, null);
    }

    public boolean hasNext() {
        return this.itty.hasNext();
    }

    public T next() {
        return this.itty.next();
    }

    public String toString() {
        return this.gremlinSupplier.get().toString();
    }

    private void buildIterator() {
        if (null != this.computeResult.getGraphMemory().getReductionMemory()) {
            final GraphMemory.ReductionMemory reductionMemory = this.computeResult.getGraphMemory().getReductionMemory();
            this.itty = StreamFactory.stream(reductionMemory.getKeys()).map(key -> Arrays.asList(key, reductionMemory.get(key).get())).iterator();
        } else if (HolderOptimizer.trackPaths(this.gremlinSupplier.get())) {
            this.itty = StreamFactory.stream(this.graph.query().vertices()).flatMap(vertex -> {
                return StreamFactory.stream(vertex)
                        .map(v -> this.computeResult.getVertexMemory().<GremlinPaths>getProperty(v, GremlinVertexProgram.GREMLIN_TRACKER).orElse(null))
                        .filter(tracker -> null != tracker)
                        .flatMap(tracker -> {
                            final List list = new ArrayList();
                            tracker.getDoneObjectTracks().entrySet().stream().forEach(entry -> {
                                entry.getValue().forEach(holder -> {
                                    if (entry.getKey() instanceof MicroPath) {
                                        list.add(((MicroPath) entry.getKey()).inflate(this.graph));
                                    } else {
                                        list.add(entry.getKey());
                                    }
                                });
                            });
                            tracker.getDoneGraphTracks().entrySet().stream().forEach(entry -> {
                                entry.getValue().forEach(holder -> list.add(holder.inflate(vertex).get()));
                            });
                            return list.stream();
                        });
            }).iterator();
        } else {
            this.itty = StreamFactory.stream(this.graph.query().vertices()).flatMap(vertex -> {
                return StreamFactory.stream(vertex)
                        .map(v -> this.computeResult.getVertexMemory().<GremlinCounters>getProperty(v, GremlinVertexProgram.GREMLIN_TRACKER).orElse(null))
                        .filter(tracker -> null != tracker)
                        .flatMap(tracker -> {
                            final List list = new ArrayList();
                            tracker.getDoneObjectTracks().entrySet().stream().forEach(entry -> {
                                for (int i = 0; i < entry.getValue(); i++) {
                                    list.add(entry.getKey().get());
                                }
                            });
                            tracker.getDoneGraphTracks().entrySet().stream().forEach(entry -> {
                                for (int i = 0; i < entry.getValue(); i++) {
                                    list.add(entry.getKey().inflate(vertex).get());
                                }
                            });
                            return list.stream();
                        });
            }).iterator();
        }
    }
}
{'fraction_non_alphanumeric': 0.11833740831295844, 'fraction_numerical': 0.00039119804400977997, 'mean_word_length': 1.8287690179806362, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 57, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 22, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1468130', 'n_tokens_mistral': 2715, 'n_tokens_neox': 2554, 'n_words': 426}