<commit_before>package org.yi.happy.archive.restore;

import java.util.ArrayList;
import java.util.Collections;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;

import org.yi.happy.annotate.MagicLiteral;
import org.yi.happy.archive.ByteString;
import org.yi.happy.archive.Bytes;
import org.yi.happy.archive.DataBlockParse;
import org.yi.happy.archive.block.Block;
import org.yi.happy.archive.block.DataBlock;
import org.yi.happy.archive.block.MapBlock;
import org.yi.happy.archive.key.FullKey;
import org.yi.happy.archive.key.FullKeyParse;

/**
 * The logic required to join data blocks back together.
 */
public class RestoreEngine {
    private static class Item {
        public FullKey key;
        public Long offset;

        public Item(FullKey key, Long offset) {
            this.key = key;
            this.offset = offset;
        }
    }

    private List<Item> items;
    private FragmentHandler handler;

    /**
     * Set up the logic to join data blocks back together.
     * 
     * @param key
     *            the key to start from.
     * @param handler
     *            where to send the fragments as they are encountered.
     */
    public RestoreEngine(FullKey key, FragmentHandler handler) {
        items = new ArrayList<Item>();
        items.add(new Item(key, 0l));

        this.handler = handler;
    }

    /**
     * @return the list of blocks that can be immediately processed. The first
     *         key in the list is the key that is needed if firstOnly is set to
     *         true.
     */
    public List<FullKey> getNeededNow() {
        LinkedHashSet<FullKey> needed = new LinkedHashSet<FullKey>();
        for (Item item : items) {
            if (item.offset == null) {
                continue;
            }

            needed.add(item.key);
        }
        return new ArrayList<FullKey>(needed);
    }

    /**
     * @return the list of blocks that are known to be needed later.
     */
    public List<FullKey> getNeededLater() {
        LinkedHashSet<FullKey> needed = new LinkedHashSet<FullKey>();
        for (Item item : items) {
            if (item.offset != null) {
                continue;
            }

            needed.add(item.key);
        }
        return new ArrayList<FullKey>(needed);
    }

    /**
     * @param index
     *            the index of the item.
     * @return the key needed by the item at the index in the list.
     */
    public FullKey getNeeded(int index) {
        return items.get(index).key;
    }

    /**
     * fetch the offset for the given item.<commit_msg>get the full needed list too<commit_after>package org.yi.happy.archive.restore;

import java.util.ArrayList;
import java.util.Collections;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;

import org.yi.happy.annotate.MagicLiteral;
import org.yi.happy.archive.ByteString;
import org.yi.happy.archive.Bytes;
import org.yi.happy.archive.DataBlockParse;
import org.yi.happy.archive.block.Block;
import org.yi.happy.archive.block.DataBlock;
import org.yi.happy.archive.block.MapBlock;
import org.yi.happy.archive.key.FullKey;
import org.yi.happy.archive.key.FullKeyParse;

/**
 * The logic required to join data blocks back together.
 */
public class RestoreEngine {
    private static class Item {
        public FullKey key;
        public Long offset;

        public Item(FullKey key, Long offset) {
            this.key = key;
            this.offset = offset;
        }
    }

    private List<Item> items;
    private FragmentHandler handler;

    /**
     * Set up the logic to join data blocks back together.
     * 
     * @param key
     *            the key to start from.
     * @param handler
     *            where to send the fragments as they are encountered.
     */
    public RestoreEngine(FullKey key, FragmentHandler handler) {
        items = new ArrayList<Item>();
        items.add(new Item(key, 0l));

        this.handler = handler;
    }

    /**
     * @return the list of blocks that can be immediately processed. The first
     *         key in the list is the key that is needed if firstOnly is set to
     *         true.
     */
    public List<FullKey> getNeededNow() {
        LinkedHashSet<FullKey> needed = new LinkedHashSet<FullKey>();
        for (Item item : items) {
            if (item.offset == null) {
                continue;
            }

            needed.add(item.key);
        }
        return new ArrayList<FullKey>(needed);
    }

    /**
     * @return the list of blocks that are known to be needed later.
     */
    public List<FullKey> getNeededLater() {
        LinkedHashSet<FullKey> needed = new LinkedHashSet<FullKey>();
        for (Item item : items) {
            if (item.offset != null) {
                continue;
            }

            needed.add(item.key);
        }
        return new ArrayList<FullKey>(needed);
    }

    /**
     * @return the list of blocks that are known to be needed, the blocks that
     *         can be immediately processed are at the beginning of the list.
     */
    public List<FullKey> getNeeded() {
        LinkedHashSet<FullKey> needed = new LinkedHashSet<FullKey>();
        LinkedHashSet<FullKey> later = new LinkedHashSet<FullKey>();
        for (Item item : items) {
            if (item.offset == null) {
                later.add(item.key);
            } else {
                needed.add(item.key);
            }
        }
        needed.addAll(later);
        return new ArrayList<FullKey>(needed);
    }

    /**
     * @param index
     *            the index of the item.
     * @return the key needed by the item at the index in the list.
     */
    public FullKey getNeeded(int index) {
        return items.get(index).key;
    }

    /**
     * fetch the offset for the given item.
{'fraction_non_alphanumeric': 0.09908981624592135, 'fraction_numerical': 0.0003434655675768504, 'mean_word_length': 2.0380803338549818, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4621652', 'n_tokens_mistral': 1705, 'n_tokens_neox': 1619, 'n_words': 492}