<commit_before>package de.zib.scalaris.examples.wikipedia;

import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Random;
import java.util.TreeMap;

import com.ericsson.otp.erlang.OtpErlangException;

import de.zib.scalaris.ErlangValue;
import de.zib.scalaris.NotFoundException;
import de.zib.scalaris.RequestList;
import de.zib.scalaris.ResultList;
import de.zib.scalaris.UnknownException;
import de.zib.scalaris.examples.wikipedia.Options.APPEND_INCREMENT_BUCKETS;
import de.zib.scalaris.examples.wikipedia.Options.Optimisation;
import de.zib.scalaris.examples.wikipedia.Options.IPartialRead;
import de.zib.scalaris.executor.ScalarisOp;
import de.zib.scalaris.operations.ReadOp;
import de.zib.scalaris.operations.ReadRandomFromListOp;

/**
 * Implements a random list entry read operation.
 *
 * @param <T> the type of objects in the list
 *
 * @author <NAME>, <EMAIL>
 */
public class ScalarisReadRandomListEntryOp1<T> implements ScalarisOp {
    final Collection<String> keys;
    final int buckets;
    final ErlangConverter<List<T>> listConv;
    final ErlangConverter<T> elemConv;
    final boolean failNotFound;
    T value = null;
    final Random random;
    final boolean usePartialReads;

    /**
     * Creates a new (random) list entry read operation.
     * 
     * @param keys
     *            the keys under which the list is stored in Scalaris
     * @param optimisation
     *            the list optimisation to use
     * @param elemConv
     *            converter to make an {@link ErlangValue} to a <tt>T</tt>
     * @param listConv
     *            converter to make an {@link ErlangValue} to a {@link List} of
     *            <tt>T</tt>
     * @param failNotFound
     *            whether to re-throw the {@link NotFoundException} if a list
     *            key was not found
     * @param random
     *            the random number generator to use
     */
    public ScalarisReadRandomListEntryOp1(final Collection<String> keys,
            final Optimisation optimisation, ErlangConverter<T> elemConv, ErlangConverter<List<T>> listConv,
            boolean failNotFound, Random random) {
        this.keys = keys;
        if (optimisation instanceof APPEND_INCREMENT_BUCKETS) {
            this.buckets = ((APPEND_INCREMENT_BUCKETS) optimisation).getBuckets();
        } else {
            this.buckets = 1;
        }
        this.usePartialReads = optimisation instanceof IPartialRead;
        this.listConv = listConv;
        this.elemConv = elemConv;
        this.failNotFound = failNotFound;
        this.random = random;
    }

    public int workPhases() {
        return 1;
    }

    public final int doPhase(final int phase, final int firstOp,
            final ResultList results, final RequestList requests)
            throws OtpErlangException, UnknownException,
            IllegalArgumentException {
        switch (phase) {
            case 0: return prepareRead(requests);
            case 1: return checkRead(firstOp, results);
            default:
                throw new IllegalArgumentException("No phase " + phase);
        }
    }

    /**
     * Adds a read operation for the list to the request list.
     *
     * @param requests the request list
     *
     * @return <tt>0</tt> (no operation processed since no results are used)
     */
    protected int prepareRead(final RequestList requests) {
        for (String key : keys) {
            if (buckets > 1) {
                if (usePartialReads) {
                    requests.addOp(new ReadRandomFromListOp(key + ":" + random.nextInt(buckets)));
                } else {
                    requests.addOp(new ReadOp(key + ":" + random.nextInt(buckets)));
                }
            } else {
                if (usePartialReads) {
                    requests.addOp(new ReadRandomFromListOp(key));
                } else {
                    requests.addOp(new ReadOp(key));
                }
            }
        }
        return 0;
    }

    /**
     * Verifies the read operation(s) and creates the full list.
     *
     * @param firstOp   the first operation to process inside the result list
     * @param results   the result list
     * @param requests  the request list
     *
     * @return number of processed operations
     */
    protected int checkRead(int firstOp, final ResultList results) throws OtpErlangException,
            UnknownException {
        if (usePartialReads) {
            /*
             * each partition may contain a different number of keys
             * -> collect all partition's random values and calculate the final
             * random value based on the number of entries in the partition's
             * lists
             * -> in the created valueMap, each random value stands for a range
             * of keys equal to the number of list elements of its partition; we
             * finally draw a random number in the overall range and use the
             * read random value responsible for that
             */
            TreeMap<Integer, T> valueMap = new TreeMap<Integer, T>();
            Integer listLen = 0;
            for (@SuppressWarnings("unused") String key : keys) {
                try {
                    ReadRandomFromListOp.Result res = ((ReadRandomFromListOp) results.get(firstOp++)).processResult();
                    listLen += res.listLength;
                    valueMap.put(listLen, elemConv.convert(res.randomElement));
                } catch (NotFoundException e) {
                    if (failNotFound) {
                        throw e;
                    }
                }
            }
            if (listLen != 0) {
                /*
                 * note: tailMap returns the tail including the given key.<commit_msg>Wiki on Scalaris: ignore an EmptyListException when retrieving random articles and some partitions are empty<commit_after>package de.zib.scalaris.examples.wikipedia;

import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Random;
import java.util.TreeMap;

import com.ericsson.otp.erlang.OtpErlangException;

import de.zib.scalaris.EmptyListException;
import de.zib.scalaris.ErlangValue;
import de.zib.scalaris.NotFoundException;
import de.zib.scalaris.RequestList;
import de.zib.scalaris.ResultList;
import de.zib.scalaris.UnknownException;
import de.zib.scalaris.examples.wikipedia.Options.APPEND_INCREMENT_BUCKETS;
import de.zib.scalaris.examples.wikipedia.Options.Optimisation;
import de.zib.scalaris.examples.wikipedia.Options.IPartialRead;
import de.zib.scalaris.executor.ScalarisOp;
import de.zib.scalaris.operations.ReadOp;
import de.zib.scalaris.operations.ReadRandomFromListOp;

/**
 * Implements a random list entry read operation.
 *
 * @param <T> the type of objects in the list
 *
 * @author <NAME>, <EMAIL>
 */
public class ScalarisReadRandomListEntryOp1<T> implements ScalarisOp {
    final Collection<String> keys;
    final int buckets;
    final ErlangConverter<List<T>> listConv;
    final ErlangConverter<T> elemConv;
    final boolean failNotFound;
    T value = null;
    final Random random;
    final boolean usePartialReads;

    /**
     * Creates a new (random) list entry read operation.
     * 
     * @param keys
     *            the keys under which the list is stored in Scalaris
     * @param optimisation
     *            the list optimisation to use
     * @param elemConv
     *            converter to make an {@link ErlangValue} to a <tt>T</tt>
     * @param listConv
     *            converter to make an {@link ErlangValue} to a {@link List} of
     *            <tt>T</tt>
     * @param failNotFound
     *            whether to re-throw the {@link NotFoundException} if a list
     *            key was not found
     * @param random
     *            the random number generator to use
     */
    public ScalarisReadRandomListEntryOp1(final Collection<String> keys,
            final Optimisation optimisation, ErlangConverter<T> elemConv, ErlangConverter<List<T>> listConv,
            boolean failNotFound, Random random) {
        this.keys = keys;
        if (optimisation instanceof APPEND_INCREMENT_BUCKETS) {
            this.buckets = ((APPEND_INCREMENT_BUCKETS) optimisation).getBuckets();
        } else {
            this.buckets = 1;
        }
        this.usePartialReads = optimisation instanceof IPartialRead;
        this.listConv = listConv;
        this.elemConv = elemConv;
        this.failNotFound = failNotFound;
        this.random = random;
    }

    public int workPhases() {
        return 1;
    }

    public final int doPhase(final int phase, final int firstOp,
            final ResultList results, final RequestList requests)
            throws OtpErlangException, UnknownException,
            IllegalArgumentException {
        switch (phase) {
            case 0: return prepareRead(requests);
            case 1: return checkRead(firstOp, results);
            default:
                throw new IllegalArgumentException("No phase " + phase);
        }
    }

    /**
     * Adds a read operation for the list to the request list.
     *
     * @param requests the request list
     *
     * @return <tt>0</tt> (no operation processed since no results are used)
     */
    protected int prepareRead(final RequestList requests) {
        for (String key : keys) {
            if (buckets > 1) {
                if (usePartialReads) {
                    requests.addOp(new ReadRandomFromListOp(key + ":" + random.nextInt(buckets)));
                } else {
                    requests.addOp(new ReadOp(key + ":" + random.nextInt(buckets)));
                }
            } else {
                if (usePartialReads) {
                    requests.addOp(new ReadRandomFromListOp(key));
                } else {
                    requests.addOp(new ReadOp(key));
                }
            }
        }
        return 0;
    }

    /**
     * Verifies the read operation(s) and creates the full list.
     *
     * @param firstOp   the first operation to process inside the result list
     * @param results   the result list
     * @param requests  the request list
     *
     * @return number of processed operations
     */
    protected int checkRead(int firstOp, final ResultList results) throws OtpErlangException,
            UnknownException {
        if (usePartialReads) {
            /*
             * each partition may contain a different number of keys
             * -> collect all partition's random values and calculate the final
             * random value based on the number of entries in the partition's
             * lists
             * -> in the created valueMap, each random value stands for a range
             * of keys equal to the number of list elements of its partition; we
             * finally draw a random number in the overall range and use the
             * read random value responsible for that
             */
            TreeMap<Integer, T> valueMap = new TreeMap<Integer, T>();
            Integer listLen = 0;
            for (@SuppressWarnings("unused") String key : keys) {
                try {
                    ReadRandomFromListOp.Result res = ((ReadRandomFromListOp) results.get(firstOp++)).processResult();
                    listLen += res.listLength;
                    valueMap.put(listLen, elemConv.convert(res.randomElement));
                } catch (NotFoundException e) {
                    if (failNotFound) {
                        throw e;
                    }
                } catch (EmptyListException e) {
                    // this is ok - we simply ignore this partition
                }
            }
            if (listLen != 0) {
                /*
                 * note: tailMap returns the tail including the given key.
{'fraction_non_alphanumeric': 0.08084566596194503, 'fraction_numerical': 0.0018604651162790699, 'mean_word_length': 2.038540596094553, 'pattern_counts': {'":': 4, '<': 43, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5111599', 'n_tokens_mistral': 3175, 'n_tokens_neox': 3073, 'n_words': 940}