<commit_before>package edu.kit.ipd.crowdcontrol.objectservice.crowdworking.mturk;

import com.amazonaws.mturk.requester.doc._2014_08_15.Assignment;
import com.amazonaws.mturk.requester.doc._2014_08_15.AssignmentStatus;
import com.amazonaws.mturk.requester.doc._2014_08_15.BonusPayment;
import com.google.common.base.Charsets;
import com.google.common.io.CharStreams;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.JsonPrimitive;
import edu.kit.ipd.crowdcontrol.objectservice.Main;
import edu.kit.ipd.crowdcontrol.objectservice.crowdworking.*;
import edu.kit.ipd.crowdcontrol.objectservice.crowdworking.mturk.command.*;
import edu.kit.ipd.crowdcontrol.objectservice.proto.Experiment;
import edu.kit.ipd.crowdcontrol.objectservice.proto.Tag;
import edu.kit.ipd.crowdcontrol.objectservice.template.Template;

import java.io.IOException;
import java.io.InputStreamReader;
import java.math.BigDecimal;
import java.net.URISyntaxException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.*;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
import java.util.function.Function;
import java.util.function.Predicate;
import java.util.function.Supplier;
import java.util.stream.Collector;
import java.util.stream.Collectors;


/**
 * Platform implementation for a mturk instance
 * @author MarcelHollerbach
 * @version 0.1
 */
public class MturkPlatform implements Platform,Payment {
    public static final int THIRTY_DAYS = 60 * 60 * 24 * 30;
    public static final int TWO_HOURS = 60 * 60 * 2;
    private final String name;
    private final MTurkConnection connection;
    private final String workerServiceUrl;
    private final String workerUIUrl;
    /**
     * A new mturk platform instance
     * @param user user to login
     * @param password <PASSWORD>
     * @param url instance to connect to
     * @param workerUIUrl path where to find the workerUI
     */
    public MturkPlatform(String user, String password, String url, String name, String workerServiceUrl, String workerUIUrl) {
        this.workerUIUrl = workerUIUrl;
        connection = new MTurkConnection(user, password, url);
        this.workerServiceUrl = workerServiceUrl;
        this.name = name;
    }

    @Override
    public String getName() {
        return "Mturk "+name;
    }

    @Override
    public String getID() {
        return "mturk"+name.toLowerCase();
    }

    @Override
    public Boolean isCalibrationAllowed() {
        return false;
    }

    @Override
    public Optional<Payment> getPayment() {
        return Optional.of(this);
    }

    @Override
    public Optional<WorkerIdentificationComputation> getWorker() {
        return Optional.of(params -> {
            String workerId = "";
            if (params != null) {
                String[] workerIdArray = params.get("mTurkWorkerId");

                if (workerIdArray != null && workerIdArray.length > 0) {
                    workerId = workerIdArray[0];
                } else {
                    throw new UnidentifiedWorkerException("mTurkWorkerId was not set!");
                }
            }

            return WorkerIdentification.findByIdentification(getID(),workerId);
        });
    }

    private static String loadFiles(String file) {
        try {
            return CharStreams
                    .toString(new InputStreamReader(
                            Main.class.getResourceAsStream(file)
                            , Charsets.UTF_8));
        } catch (IOException e) {
            e.printStackTrace();
            return "";
        }
    }

    @Override
    public CompletableFuture<JsonElement> publishTask(Experiment experiment) {
        String tags = experiment.getTagsList().stream().map(Tag::getName).collect(Collectors.joining(","));
        String jsContent = loadFiles("/mturk/worker-ui/mturk.js");
        String htmlContent = loadFiles("/mturk/worker-ui/MturkContent.html");

        Map<String, String> params = new HashMap<>();
        params.put("PlatformName", getID());
        params.put("WorkerServiceUrl", workerServiceUrl);
        params.put("WorkerUIUrl", workerUIUrl);
        params.put("ExperimentId", experiment.getId()+"");
        params.put("JsEmbed", jsContent);

        String content = Template.apply(htmlContent, params);

        return new PublishHIT(connection,experiment.getTitle(),experiment.getDescription(),
                experiment.getPaymentBase().getValue()/100d, //we are getting cents passed and have to pass dollars
                TWO_HOURS, //you have 2 hours to do the assignment
                THIRTY_DAYS, // the experiment is staying for 30 days
                tags,
                experiment.getNeededAnswers().getValue()*experiment.getRatingsPerAnswer().getValue(),
                2592000, //this is a little problem we have to specify when autoapproval is kicking in this is happening after 2592000s
                "",
                content)
                .thenApply(id -> {
                    JsonObject jsonObject = new JsonObject();
                    jsonObject.add("identification", new JsonPrimitive(id));
                    return jsonObject;
                });
    }

    @Override
    public CompletableFuture<Boolean> unpublishTask(JsonElement data) {
        return new UnpublishHIT(connection, data.getAsJsonObject().get("identification").getAsString());
    }

    /**
     * Returns the full list of a paginated request
     * @param startIndex the index to start
     * @param producer execute the statement and returns a patial list of the complete
     * @param <A> Type of the list
     * @return The full list of elements
     * @throws ExecutionException If something bad happens while executing
     * @throws InterruptedException the execution was interrupted
     */
    public <A> List<A> getFullList(int startIndex, Function<Integer, CompletableFuture<List<A>>> producer) throws ExecutionException, InterruptedException {
        int i = startIndex;
        List<A> part = producer.apply(i).get();
        List<A> complete = new ArrayList<>();
        while(part != null && part.size() != 0) {
            complete.addAll(part);
            i++;
            part = producer.apply(i).get();
        }
        return complete;
    }

    @Override
    public CompletableFuture<Boolean> payExperiment(int dbId, JsonElement data, Experiment experiment, List<PaymentJob> paymentJobs) {
        /**
         * this code works under the assumation that basepayment is part of the amout!!!!!!
         */
        Map<String, Assignment> workerAssignmentId = new HashMap<>();
        Map<String, BigDecimal> bonusPayed = new HashMap<>();
        String id = data.getAsJsonObject().get("identification").getAsString();
        try {
            // get all assignments from the project and sort them to the function
            getFullList(1, index -> new GetAssignments(connection, id, index))
                    .forEach(assignment -> workerAssignmentId.put(assignment.getWorkerId(), assignment));

            //get all done payments
            getFullList(1, index -> new GetBonusPayments(connection, id, index))
                    .forEach(bonusPayment -> {
                        BigDecimal bigDecimal = bonusPayed.get(bonusPayment.getWorkerId());
                        if (bigDecimal != null) {
                            bigDecimal = bigDecimal.add(bonusPayment.getBonusAmount().getAmount());
                        } else {
                            bigDecimal = bonusPayment.getBonusAmount().getAmount();
                        }
                        bonusPayed.put(bonusPayment.getWorkerId(), bigDecimal);
                    });

        } catch (InterruptedException | ExecutionException e) {
            CompletableFuture<Boolean> completableFuture= new CompletableFuture<>();
            completableFuture.completeExceptionally(e);
            return completableFuture;
        }

        //verify and get all data<commit_msg>its always 1 and it should stay 1 since mturk will not work otherwise.<commit_after>package edu.kit.ipd.crowdcontrol.objectservice.crowdworking.mturk;

import com.amazonaws.mturk.requester.doc._2014_08_15.Assignment;
import com.amazonaws.mturk.requester.doc._2014_08_15.AssignmentStatus;
import com.amazonaws.mturk.requester.doc._2014_08_15.BonusPayment;
import com.google.common.base.Charsets;
import com.google.common.io.CharStreams;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.JsonPrimitive;
import edu.kit.ipd.crowdcontrol.objectservice.Main;
import edu.kit.ipd.crowdcontrol.objectservice.crowdworking.*;
import edu.kit.ipd.crowdcontrol.objectservice.crowdworking.mturk.command.*;
import edu.kit.ipd.crowdcontrol.objectservice.proto.Experiment;
import edu.kit.ipd.crowdcontrol.objectservice.proto.Tag;
import edu.kit.ipd.crowdcontrol.objectservice.template.Template;

import java.io.IOException;
import java.io.InputStreamReader;
import java.math.BigDecimal;
import java.net.URISyntaxException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.*;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
import java.util.function.Function;
import java.util.function.Predicate;
import java.util.function.Supplier;
import java.util.stream.Collector;
import java.util.stream.Collectors;


/**
 * Platform implementation for a mturk instance
 * @author MarcelHollerbach
 * @version 0.1
 */
public class MturkPlatform implements Platform,Payment {
    public static final int THIRTY_DAYS = 60 * 60 * 24 * 30;
    public static final int TWO_HOURS = 60 * 60 * 2;
    private final String name;
    private final MTurkConnection connection;
    private final String workerServiceUrl;
    private final String workerUIUrl;
    /**
     * A new mturk platform instance
     * @param user user to login
     * @param password <PASSWORD>
     * @param url instance to connect to
     * @param workerUIUrl path where to find the workerUI
     */
    public MturkPlatform(String user, String password, String url, String name, String workerServiceUrl, String workerUIUrl) {
        this.workerUIUrl = workerUIUrl;
        connection = new MTurkConnection(user, password, url);
        this.workerServiceUrl = workerServiceUrl;
        this.name = name;
    }

    @Override
    public String getName() {
        return "Mturk "+name;
    }

    @Override
    public String getID() {
        return "mturk"+name.toLowerCase();
    }

    @Override
    public Boolean isCalibrationAllowed() {
        return false;
    }

    @Override
    public Optional<Payment> getPayment() {
        return Optional.of(this);
    }

    @Override
    public Optional<WorkerIdentificationComputation> getWorker() {
        return Optional.of(params -> {
            String workerId = "";
            if (params != null) {
                String[] workerIdArray = params.get("mTurkWorkerId");

                if (workerIdArray != null && workerIdArray.length > 0) {
                    workerId = workerIdArray[0];
                } else {
                    throw new UnidentifiedWorkerException("mTurkWorkerId was not set!");
                }
            }

            return WorkerIdentification.findByIdentification(getID(),workerId);
        });
    }

    private static String loadFiles(String file) {
        try {
            return CharStreams
                    .toString(new InputStreamReader(
                            Main.class.getResourceAsStream(file)
                            , Charsets.UTF_8));
        } catch (IOException e) {
            e.printStackTrace();
            return "";
        }
    }

    @Override
    public CompletableFuture<JsonElement> publishTask(Experiment experiment) {
        String tags = experiment.getTagsList().stream().map(Tag::getName).collect(Collectors.joining(","));
        String jsContent = loadFiles("/mturk/worker-ui/mturk.js");
        String htmlContent = loadFiles("/mturk/worker-ui/MturkContent.html");

        Map<String, String> params = new HashMap<>();
        params.put("PlatformName", getID());
        params.put("WorkerServiceUrl", workerServiceUrl);
        params.put("WorkerUIUrl", workerUIUrl);
        params.put("ExperimentId", experiment.getId()+"");
        params.put("JsEmbed", jsContent);

        String content = Template.apply(htmlContent, params);

        return new PublishHIT(connection,experiment.getTitle(),experiment.getDescription(),
                experiment.getPaymentBase().getValue()/100d, //we are getting cents passed and have to pass dollars
                TWO_HOURS, //you have 2 hours to do the assignment
                THIRTY_DAYS, // the experiment is staying for 30 days
                tags,
                experiment.getNeededAnswers().getValue()*experiment.getRatingsPerAnswer().getValue(),
                2592000, //this is a little problem we have to specify when autoapproval is kicking in this is happening after 2592000s
                "",
                content)
                .thenApply(id -> {
                    JsonObject jsonObject = new JsonObject();
                    jsonObject.add("identification", new JsonPrimitive(id));
                    return jsonObject;
                });
    }

    @Override
    public CompletableFuture<Boolean> unpublishTask(JsonElement data) {
        return new UnpublishHIT(connection, data.getAsJsonObject().get("identification").getAsString());
    }

    /**
     * Returns the full list of a paginated request
     * @param producer execute the statement and returns a patial list of the complete
     * @param <A> Type of the list
     * @return The full list of elements
     * @throws ExecutionException If something bad happens while executing
     * @throws InterruptedException the execution was interrupted
     */
    public <A> List<A> getFullList(Function<Integer, CompletableFuture<List<A>>> producer) throws ExecutionException, InterruptedException {
        int i = 1;
        List<A> part = producer.apply(i).get();
        List<A> complete = new ArrayList<>();
        while(part != null && part.size() != 0) {
            complete.addAll(part);
            i++;
            part = producer.apply(i).get();
        }
        return complete;
    }

    @Override
    public CompletableFuture<Boolean> payExperiment(int dbId, JsonElement data, Experiment experiment, List<PaymentJob> paymentJobs) {
        /**
         * this code works under the assumation that basepayment is part of the amout!!!!!!
         */
        Map<String, Assignment> workerAssignmentId = new HashMap<>();
        Map<String, BigDecimal> bonusPayed = new HashMap<>();
        String id = data.getAsJsonObject().get("identification").getAsString();
        try {
            // get all assignments from the project and sort them to the function
            getFullList( index -> new GetAssignments(connection, id, index))
                    .forEach(assignment -> workerAssignmentId.put(assignment.getWorkerId(), assignment));

            //get all done payments
            getFullList(index -> new GetBonusPayments(connection, id, index))
                    .forEach(bonusPayment -> {
                        BigDecimal bigDecimal = bonusPayed.get(bonusPayment.getWorkerId());
                        if (bigDecimal != null) {
                            bigDecimal = bigDecimal.add(bonusPayment.getBonusAmount().getAmount());
                        } else {
                            bigDecimal = bonusPayment.getBonusAmount().getAmount();
                        }
                        bonusPayed.put(bonusPayment.getWorkerId(), bigDecimal);
                    });

        } catch (InterruptedException | ExecutionException e) {
            CompletableFuture<Boolean> completableFuture= new CompletableFuture<>();
            completableFuture.completeExceptionally(e);
            return completableFuture;
        }

        //verify and get all data
{'fraction_non_alphanumeric': 0.09879144716454911, 'fraction_numerical': 0.008118995971490548, 'mean_word_length': 2.5448154657293496, 'pattern_counts': {'":': 0, '<': 51, '<?xml version=': 0, '>': 65, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6748346', 'n_tokens_mistral': 4413, 'n_tokens_neox': 4215, 'n_words': 1043}