<commit_before>package edu.kit.ipd.crowdcontrol.workerservice.command;

import com.google.common.collect.LinkedHashMultimap;
import com.google.common.collect.LinkedListMultimap;
import com.google.common.collect.ListMultimap;
import com.google.common.net.MediaType;
import com.google.protobuf.InvalidProtocolBufferException;
import com.google.protobuf.Message;
import com.google.protobuf.util.JsonFormat;
import edu.kit.ipd.crowdcontrol.workerservice.*;
import edu.kit.ipd.crowdcontrol.workerservice.database.operations.ExperimentOperations;
import edu.kit.ipd.crowdcontrol.workerservice.objectservice.Communication;
import edu.kit.ipd.crowdcontrol.workerservice.proto.*;
import edu.kit.ipd.crowdcontrol.workerservice.proto.Answer;
import edu.kit.ipd.crowdcontrol.workerservice.proto.Calibration;
import edu.kit.ipd.crowdcontrol.workerservice.proto.Rating;
import org.apache.commons.validator.routines.EmailValidator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import ratpack.exec.Promise;
import ratpack.handling.Context;
import ratpack.http.TypedData;

import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.function.BiFunction;
import java.util.stream.Collector;
import java.util.stream.Collectors;

/**
 * The class commands is responsible for the command-part of the CQRS-pattern. Therefore it provides the submit methods
 * matching the available REST-POST commands.
 *
 * @author LeanderK
 * @version 1.0
 */
public class Commands implements RequestHelper {
    private static final Logger logger = LoggerFactory.getLogger(Commands.class);
    private final Communication communication;
    private final JsonFormat.Parser parser = JsonFormat.parser();
    private final ExperimentOperations experimentOperations;
    private final JWTHelper jwtHelper;

    /**
     * creates an instance of Commands
     * @param communication        the communication used to communicate with the object-service
     * @param experimentOperations the experiment-operations used to communicate with the database
     * @param jwtHelper the Helper used to parse/generate the JWT-authentication
     */
    public Commands(Communication communication, ExperimentOperations experimentOperations, JWTHelper jwtHelper) {
        this.communication = communication;
        this.experimentOperations = experimentOperations;
        this.jwtHelper = jwtHelper;
    }

    /**
     * The POST-command /email, where this methods corresponds to, persists the email for the specified worker or
     * create a new if none specified.
     * this method expects the JSON-Representation of the protobuf-message email in the body of the request.
     * the response is the corresponding workerID for the email.
     *
     * @param context the Context of the Request
     * @return an the JSON-Representation of of the EmailAnswer protobuf-message
     */
    public Promise<EmailAnswer> submitEmail(Context context) {
        String platform = assertParameter(context, "platform");

        return merge(context, Email.newBuilder(), new ArrayList<>())
                .map(Email.Builder::build)
                .map(toValidate -> {
                    logger.debug("Request submit to submit email {} for platform {}", toValidate, platform);
                    if (!EmailValidator.getInstance(false).isValid(toValidate.getEmail())) {
                        throw new BadRequestException("invalid email: " + toValidate.getEmail());
                    }
                    return toValidate;
                })
                .flatMap(email -> {
                    LinkedListMultimap<String, String> parameters = email.getPlatformParametersList().stream()
                            .collect(Collector.of(
                                    LinkedListMultimap::<String, String>create,
                                    (map, param) -> map.putAll(param.getKey(), param.getValuesList()),
                                    (map1, map2) -> {
                                        map1.putAll(map2);
                                        return map2;
                                    }
                            ));
                    return Promise.<Integer>of(downstream -> downstream.accept(
                            communication.submitWorker(email.getEmail(), platform, parameters)
                                    .handle((emailAnswer, throwable) -> wrapExceptionOr201(emailAnswer, throwable, context))
                            ))
                            .map(workerID -> EmailAnswer.newBuilder().setAuthorization(jwtHelper.generateJWT(workerID)).build())
                            .map(emailAnswer -> {
                                logger.debug("Answer from Object-Service for submitting email {}: {}", email, emailAnswer.toString());
                                return emailAnswer;
                            });
                });
    }

    /**
     * The POST-command /calibration, where this method corresponds to, persists the answer of an worker to the
     * calibration questions.
     * this method expects the JSON-Representation of the protobuf-message calibration.
     *
     * @param context the Context of the Request
     * @return empty body (null)
     */
    public Promise<String> submitCalibration(Context context) {
        return doSubmit(context, Calibration.newBuilder(), (calibration, workerID) -> {
            logger.debug("Trying to submit calibration {} for worker {}.", calibration, workerID);
            return communication.submitCalibration(calibration.getAnswerOption(), workerID);
        }).map(result -> {
            logger.debug("Object service answered with OK.");
            return "Calibration submitted";
        });
    }

    /**
     * The POST-command /answer, where this method corresponds to, persists the answer of an worker to a creative-task.
     * this method expects the JSON-Representation of the protobuf-message answer.
     *
     * @param context the Context of the Request
     * @return empty body (null)
     */
    public Promise<String> submitAnswer(Context context) {
        return doSubmit(context, Answer.newBuilder(), (answer, workerID) -> {
            logger.debug("Request to submit answer {} for worker {}", answer, workerID);
            String answerType = experimentOperations.getExperiment(answer.getExperiment()).getAnswerType();
            try {
                if (answerType != null) {
                    String mime = getContentType(answer.getAnswer());
                    MediaType expected = MediaType.parse(answerType);
                    MediaType got = MediaType.parse(mime);
                    if (!got.is(expected)) {
                        throw new BadRequestException("the content-type of the URL does not match the desired type: " +
                                answerType);
                    }
                }
            } catch (IOException e) {
                throw new InternalServerErrorException("unable to probe Content-type, aborting", e);
            }
            logger.trace("answer {} for reservation {} from worker {} is valid", answer, answer.getReservation(), workerID);
            return communication.submitAnswer(answer.getAnswer(), answer.getReservation(), answer.getExperiment(), workerID);<commit_msg>command: clean test-answer from potential malicious html-tags<commit_after>package edu.kit.ipd.crowdcontrol.workerservice.command;

import com.google.common.collect.LinkedHashMultimap;
import com.google.common.collect.LinkedListMultimap;
import com.google.common.collect.ListMultimap;
import com.google.common.net.MediaType;
import com.google.protobuf.InvalidProtocolBufferException;
import com.google.protobuf.Message;
import com.google.protobuf.util.JsonFormat;
import edu.kit.ipd.crowdcontrol.workerservice.*;
import edu.kit.ipd.crowdcontrol.workerservice.database.operations.ExperimentOperations;
import edu.kit.ipd.crowdcontrol.workerservice.objectservice.Communication;
import edu.kit.ipd.crowdcontrol.workerservice.proto.*;
import edu.kit.ipd.crowdcontrol.workerservice.proto.Answer;
import edu.kit.ipd.crowdcontrol.workerservice.proto.Calibration;
import edu.kit.ipd.crowdcontrol.workerservice.proto.Rating;
import org.apache.commons.validator.routines.EmailValidator;
import org.jsoup.Jsoup;
import org.jsoup.safety.Whitelist;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import ratpack.exec.Promise;
import ratpack.handling.Context;
import ratpack.http.TypedData;

import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.function.BiFunction;
import java.util.stream.Collector;
import java.util.stream.Collectors;

/**
 * The class commands is responsible for the command-part of the CQRS-pattern. Therefore it provides the submit methods
 * matching the available REST-POST commands.
 *
 * @author LeanderK
 * @version 1.0
 */
public class Commands implements RequestHelper {
    private static final Logger logger = LoggerFactory.getLogger(Commands.class);
    private final Communication communication;
    private final JsonFormat.Parser parser = JsonFormat.parser();
    private final ExperimentOperations experimentOperations;
    private final JWTHelper jwtHelper;

    /**
     * creates an instance of Commands
     * @param communication        the communication used to communicate with the object-service
     * @param experimentOperations the experiment-operations used to communicate with the database
     * @param jwtHelper the Helper used to parse/generate the JWT-authentication
     */
    public Commands(Communication communication, ExperimentOperations experimentOperations, JWTHelper jwtHelper) {
        this.communication = communication;
        this.experimentOperations = experimentOperations;
        this.jwtHelper = jwtHelper;
    }

    /**
     * The POST-command /email, where this methods corresponds to, persists the email for the specified worker or
     * create a new if none specified.
     * this method expects the JSON-Representation of the protobuf-message email in the body of the request.
     * the response is the corresponding workerID for the email.
     *
     * @param context the Context of the Request
     * @return an the JSON-Representation of of the EmailAnswer protobuf-message
     */
    public Promise<EmailAnswer> submitEmail(Context context) {
        String platform = assertParameter(context, "platform");

        return merge(context, Email.newBuilder(), new ArrayList<>())
                .map(Email.Builder::build)
                .map(toValidate -> {
                    logger.debug("Request submit to submit email {} for platform {}", toValidate, platform);
                    if (!EmailValidator.getInstance(false).isValid(toValidate.getEmail())) {
                        throw new BadRequestException("invalid email: " + toValidate.getEmail());
                    }
                    return toValidate;
                })
                .flatMap(email -> {
                    LinkedListMultimap<String, String> parameters = email.getPlatformParametersList().stream()
                            .collect(Collector.of(
                                    LinkedListMultimap::<String, String>create,
                                    (map, param) -> map.putAll(param.getKey(), param.getValuesList()),
                                    (map1, map2) -> {
                                        map1.putAll(map2);
                                        return map2;
                                    }
                            ));
                    return Promise.<Integer>of(downstream -> downstream.accept(
                            communication.submitWorker(email.getEmail(), platform, parameters)
                                    .handle((emailAnswer, throwable) -> wrapExceptionOr201(emailAnswer, throwable, context))
                            ))
                            .map(workerID -> EmailAnswer.newBuilder().setAuthorization(jwtHelper.generateJWT(workerID)).build())
                            .map(emailAnswer -> {
                                logger.debug("Answer from Object-Service for submitting email {}: {}", email, emailAnswer.toString());
                                return emailAnswer;
                            });
                });
    }

    /**
     * The POST-command /calibration, where this method corresponds to, persists the answer of an worker to the
     * calibration questions.
     * this method expects the JSON-Representation of the protobuf-message calibration.
     *
     * @param context the Context of the Request
     * @return empty body (null)
     */
    public Promise<String> submitCalibration(Context context) {
        return doSubmit(context, Calibration.newBuilder(), (calibration, workerID) -> {
            logger.debug("Trying to submit calibration {} for worker {}.", calibration, workerID);
            return communication.submitCalibration(calibration.getAnswerOption(), workerID);
        }).map(result -> {
            logger.debug("Object service answered with OK.");
            return "Calibration submitted";
        });
    }

    /**
     * The POST-command /answer, where this method corresponds to, persists the answer of an worker to a creative-task.
     * this method expects the JSON-Representation of the protobuf-message answer.
     *
     * @param context the Context of the Request
     * @return empty body (null)
     */
    public Promise<String> submitAnswer(Context context) {
        return doSubmit(context, Answer.newBuilder(), (answer, workerID) -> {
            logger.debug("Request to submit answer {} for worker {}", answer, workerID);
            String answerType = experimentOperations.getExperiment(answer.getExperiment()).getAnswerType();
            try {
                if (answerType != null) {
                    String mime = getContentType(answer.getAnswer());
                    MediaType expected = MediaType.parse(answerType);
                    MediaType got = MediaType.parse(mime);
                    if (!got.is(expected)) {
                        throw new BadRequestException("the content-type of the URL does not match the desired type: " +
                                answerType);
                    }
                }
            } catch (IOException e) {
                throw new InternalServerErrorException("unable to probe Content-type, aborting", e);
            }
            if (answerType == null) {
                logger.trace("cleaning answer {} for worker {} form potential malicious html tags");
                answer.setAnswer(Jsoup.clean(answer.getAnswer(), Whitelist.basic()));
            }
            logger.trace("answer {} for reservation {} from worker {} is valid", answer, answer.getReservation(), workerID);
            return communication.submitAnswer(answer.getAnswer(), answer.getReservation(), answer.getExperiment(), workerID);
{'fraction_non_alphanumeric': 0.09139855548260013, 'fraction_numerical': 0.0015758371634931056, 'mean_word_length': 2.416554508748318, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3438645', 'n_tokens_mistral': 3756, 'n_tokens_neox': 3624, 'n_words': 979}