<commit_before>/*
 * Made with all the love in the world
 * by scireum in Remshalden, Germany
 *
 * Copyright by scireum GmbH
 * http://www.scireum.de - <EMAIL>
 */

package ninja;

import com.google.common.collect.Maps;
import com.google.common.hash.HashCode;
import com.google.common.hash.Hashing;
import com.google.common.io.BaseEncoding;
import com.google.common.io.ByteStreams;
import com.google.common.io.Files;
import io.netty.handler.codec.http.HttpHeaders;
import io.netty.handler.codec.http.HttpMethod;
import io.netty.handler.codec.http.HttpResponseStatus;
import sirius.kernel.async.CallContext;
import sirius.kernel.commons.Strings;
import sirius.kernel.commons.Value;
import sirius.kernel.di.std.ConfigValue;
import sirius.kernel.di.std.Part;
import sirius.kernel.di.std.Register;
import sirius.kernel.health.Counter;
import sirius.kernel.health.Exceptions;
import sirius.kernel.health.HandledException;
import sirius.kernel.xml.XMLReader;
import sirius.kernel.xml.XMLStructuredOutput;
import sirius.web.controller.Controller;
import sirius.web.controller.Routed;
import sirius.web.http.InputStreamHandler;
import sirius.web.http.Response;
import sirius.web.http.WebContext;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.time.Instant;
import java.time.ZoneOffset;
import java.time.chrono.IsoChronology;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeFormatterBuilder;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import java.util.regex.Matcher;
import java.util.stream.Collectors;

import static io.netty.handler.codec.http.HttpMethod.*;
import static ninja.Aws4HashCalculator.AWS_AUTH4_PATTERN;
import static ninja.AwsHashCalculator.AWS_AUTH_PATTERN;

/**
 * Handles calls to the S3 API.
 */
@Register
public class S3Controller implements Controller {

    @Override
    public void onError(WebContext ctx, HandledException error) {
        signalObjectError(ctx, HttpResponseStatus.BAD_REQUEST, error.getMessage());
    }

    @Part
    private Storage storage;

    @Part
    private APILog log;

    @Part
    private AwsHashCalculator hashCalculator;

    @ConfigValue("storage.multipartDir")
    private String multipartDir;

    private Set<String> multipartUploads = Collections.synchronizedSet(new TreeSet<>());

    private Counter uploadIdCounter = new Counter();

    public static final DateTimeFormatter ISO_INSTANT =
            new DateTimeFormatterBuilder().appendPattern("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'")
                                          .toFormatter()
                                          .withChronology(IsoChronology.INSTANCE)
                                          .withZone(ZoneOffset.UTC);

    /**
     * Extracts the given hash from the given request. Returns null if no hash was given.
     */
    private String getAuthHash(WebContext ctx) {
        Value authorizationHeaderValue = ctx.getHeaderValue(HttpHeaders.Names.AUTHORIZATION);
        if (!authorizationHeaderValue.isFilled()) {
            return ctx.get("Signature").getString();
        }
        String authentication =
                Strings.isEmpty(authorizationHeaderValue.getString()) ? "" : authorizationHeaderValue.getString();
        Matcher m = AWS_AUTH_PATTERN.matcher(authentication);
        if (m.matches()) {
            return m.group(2);
        }

        m = AWS_AUTH4_PATTERN.matcher(authentication);
        if (m.matches()) {
            return m.group(5);
        }

        return null;
    }

    /**
     * Writes an API error to the log
     */
    private void signalObjectError(WebContext ctx, HttpResponseStatus status, String message) {
        if (ctx.getRequest().getMethod() == HEAD) {
            ctx.respondWith().status(status);
        } else {
            ctx.respondWith().error(status, message);
        }
        log.log("OBJECT " + ctx.getRequest().getMethod().name(),
                message + " - " + ctx.getRequestedURI(),
                APILog.Result.ERROR,
                CallContext.getCurrent().getWatch());
    }

    /**
     * Writes an API success entry to the log
     */
    private void signalObjectSuccess(WebContext ctx) {
        log.log("OBJECT " + ctx.getRequest().getMethod().name(),
                ctx.getRequestedURI(),
                APILog.Result.OK,
                CallContext.getCurrent().getWatch());
    }

    /**
     * Dispatching method handling bucket specific calls without content (HEAD and DELETE)
     *
     * @param ctx        the context describing the current request
     * @param bucketName name of the bucket of interest
     */
    @Routed(value = "/s3/:1", priority = 99)
    public void bucket(WebContext ctx, String bucketName) {
        Bucket bucket = storage.getBucket(bucketName);

        HttpMethod method = ctx.getRequest().getMethod();

        if (HEAD == method) {
            if (bucket.exists()) {
                signalObjectSuccess(ctx);
                ctx.respondWith().status(HttpResponseStatus.OK);
            } else {
                signalObjectError(ctx, HttpResponseStatus.NOT_FOUND, "Bucket does not exist");
            }
        } else if (GET == method) {
            if (bucket.exists()) {
                listObjects(ctx, bucket);
            } else {
                signalObjectError(ctx, HttpResponseStatus.NOT_FOUND, "Bucket does not exist");
            }
        } else if (DELETE == method) {
            bucket.delete();
            signalObjectSuccess(ctx);
            ctx.respondWith().status(HttpResponseStatus.OK);<commit_msg>Add support for listing buckets<commit_after>/*
 * Made with all the love in the world
 * by scireum in Remshalden, Germany
 *
 * Copyright by scireum GmbH
 * http://www.scireum.de - <EMAIL>
 */

package ninja;

import com.google.common.collect.Maps;
import com.google.common.hash.HashCode;
import com.google.common.hash.Hashing;
import com.google.common.io.BaseEncoding;
import com.google.common.io.ByteStreams;
import com.google.common.io.Files;
import io.netty.handler.codec.http.HttpHeaders;
import io.netty.handler.codec.http.HttpMethod;
import io.netty.handler.codec.http.HttpResponseStatus;
import sirius.kernel.async.CallContext;
import sirius.kernel.commons.Strings;
import sirius.kernel.commons.Value;
import sirius.kernel.di.std.ConfigValue;
import sirius.kernel.di.std.Part;
import sirius.kernel.di.std.Register;
import sirius.kernel.health.Counter;
import sirius.kernel.health.Exceptions;
import sirius.kernel.health.HandledException;
import sirius.kernel.xml.Attribute;
import sirius.kernel.xml.XMLReader;
import sirius.kernel.xml.XMLStructuredOutput;
import sirius.web.controller.Controller;
import sirius.web.controller.Routed;
import sirius.web.http.InputStreamHandler;
import sirius.web.http.Response;
import sirius.web.http.WebContext;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.time.Instant;
import java.time.ZoneOffset;
import java.time.chrono.IsoChronology;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeFormatterBuilder;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import java.util.regex.Matcher;
import java.util.stream.Collectors;

import static io.netty.handler.codec.http.HttpMethod.*;
import static ninja.Aws4HashCalculator.AWS_AUTH4_PATTERN;
import static ninja.AwsHashCalculator.AWS_AUTH_PATTERN;

/**
 * Handles calls to the S3 API.
 */
@Register
public class S3Controller implements Controller {

    @Override
    public void onError(WebContext ctx, HandledException error) {
        signalObjectError(ctx, HttpResponseStatus.BAD_REQUEST, error.getMessage());
    }

    @Part
    private Storage storage;

    @Part
    private APILog log;

    @Part
    private AwsHashCalculator hashCalculator;

    @ConfigValue("storage.multipartDir")
    private String multipartDir;

    private Set<String> multipartUploads = Collections.synchronizedSet(new TreeSet<>());

    private Counter uploadIdCounter = new Counter();

    public static final DateTimeFormatter ISO_INSTANT =
            new DateTimeFormatterBuilder().appendPattern("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'")
                                          .toFormatter()
                                          .withChronology(IsoChronology.INSTANCE)
                                          .withZone(ZoneOffset.UTC);

    /**
     * Extracts the given hash from the given request. Returns null if no hash was given.
     */
    private String getAuthHash(WebContext ctx) {
        Value authorizationHeaderValue = ctx.getHeaderValue(HttpHeaders.Names.AUTHORIZATION);
        if (!authorizationHeaderValue.isFilled()) {
            return ctx.get("Signature").getString();
        }
        String authentication =
                Strings.isEmpty(authorizationHeaderValue.getString()) ? "" : authorizationHeaderValue.getString();
        Matcher m = AWS_AUTH_PATTERN.matcher(authentication);
        if (m.matches()) {
            return m.group(2);
        }

        m = AWS_AUTH4_PATTERN.matcher(authentication);
        if (m.matches()) {
            return m.group(5);
        }

        return null;
    }

    /**
     * Writes an API error to the log
     */
    private void signalObjectError(WebContext ctx, HttpResponseStatus status, String message) {
        if (ctx.getRequest().getMethod() == HEAD) {
            ctx.respondWith().status(status);
        } else {
            ctx.respondWith().error(status, message);
        }
        log.log("OBJECT " + ctx.getRequest().getMethod().name(),
                message + " - " + ctx.getRequestedURI(),
                APILog.Result.ERROR,
                CallContext.getCurrent().getWatch());
    }

    /**
     * Writes an API success entry to the log
     */
    private void signalObjectSuccess(WebContext ctx) {
        log.log("OBJECT " + ctx.getRequest().getMethod().name(),
                ctx.getRequestedURI(),
                APILog.Result.OK,
                CallContext.getCurrent().getWatch());
    }
    
    /**
	 * GET a list of all buckets
	 *
	 * @param ctx
	 *            the context describing the current request
	 */
	@Routed(value = "/s3", priority = 99)
	public void listBuckets(WebContext ctx) {
		HttpMethod method = ctx.getRequest().getMethod();

		if (GET == method) {
			List<Bucket> buckets = storage.getBuckets();
			Response response = ctx.respondWith();

			response.setHeader("Content-Type", "application/xml");

			XMLStructuredOutput out = response.xml();
			out.beginOutput("ListAllMyBucketsResult",
					Attribute.set("xmlns", "http://s3.amazonaws.com/doc/2006-03-01/"));
			out.beginObject("Owner");
			out.property("ID", "initiatorId");
			out.property("DisplayName", "initiatorName");
			out.endObject();

			out.beginObject("Buckets");
			for (Bucket bucket : buckets) {
				out.beginObject("Bucket");
				out.property("Name", bucket.getName());
				out.property("CreationDate", ISO_INSTANT.format(Instant.ofEpochMilli(bucket.getFile().lastModified())));
				out.endObject();
			}
			out.endObject();
			out.endOutput();
		} else {
			throw new IllegalArgumentException(ctx.getRequest().getMethod().name());
		}
	}

    /**
     * Dispatching method handling bucket specific calls without content (HEAD and DELETE)
     *
     * @param ctx        the context describing the current request
     * @param bucketName name of the bucket of interest
     */
    @Routed(value = "/s3/:1", priority = 99)
    public void bucket(WebContext ctx, String bucketName) {
        Bucket bucket = storage.getBucket(bucketName);

        HttpMethod method = ctx.getRequest().getMethod();

        if (HEAD == method) {
            if (bucket.exists()) {
                signalObjectSuccess(ctx);
                ctx.respondWith().status(HttpResponseStatus.OK);
            } else {
                signalObjectError(ctx, HttpResponseStatus.NOT_FOUND, "Bucket does not exist");
            }
        } else if (GET == method) {
            if (bucket.exists()) {
                listObjects(ctx, bucket);
            } else {
                signalObjectError(ctx, HttpResponseStatus.NOT_FOUND, "Bucket does not exist");
            }
        } else if (DELETE == method) {
            bucket.delete();
            signalObjectSuccess(ctx);
            ctx.respondWith().status(HttpResponseStatus.OK);
{'fraction_non_alphanumeric': 0.11219779359987482, 'fraction_numerical': 0.0026601987324935453, 'mean_word_length': 2.8488407106293283, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6944487', 'n_tokens_mistral': 3722, 'n_tokens_neox': 3555, 'n_words': 766}