<commit_before>/*
 * Minimal Object Storage Library, (C) 2015 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package io.minio.objectstorage.client;

import com.google.api.client.http.*;
import com.google.api.client.http.javanet.NetHttpTransport;
import com.google.api.client.xml.Xml;
import com.google.api.client.xml.XmlNamespaceDictionary;
import io.minio.objectstorage.client.errors.*;
import io.minio.objectstorage.client.messages.*;
import org.xmlpull.v1.XmlPullParser;
import org.xmlpull.v1.XmlPullParserException;

import javax.xml.bind.DatatypeConverter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.MalformedURLException;
import java.net.URL;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Arrays;
import java.util.Date;
import java.util.LinkedList;
import java.util.List;
import java.util.concurrent.atomic.AtomicReference;
import java.util.logging.Handler;
import java.util.logging.Level;
import java.util.logging.LogRecord;
import java.util.logging.Logger;

/**
 * <p>
 * This class implements a simple object storage client. This client consists
 * of a useful subset of S3 compatible functionality.
 * </p>
 * <p/>
 * <h2>Service</h2>
 * <ul>
 * <li>Creating a bucket</li>
 * <li>Listing buckets</li>
 * </ul>
 * <p/>
 * <h2>Bucket</h2>
 * <ul>
 * <li> Creating an object, including automatic upload resuming for large objects.</li>
 * <li> Listing objects in a bucket</li>
 * <li> Listing active multipart uploads</li>
 * <li> Dropping all active multipart uploads</li>
 * <li> Setting canned ACLs on buckets</li>
 * </ul>
 * <p/>
 * <h2>Object</h2>Object
 * <ul>
 * <li>Dropping an active multipart upload for a specific object and uploadId</li>
 * <li>Read object metadata</li>
 * <li>Reading an object</li>
 * <li>Reading a range of bytes of an object</li>
 * <li>Deleting an object</li>
 * </ul>
 * <p/>
 * Optionally, users can also provide access/secret keys or a precomputed
 * signing key to the client. If keys are provided, all requests by the
 * client will be signed using AWS Signature Version 4. @see #setKeys(String, String)
 * <p/>
 * For an example of using this library, please see <a href="https://github.com/minio/objectstorage-java/blob/master/src/test/java/io/minio/objectstorage/example/Example.java">this example</a>.
 */
@SuppressWarnings({"SameParameterValue", "WeakerAccess"})
public class Client {
    /**
     * Canned acl: public-read-write
     * <p/>
     * Read: public
     * Write: public
     */
    @SuppressWarnings("unused")
    public static final String ACL_PUBLIC_READ_WRITE = "public-read-write";
    /**
     * Canned acl: private
     * <p/>
     * Read: authorized users only
     * Write: authorized users only
     */
    @SuppressWarnings("unused")
    public static final String ACL_PRIVATE = "private";
    /**
     * Canned acl: public-read
     * <p/>
     * Read: public
     * Write: authorized users only
     */
    @SuppressWarnings("unused")
    public static final String ACL_PUBLIC_READ = "public-read";
    /**
     * Canned acl: authenticated-read
     * <p/>
     * Read: Only users with a valid account, all valid users authorized
     * Write: acl authorized users only
     */
    @SuppressWarnings("unused")
    public static final String ACL_AUTHENTICATED_READ = "authenticated-read";
    /**
     * Canned acl: bucket-owner-read
     * <p/>
     * Read: Object owner and bucket owner
     * Write: Object owner only
     */
    @SuppressWarnings("unused")
    public static final String ACL_BUCKET_OWNER_READ = "bucket-owner-read";
    /**
     * Canned acl: bucket-owner-read
     * <p/>
     * Read: Object owner and bucket owner
     * Write: Object owner and bucket owner
     */
    @SuppressWarnings("unused")
    public static final String ACL_BUCKET_OWNER_FULL_CONTROL = "bucket-owner-full-control";

    private static final int PART_SIZE = 5 * 1024 * 1024;
    private final URL url;
    private final AtomicReference<Logger> logger = new AtomicReference<Logger>();
    private HttpTransport transport = new NetHttpTransport();
    private String accessKey;
    private String secretKey;
    private String userAgent = "objectstorage-java/0.0.1" + " (" + System.getProperty("os.name") + ", " + System.getProperty("os.arch") + ") ";
    private byte[] signingKey;

    private Client(URL url) {
        this.url = url;
    }


    /**
     * Create a new client given a url
     *
     * @param url must be the full url to the object storage server, excluding both bucket or object paths.
     *            For example: http://play.minio.io
     *            Valid:
     *            * https://s3-us-west-2.amazonaws.com
     *            * http://play.minio.io
     *            Invalid:
     *            * https://s3-us-west-2.amazonaws.com/example/
     *            * https://s3-us-west-2.amazonaws.com/example/object
     * @return an object storage client backed by an S3 compatible server.
     * @throws MalformedURLException
     */
    public static Client getClient(URL url) throws MalformedURLException {
        if (url == null) {
            throw new NullPointerException();
        }
        // Set trailing / in path
        if (url.getPath().length() == 0) {
            String path = url.toString() + "/";
            url = new URL(path);
        }

        // Only a trailing path should be present in the path
        if (url.getPath().length() > 0 && !url.getPath().equals("/")) {
            throw new MalformedURLException("Path should be empty: '" + url.getPath() + "'");
        }

        // return a new http client
        return new Client(url);
    }

    /**
     * @return an object storage client backed by an S3 compatible server.
     * @throws MalformedURLException
     * @see #getClient(URL url)
     */
    public static Client getClient(String url) throws MalformedURLException {
        if (url == null) {
            throw new NullPointerException();
        }
        return getClient(new URL(url));
    }

    /**
     * Returns the URL this client uses
     *
     * @return the URL backed by this.
     */
    public URL getUrl() {
        return url;
    }

    /**
     * Returns metadata about the object
     *
     * @param bucket object's bucket
     * @param key    object's key
     * @return Populated object metadata
     * @throws IOException
     * @see ObjectMetadata
     */
    public ObjectMetadata getObjectMetadata(String bucket, String key) throws IOException, ObjectStorageException {
        GenericUrl url = getGenericUrlOfKey(bucket, key);
        HttpRequest request = getHttpRequest("HEAD", url);
        request.setThrowExceptionOnExecuteError(false);
        HttpResponse response = request.execute();
        if (response != null) {
            try {
                if (response.isSuccessStatusCode()) {
                    HttpHeaders responseHeaders = response.getHeaders();
                    SimpleDateFormat formatter = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss zzz");
                    Date lastModified = formatter.parse(responseHeaders.getLastModified());
                    return new ObjectMetadata(bucket, key, lastModified, responseHeaders.getContentLength(), responseHeaders.getETag());
                } else {
                    parseError(response);
                }
            } catch (ParseException e) {
                e.printStackTrace();
            } finally {
                response.disconnect();
            }
        }
        throw new IOException();
    }

    private void parseError(HttpResponse response) throws IOException, ObjectStorageException {
        if (response.getContent() == null) {
            throw new IOException("Unsuccessful response from server without error: " + response.getStatusCode());
        }
        XmlError xmlError = new XmlError();
        parseXml(response, xmlError);

        String code = xmlError.getCode();
        ObjectStorageException e;
        if (code.equals("NoSuchBucket")) e = new BucketNotFoundException();
        else if (code.equals("NoSuchKey")) e = new ObjectNotFoundException();
        else if (code.equals("InvalidBucketName")) e = new InvalidObjectNameException();
        else if (code.equals("InvalidObjectName")) e = new InvalidObjectNameException();
        else if (code.equals("AccessDenied")) e = new AccessDeniedException();
        else if (code.equals("BucketAlreadyExists")) e = new BucketExistsException();
        else if (code.equals("ObjectAlreadyExists")) e = new ObjectExistsException();
        else if (code.equals("InternalError")) e = new InternalServerException();
        else if (code.equals("KeyTooLong")) e = new InvalidObjectNameException();
        else if (code.equals("TooManyBuckets")) e = new MaxBucketsReachedException();
        else if (code.equals("PermanentRedirect")) e = new RedirectionException();
        else e = new InvalidStateException();
        e.setXmlError(xmlError);
        throw e;
    }

    private void parseXml(HttpResponse response, XmlError objectToPopulate) throws IOException, InvalidStateException {
        XmlPullParser parser;
        try {
            parser = Xml.createParser();
            InputStreamReader reader = new InputStreamReader(response.getContent(), "UTF-8");
            parser.setInput(reader);
            XmlNamespaceDictionary dictionary = new XmlNamespaceDictionary();
            dictionary.set("s3", "http://s3.amazonaws.com/doc/2006-03-01/");
            dictionary.set("", "");
            Xml.parseElement(parser, objectToPopulate, dictionary, null);
        } catch (XmlPullParserException e) {
            e.printStackTrace();
            InvalidStateException invalidStateException = new InvalidStateException();
            invalidStateException.initCause(e);
            throw invalidStateException;
        }
    }

    private HttpRequest getHttpRequest(String method, GenericUrl url) throws IOException {
        return getHttpRequest(method, url, null);
    }

    private HttpRequest getHttpRequest(String method, GenericUrl url, final byte[] data) throws IOException {
        HttpRequestFactory requestFactory = this.transport.createRequestFactory(new HttpRequestInitializer() {
            @Override
            public void initialize(HttpRequest request) throws IOException {
                RequestSigner signer = new RequestSigner(data);
                signer.setAccessKeys(accessKey, secretKey);
                signer.setSigningKey(signingKey);
                request.setInterceptor(signer);
            }
        });
        HttpRequest request = requestFactory.buildRequest(method, url, null);
        request.setSuppressUserAgentSuffix(true);
        return request;
    }

    private GenericUrl getGenericUrlOfKey(String bucket, String key) {
        GenericUrl url = new GenericUrl(this.url);

        List<String> pathParts = new LinkedList<String>();
        pathParts.add("");
        pathParts.add(bucket);
        pathParts.add(key);

        url.setPathParts(pathParts);
        return url;
    }

    private GenericUrl getGenericUrlOfBucket(String bucket) {
        GenericUrl url = new GenericUrl(this.url);

        List<String> pathParts = new LinkedList<String>();
        pathParts.add("");
        pathParts.add(bucket);

        url.setPathParts(pathParts);
        return url;
    }

    /**
     * Returns an InputStream containing the object. The InputStream must be closed when
     * complete or the connection will remain open.
     *
     * @param bucket object's bucket
     * @param key    object's key
     * @return an InputStream containing the object. Close the InputStream when done.
     * @throws IOException
     */
    public InputStream getObject(String bucket, String key) throws IOException, ObjectStorageException {
        GenericUrl url = getGenericUrlOfKey(bucket, key);

        HttpRequest request = getHttpRequest("GET", url);
        request.setThrowExceptionOnExecuteError(false);
        HttpResponse response = request.execute();
        if (response != null) {
            if (!response.isSuccessStatusCode()) {
                try {
                    parseError(response);
                } finally {
                    response.disconnect();
                }
            }
            return response.getContent();
        }
        // TODO create a better exception
        throw new IOException();
    }

    /**
     * Delete an object.
     *
     * @param bucket object's bucket
     * @param key    object's key
     * @throws IOException if the connection fails
     */
    public void deleteObject(String bucket, String key) throws IOException, ObjectStorageException {
        GenericUrl url = getGenericUrlOfKey(bucket, key);
        HttpRequest request = getHttpRequest("DELETE", url);
        request.setThrowExceptionOnExecuteError(false);
        HttpResponse response = request.execute();
        if (response != null) {
            try {
                if (response.isSuccessStatusCode()) {
                    return;
                }
                parseError(response);
            } finally {
                response.disconnect();
            }
        }
        throw new IOException();
    }

    /**
     * Returns an InputStream containing a subset of the object. The InputStream must be
     * closed or the connection will remain open.
     *
     * @param bucket object's bucket
     * @param key    object's key
     * @param offset Offset from the start of the object.
     * @param length Length of bytes to retrieve.
     * @return an InputStream containing the object. Close the InputStream when done.
     * @throws IOException if the connection does not succeed
     */
    public InputStream getObject(String bucket, String key, long offset, long length) throws IOException, ObjectStorageException {
        GenericUrl url = getGenericUrlOfKey(bucket, key);

        HttpRequest request = getHttpRequest("GET", url);
        request.setThrowExceptionOnExecuteError(false);
        request.getHeaders().setRange(offset + "-" + offset + length);

        HttpResponse response = request.execute();
        if (response != null) {
            if (response.isSuccessStatusCode()) {
                return response.getContent();
            }
            try {
                parseError(response);
            } finally {
                response.disconnect();
            }
        }
        throw new IOException();
    }

    public ExceptionIterator<Item> listObjectsInBucket(final String bucket, final String prefix) {
        return new ExceptionIterator<Item>() {
            private String marker = null;
            private boolean isComplete = false;

            @Override
            protected List<Item> populate() throws ObjectStorageException, IOException {
                if (!isComplete) {
                    try {
                        ListBucketResult listBucketResult = listObjectsInBucket(bucket, marker, prefix, null, 1000);
                        if (listBucketResult.isTruncated()) {
                            marker = listBucketResult.getNextMarker();
                        } else {
                            isComplete = true;
                        }
                        return listBucketResult.getContents();
                    } catch (XmlPullParserException e) {
                        InternalClientException xmlParsingError = new InternalClientException();
                        xmlParsingError.initCause(e);
                        throw xmlParsingError;
                    }
                }
                return new LinkedList<Item>();
            }
        };
    }

    public ExceptionIterator<Item> listObjectsInBucket(final String bucket) {
        return listObjectsInBucket(bucket, null);
    }

    private ListBucketResult listObjectsInBucket(String bucket, String marker, String prefix, String delimiter, Integer maxKeys) throws IOException, XmlPullParserException, ObjectStorageException {
        GenericUrl url = getGenericUrlOfBucket(bucket);
        if (maxKeys != null) {
            url.set("max-keys", maxKeys);
        }
        if (marker != null) {
            url.set("marker", marker);
        }
        if (prefix != null) {
            url.set("prefix", prefix);
        }
        if (delimiter != null) {
            url.set("delimiter", delimiter);
        }

        HttpRequest request = getHttpRequest("GET", url);
        request.setThrowExceptionOnExecuteError(false);

        HttpResponse response = request.execute();

        if (response != null) {
            try {
                if (response.isSuccessStatusCode()) {
                    XmlPullParser parser = Xml.createParser();
                    InputStreamReader reader = new InputStreamReader(response.getContent(), "UTF-8");
                    parser.setInput(reader);

                    ListBucketResult result = new ListBucketResult();

                    Xml.parseElement(parser, result, new XmlNamespaceDictionary(), null);
                    return result;
                }
                parseError(response);
            } finally {
                response.disconnect();
            }
        }
        throw new IOException();
    }

    void setTransport(HttpTransport transport) {
        this.transport = transport;
    }

    /**
     * List buckets owned by the current user.
     *
     * @return a list of buckets owned by the current user
     * @throws IOException            if the connection fails
     * @throws XmlPullParserException
     */
    public ListAllMyBucketsResult listBuckets() throws IOException, XmlPullParserException, ObjectStorageException {
        GenericUrl url = new GenericUrl(this.url);

        HttpRequest request = getHttpRequest("GET", url);
        request.setFollowRedirects(false);

        HttpResponse response = request.execute();
        if (response != null) {
            try {
                if (response.isSuccessStatusCode()) {
                    XmlPullParser parser = Xml.createParser();
                    InputStreamReader reader = new InputStreamReader(response.getContent(), "UTF-8");
                    parser.setInput(reader);

                    ListAllMyBucketsResult result = new ListAllMyBucketsResult();

                    Xml.parseElement(parser, result, new XmlNamespaceDictionary(), null);
                    return result;
                }
                parseError(response);
            } finally {<commit_msg>NetHttpTransport is now shared among all instances of Client<commit_after>/*
 * Minimal Object Storage Library, (C) 2015 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package io.minio.objectstorage.client;

import com.google.api.client.http.*;
import com.google.api.client.http.javanet.NetHttpTransport;
import com.google.api.client.xml.Xml;
import com.google.api.client.xml.XmlNamespaceDictionary;
import io.minio.objectstorage.client.errors.*;
import io.minio.objectstorage.client.messages.*;
import org.xmlpull.v1.XmlPullParser;
import org.xmlpull.v1.XmlPullParserException;

import javax.xml.bind.DatatypeConverter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.MalformedURLException;
import java.net.URL;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Arrays;
import java.util.Date;
import java.util.LinkedList;
import java.util.List;
import java.util.concurrent.atomic.AtomicReference;
import java.util.logging.Handler;
import java.util.logging.Level;
import java.util.logging.LogRecord;
import java.util.logging.Logger;

/**
 * <p>
 * This class implements a simple object storage client. This client consists
 * of a useful subset of S3 compatible functionality.
 * </p>
 * <p/>
 * <h2>Service</h2>
 * <ul>
 * <li>Creating a bucket</li>
 * <li>Listing buckets</li>
 * </ul>
 * <p/>
 * <h2>Bucket</h2>
 * <ul>
 * <li> Creating an object, including automatic upload resuming for large objects.</li>
 * <li> Listing objects in a bucket</li>
 * <li> Listing active multipart uploads</li>
 * <li> Dropping all active multipart uploads</li>
 * <li> Setting canned ACLs on buckets</li>
 * </ul>
 * <p/>
 * <h2>Object</h2>Object
 * <ul>
 * <li>Dropping an active multipart upload for a specific object and uploadId</li>
 * <li>Read object metadata</li>
 * <li>Reading an object</li>
 * <li>Reading a range of bytes of an object</li>
 * <li>Deleting an object</li>
 * </ul>
 * <p/>
 * Optionally, users can also provide access/secret keys or a precomputed
 * signing key to the client. If keys are provided, all requests by the
 * client will be signed using AWS Signature Version 4. @see #setKeys(String, String)
 * <p/>
 * For an example of using this library, please see <a href="https://github.com/minio/objectstorage-java/blob/master/src/test/java/io/minio/objectstorage/example/Example.java">this example</a>.
 */
@SuppressWarnings({"SameParameterValue", "WeakerAccess"})
public class Client {
    /**
     * Canned acl: public-read-write
     * <p/>
     * Read: public
     * Write: public
     */
    @SuppressWarnings("unused")
    public static final String ACL_PUBLIC_READ_WRITE = "public-read-write";
    /**
     * Canned acl: private
     * <p/>
     * Read: authorized users only
     * Write: authorized users only
     */
    @SuppressWarnings("unused")
    public static final String ACL_PRIVATE = "private";
    /**
     * Canned acl: public-read
     * <p/>
     * Read: public
     * Write: authorized users only
     */
    @SuppressWarnings("unused")
    public static final String ACL_PUBLIC_READ = "public-read";
    /**
     * Canned acl: authenticated-read
     * <p/>
     * Read: Only users with a valid account, all valid users authorized
     * Write: acl authorized users only
     */
    @SuppressWarnings("unused")
    public static final String ACL_AUTHENTICATED_READ = "authenticated-read";
    /**
     * Canned acl: bucket-owner-read
     * <p/>
     * Read: Object owner and bucket owner
     * Write: Object owner only
     */
    @SuppressWarnings("unused")
    public static final String ACL_BUCKET_OWNER_READ = "bucket-owner-read";
    /**
     * Canned acl: bucket-owner-read
     * <p/>
     * Read: Object owner and bucket owner
     * Write: Object owner and bucket owner
     */
    @SuppressWarnings("unused")
    public static final String ACL_BUCKET_OWNER_FULL_CONTROL = "bucket-owner-full-control";

    private static final int PART_SIZE = 5 * 1024 * 1024;
    private final URL url;
    private final AtomicReference<Logger> logger = new AtomicReference<Logger>();
    private static final HttpTransport defaultTransport = new NetHttpTransport();
    private HttpTransport transport = defaultTransport;
    private String accessKey;
    private String secretKey;
    private String userAgent = "objectstorage-java/0.0.1" + " (" + System.getProperty("os.name") + ", " + System.getProperty("os.arch") + ") ";
    private byte[] signingKey;

    private Client(URL url) {
        this.url = url;
    }


    /**
     * Create a new client given a url
     *
     * @param url must be the full url to the object storage server, excluding both bucket or object paths.
     *            For example: http://play.minio.io
     *            Valid:
     *            * https://s3-us-west-2.amazonaws.com
     *            * http://play.minio.io
     *            Invalid:
     *            * https://s3-us-west-2.amazonaws.com/example/
     *            * https://s3-us-west-2.amazonaws.com/example/object
     * @return an object storage client backed by an S3 compatible server.
     * @throws MalformedURLException
     */
    public static Client getClient(URL url) throws MalformedURLException {
        if (url == null) {
            throw new NullPointerException();
        }
        // Set trailing / in path
        if (url.getPath().length() == 0) {
            String path = url.toString() + "/";
            url = new URL(path);
        }

        // Only a trailing path should be present in the path
        if (url.getPath().length() > 0 && !url.getPath().equals("/")) {
            throw new MalformedURLException("Path should be empty: '" + url.getPath() + "'");
        }

        // return a new http client
        return new Client(url);
    }

    /**
     * @return an object storage client backed by an S3 compatible server.
     * @throws MalformedURLException
     * @see #getClient(URL url)
     */
    public static Client getClient(String url) throws MalformedURLException {
        if (url == null) {
            throw new NullPointerException();
        }
        return getClient(new URL(url));
    }

    /**
     * Returns the URL this client uses
     *
     * @return the URL backed by this.
     */
    public URL getUrl() {
        return url;
    }

    /**
     * Returns metadata about the object
     *
     * @param bucket object's bucket
     * @param key    object's key
     * @return Populated object metadata
     * @throws IOException
     * @see ObjectMetadata
     */
    public ObjectMetadata getObjectMetadata(String bucket, String key) throws IOException, ObjectStorageException {
        GenericUrl url = getGenericUrlOfKey(bucket, key);
        HttpRequest request = getHttpRequest("HEAD", url);
        request.setThrowExceptionOnExecuteError(false);
        HttpResponse response = request.execute();
        if (response != null) {
            try {
                if (response.isSuccessStatusCode()) {
                    HttpHeaders responseHeaders = response.getHeaders();
                    SimpleDateFormat formatter = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss zzz");
                    Date lastModified = formatter.parse(responseHeaders.getLastModified());
                    return new ObjectMetadata(bucket, key, lastModified, responseHeaders.getContentLength(), responseHeaders.getETag());
                } else {
                    parseError(response);
                }
            } catch (ParseException e) {
                e.printStackTrace();
            } finally {
                response.disconnect();
            }
        }
        throw new IOException();
    }

    private void parseError(HttpResponse response) throws IOException, ObjectStorageException {
        if (response.getContent() == null) {
            throw new IOException("Unsuccessful response from server without error: " + response.getStatusCode());
        }
        XmlError xmlError = new XmlError();
        parseXml(response, xmlError);

        String code = xmlError.getCode();
        ObjectStorageException e;
        if (code.equals("NoSuchBucket")) e = new BucketNotFoundException();
        else if (code.equals("NoSuchKey")) e = new ObjectNotFoundException();
        else if (code.equals("InvalidBucketName")) e = new InvalidObjectNameException();
        else if (code.equals("InvalidObjectName")) e = new InvalidObjectNameException();
        else if (code.equals("AccessDenied")) e = new AccessDeniedException();
        else if (code.equals("BucketAlreadyExists")) e = new BucketExistsException();
        else if (code.equals("ObjectAlreadyExists")) e = new ObjectExistsException();
        else if (code.equals("InternalError")) e = new InternalServerException();
        else if (code.equals("KeyTooLong")) e = new InvalidObjectNameException();
        else if (code.equals("TooManyBuckets")) e = new MaxBucketsReachedException();
        else if (code.equals("PermanentRedirect")) e = new RedirectionException();
        else e = new InvalidStateException();
        e.setXmlError(xmlError);
        throw e;
    }

    private void parseXml(HttpResponse response, XmlError objectToPopulate) throws IOException, InvalidStateException {
        XmlPullParser parser;
        try {
            parser = Xml.createParser();
            InputStreamReader reader = new InputStreamReader(response.getContent(), "UTF-8");
            parser.setInput(reader);
            XmlNamespaceDictionary dictionary = new XmlNamespaceDictionary();
            dictionary.set("s3", "http://s3.amazonaws.com/doc/2006-03-01/");
            dictionary.set("", "");
            Xml.parseElement(parser, objectToPopulate, dictionary, null);
        } catch (XmlPullParserException e) {
            e.printStackTrace();
            InvalidStateException invalidStateException = new InvalidStateException();
            invalidStateException.initCause(e);
            throw invalidStateException;
        }
    }

    private HttpRequest getHttpRequest(String method, GenericUrl url) throws IOException {
        return getHttpRequest(method, url, null);
    }

    private HttpRequest getHttpRequest(String method, GenericUrl url, final byte[] data) throws IOException {
        HttpRequestFactory requestFactory = this.transport.createRequestFactory(new HttpRequestInitializer() {
            @Override
            public void initialize(HttpRequest request) throws IOException {
                RequestSigner signer = new RequestSigner(data);
                signer.setAccessKeys(accessKey, secretKey);
                signer.setSigningKey(signingKey);
                request.setInterceptor(signer);
            }
        });
        HttpRequest request = requestFactory.buildRequest(method, url, null);
        request.setSuppressUserAgentSuffix(true);
        return request;
    }

    private GenericUrl getGenericUrlOfKey(String bucket, String key) {
        GenericUrl url = new GenericUrl(this.url);

        List<String> pathParts = new LinkedList<String>();
        pathParts.add("");
        pathParts.add(bucket);
        pathParts.add(key);

        url.setPathParts(pathParts);
        return url;
    }

    private GenericUrl getGenericUrlOfBucket(String bucket) {
        GenericUrl url = new GenericUrl(this.url);

        List<String> pathParts = new LinkedList<String>();
        pathParts.add("");
        pathParts.add(bucket);

        url.setPathParts(pathParts);
        return url;
    }

    /**
     * Returns an InputStream containing the object. The InputStream must be closed when
     * complete or the connection will remain open.
     *
     * @param bucket object's bucket
     * @param key    object's key
     * @return an InputStream containing the object. Close the InputStream when done.
     * @throws IOException
     */
    public InputStream getObject(String bucket, String key) throws IOException, ObjectStorageException {
        GenericUrl url = getGenericUrlOfKey(bucket, key);

        HttpRequest request = getHttpRequest("GET", url);
        request.setThrowExceptionOnExecuteError(false);
        HttpResponse response = request.execute();
        if (response != null) {
            if (!response.isSuccessStatusCode()) {
                try {
                    parseError(response);
                } finally {
                    response.disconnect();
                }
            }
            return response.getContent();
        }
        // TODO create a better exception
        throw new IOException();
    }

    /**
     * Delete an object.
     *
     * @param bucket object's bucket
     * @param key    object's key
     * @throws IOException if the connection fails
     */
    public void deleteObject(String bucket, String key) throws IOException, ObjectStorageException {
        GenericUrl url = getGenericUrlOfKey(bucket, key);
        HttpRequest request = getHttpRequest("DELETE", url);
        request.setThrowExceptionOnExecuteError(false);
        HttpResponse response = request.execute();
        if (response != null) {
            try {
                if (response.isSuccessStatusCode()) {
                    return;
                }
                parseError(response);
            } finally {
                response.disconnect();
            }
        }
        throw new IOException();
    }

    /**
     * Returns an InputStream containing a subset of the object. The InputStream must be
     * closed or the connection will remain open.
     *
     * @param bucket object's bucket
     * @param key    object's key
     * @param offset Offset from the start of the object.
     * @param length Length of bytes to retrieve.
     * @return an InputStream containing the object. Close the InputStream when done.
     * @throws IOException if the connection does not succeed
     */
    public InputStream getObject(String bucket, String key, long offset, long length) throws IOException, ObjectStorageException {
        GenericUrl url = getGenericUrlOfKey(bucket, key);

        HttpRequest request = getHttpRequest("GET", url);
        request.setThrowExceptionOnExecuteError(false);
        request.getHeaders().setRange(offset + "-" + offset + length);

        HttpResponse response = request.execute();
        if (response != null) {
            if (response.isSuccessStatusCode()) {
                return response.getContent();
            }
            try {
                parseError(response);
            } finally {
                response.disconnect();
            }
        }
        throw new IOException();
    }

    public ExceptionIterator<Item> listObjectsInBucket(final String bucket, final String prefix) {
        return new ExceptionIterator<Item>() {
            private String marker = null;
            private boolean isComplete = false;

            @Override
            protected List<Item> populate() throws ObjectStorageException, IOException {
                if (!isComplete) {
                    try {
                        ListBucketResult listBucketResult = listObjectsInBucket(bucket, marker, prefix, null, 1000);
                        if (listBucketResult.isTruncated()) {
                            marker = listBucketResult.getNextMarker();
                        } else {
                            isComplete = true;
                        }
                        return listBucketResult.getContents();
                    } catch (XmlPullParserException e) {
                        InternalClientException xmlParsingError = new InternalClientException();
                        xmlParsingError.initCause(e);
                        throw xmlParsingError;
                    }
                }
                return new LinkedList<Item>();
            }
        };
    }

    public ExceptionIterator<Item> listObjectsInBucket(final String bucket) {
        return listObjectsInBucket(bucket, null);
    }

    private ListBucketResult listObjectsInBucket(String bucket, String marker, String prefix, String delimiter, Integer maxKeys) throws IOException, XmlPullParserException, ObjectStorageException {
        GenericUrl url = getGenericUrlOfBucket(bucket);
        if (maxKeys != null) {
            url.set("max-keys", maxKeys);
        }
        if (marker != null) {
            url.set("marker", marker);
        }
        if (prefix != null) {
            url.set("prefix", prefix);
        }
        if (delimiter != null) {
            url.set("delimiter", delimiter);
        }

        HttpRequest request = getHttpRequest("GET", url);
        request.setThrowExceptionOnExecuteError(false);

        HttpResponse response = request.execute();

        if (response != null) {
            try {
                if (response.isSuccessStatusCode()) {
                    XmlPullParser parser = Xml.createParser();
                    InputStreamReader reader = new InputStreamReader(response.getContent(), "UTF-8");
                    parser.setInput(reader);

                    ListBucketResult result = new ListBucketResult();

                    Xml.parseElement(parser, result, new XmlNamespaceDictionary(), null);
                    return result;
                }
                parseError(response);
            } finally {
                response.disconnect();
            }
        }
        throw new IOException();
    }

    void setTransport(HttpTransport transport) {
        this.transport = transport;
    }

    void resetTransport() {
        this.transport = defaultTransport;
    }

    /**
     * List buckets owned by the current user.
     *
     * @return a list of buckets owned by the current user
     * @throws IOException            if the connection fails
     * @throws XmlPullParserException
     */
    public ListAllMyBucketsResult listBuckets() throws IOException, XmlPullParserException, ObjectStorageException {
        GenericUrl url = new GenericUrl(this.url);

        HttpRequest request = getHttpRequest("GET", url);
        request.setFollowRedirects(false);

        HttpResponse response = request.execute();
        if (response != null) {
            try {
                if (response.isSuccessStatusCode()) {
                    XmlPullParser parser = Xml.createParser();
                    InputStreamReader reader = new InputStreamReader(response.getContent(), "UTF-8");
                    parser.setInput(reader);

                    ListAllMyBucketsResult result = new ListAllMyBucketsResult();

                    Xml.parseElement(parser, result, new XmlNamespaceDictionary(), null);
                    return result;
                }
                parseError(response);
            } finally {
{'fraction_non_alphanumeric': 0.09803412366582699, 'fraction_numerical': 0.0029605006881865633, 'mean_word_length': 2.3343146592778594, 'pattern_counts': {'":': 0, '<': 127, '<?xml version=': 0, '>': 129, 'https://': 8, 'lorem ipsum': 0, 'www.': 2, 'xml': 24}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3135744', 'n_tokens_mistral': 9936, 'n_tokens_neox': 9598, 'n_words': 2784}