<commit_before>/*
 * Copyright (C) 2016 <NAME>.
 * All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package aot.storage.s3;

import aot.storage.Storage;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.regions.Region;
import com.amazonaws.regions.Regions;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.iterable.S3Objects;
import com.amazonaws.services.s3.model.ListObjectsRequest;
import com.amazonaws.services.s3.model.ObjectListing;
import com.amazonaws.services.s3.model.S3ObjectSummary;

import java.io.InputStream;
import java.net.URL;
import java.util.Iterator;
import java.util.Map;
import java.util.NoSuchElementException;

/**
 * @author <NAME>
 * @since 1.0
 */
public class CustomStorage extends Storage {
    protected final AmazonS3 s3;

    public CustomStorage(URL url, String[] ids) {
        this(url.getHost(), url.getPath(), createS3(ids));
    }

    public CustomStorage(String bucket, String prefix, AmazonS3 s3) {
        super("s3", bucket, prefix);

        this.s3 = s3;
    }

    public AmazonS3 getS3() {
        return s3;
    }

    @Override
    public String getHttpsUrl() {
        return null;
    }

    @Override
    public Storage getSubstorage(String prefix) {
        return new CustomStorage(bucket, this.prefix + prefix, s3);
    }

    @Override
    public Iterable<String> find(final String prefix, String filter) {
        final String pfx = this.prefix + prefix;
        return new Iterable<String>() {
            private final S3Objects s3Objects = S3Objects.withPrefix(s3, bucket, pfx).withBatchSize(65536);

            @Override
            public Iterator<String> iterator() {
                return new Iterator<String>() {
                    private final int length = pfx.length();
                    private final Iterator<S3ObjectSummary> iterator = s3Objects.iterator();

                    @Override
                    public boolean hasNext() {
                        return iterator.hasNext();
                    }

                    @Override
                    public String next() {
                        return iterator.next().getKey().substring(length);
                    }

                    @Override
                    public void remove() {
                        throw new UnsupportedOperationException("remove()");
                    }
                };
            }
        };
    }

    @Override
    public Iterable<String> list(String prefix, String filter) {
        final String pfx = this.prefix + prefix;
        return new Iterable<String>() {
            private final ListObjectsRequest request = new ListObjectsRequest(bucket, pfx, null, "/", 65536);

            @Override
            public Iterator<String> iterator() {
                return new Iterator<String>() {
                    private ObjectListing listing = s3.listObjects(request);
                    private Iterator<String> iterator = listing.getCommonPrefixes().iterator();
                    private String directory = findNext();

                    private String findNext() {
                        if (iterator.hasNext()) {
                            return iterator.next();
                        } else {
                            while (listing.isTruncated()) {
                                listing = s3.listNextBatchOfObjects(listing);
                                iterator = listing.getCommonPrefixes().iterator();
                                if (iterator.hasNext()) {
                                    return iterator.next();
                                }
                            }
                            return null;
                        }
                    }

                    @Override
                    public boolean hasNext() {
                        return directory != null;
                    }

                    @Override
                    public String next() {
                        String v = directory;
                        if (v != null) {
                            directory = findNext();
                            return v;
                        } else {
                            throw new NoSuchElementException("Next directory is not found");
                        }
                    }

                    @Override
                    public void remove() {
                        throw new UnsupportedOperationException("remove()");
                    }
                };
            }
        };
    }

    @Override
    public byte[] get(String key, Map<String, String> meta) {
        return new byte[0];
    }

    @Override
    public InputStream getStream(String key, Map<String, String> meta) {
        return null;
    }

    @Override
    public long put(String key, byte[] data, Map<String, String> meta) {
        return 0;
    }

    @Override
    public long upload(String key, InputStream input, long size, Map<String, String> meta) {<commit_msg>added list exception to storage implementation<commit_after>/*
 * Copyright (C) 2016 <NAME>.
 * All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package aot.storage.s3;

import aot.storage.ListStorageException;
import aot.storage.Storage;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.regions.Region;
import com.amazonaws.regions.Regions;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.iterable.S3Objects;
import com.amazonaws.services.s3.model.ListObjectsRequest;
import com.amazonaws.services.s3.model.ObjectListing;
import com.amazonaws.services.s3.model.S3ObjectSummary;

import java.io.InputStream;
import java.net.URL;
import java.util.Iterator;
import java.util.Map;
import java.util.NoSuchElementException;

/**
 * @author <NAME>
 * @since 1.0
 */
public class CustomStorage extends Storage {
    protected final AmazonS3 s3;

    public CustomStorage(URL url, String[] ids) {
        this(url.getHost(), url.getPath(), createS3(ids));
    }

    public CustomStorage(String bucket, String prefix, AmazonS3 s3) {
        super("s3", bucket, prefix);

        this.s3 = s3;
    }

    public AmazonS3 getS3() {
        return s3;
    }

    @Override
    public String getHttpsUrl() {
        return null;
    }

    @Override
    public Storage getSubstorage(String prefix) {
        return new CustomStorage(bucket, this.prefix + prefix, s3);
    }

    @Override
    public Iterable<String> find(String prefix, String filter) {
        final String pfx = this.prefix + prefix;
        return new Iterable<String>() {
            private final S3Objects s3Objects = S3Objects.withPrefix(s3, bucket, pfx).withBatchSize(65536);

            @Override
            public Iterator<String> iterator() {
                return new Iterator<String>() {
                    private final int length = pfx.length();
                    private final Iterator<S3ObjectSummary> iterator = s3Objects.iterator();

                    @Override
                    public boolean hasNext() {
                        return iterator.hasNext();
                    }

                    @Override
                    public String next() {
                        return iterator.next().getKey().substring(length);
                    }

                    @Override
                    public void remove() {
                        throw new UnsupportedOperationException("remove()");
                    }
                };
            }
        };
    }

    @Override
    public Iterable<String> list(String prefix, String filter) {
        final String pfx = this.prefix + prefix;
        return new Iterable<String>() {
            private final ListObjectsRequest request = new ListObjectsRequest(bucket, pfx, null, "/", 65536);

            @Override
            public Iterator<String> iterator() {
                try {
                    return new Iterator<String>() {
                        private ObjectListing listing = s3.listObjects(request);
                        private Iterator<String> iterator = listing.getCommonPrefixes().iterator();
                        private String directory = findNext();

                        private String findNext() {
                            if (iterator.hasNext()) {
                                return iterator.next();
                            } else {
                                while (listing.isTruncated()) {
                                    listing = s3.listNextBatchOfObjects(listing);
                                    iterator = listing.getCommonPrefixes().iterator();
                                    if (iterator.hasNext()) {
                                        return iterator.next();
                                    }
                                }
                                return null;
                            }
                        }

                        @Override
                        public boolean hasNext() {
                            return directory != null;
                        }

                        @Override
                        public String next() {
                            String v = directory;
                            if (v != null) {
                                try {
                                    directory = findNext();
                                    return v;
                                } catch (Exception e) {
                                    throw new ListStorageException(CustomStorage.this, e);
                                }
                            } else {
                                throw new NoSuchElementException("Next directory is not found");
                            }
                        }

                        @Override
                        public void remove() {
                            throw new UnsupportedOperationException("remove()");
                        }
                    };
                } catch (Exception e) {
                    throw new ListStorageException(CustomStorage.this, e);
                }
            }
        };
    }

    @Override
    public byte[] get(String key, Map<String, String> meta) {
        return new byte[0];
    }

    @Override
    public InputStream getStream(String key, Map<String, String> meta) {
        return null;
    }

    @Override
    public long put(String key, byte[] data, Map<String, String> meta) {
        return 0;
    }

    @Override
    public long upload(String key, InputStream input, long size, Map<String, String> meta) {
{'fraction_non_alphanumeric': 0.0849378088260351, 'fraction_numerical': 0.0090304992332595, 'mean_word_length': 1.487603305785124, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 11, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '127076', 'n_tokens_mistral': 2844, 'n_tokens_neox': 2684, 'n_words': 774}