<commit_before>package com.conveyal.datatools.manager.persistence;

import java.io.*;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.util.ArrayList;
import java.util.List;

import com.amazonaws.AmazonClientException;
import com.amazonaws.AmazonServiceException;
import com.amazonaws.services.s3.model.*;
import com.amazonaws.services.s3.transfer.TransferManager;
import com.amazonaws.services.s3.transfer.Upload;
import com.conveyal.datatools.manager.DataManager;
import com.conveyal.datatools.manager.controllers.api.GtfsApiController;
import com.conveyal.datatools.manager.models.FeedSource;
import com.conveyal.gtfs.GTFSFeed;
import gnu.trove.list.TLongList;
import gnu.trove.list.array.TLongArrayList;
import org.apache.commons.io.IOUtils;

import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
import com.amazonaws.auth.profile.ProfileCredentialsProvider;
import com.amazonaws.services.s3.AmazonS3Client;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * Store a feed on the file system or s3
 * @author mattwigway
 *
 */
public class FeedStore {

    public static final Logger LOG = LoggerFactory.getLogger(FeedStore.class);

    /** Local file storage path if working offline */
    public static final File basePath = new File(DataManager.getConfigPropertyAsText("application.data.gtfs"));
    private final File path;
    /** An optional AWS S3 bucket to store the feeds */
    private String s3Bucket;

    public static final String s3Prefix = "gtfs/";

    public static AmazonS3Client s3Client;
    /** An AWS credentials file to use when uploading to S3 */
    private static final String s3CredentialsFilename = DataManager.getConfigPropertyAsText("application.data.s3_credentials_file");

    public FeedStore() {
        this(null);
    }

    /**
     * Allow construction of feed store in a subdirectory (e.g., "gtfsplus")
     * @param subdir
     */
    public FeedStore(String subdir) {

        // even with s3 storage, we keep a local copy, so we'll still set path.
        String pathString = basePath.getAbsolutePath();
        if (subdir != null) pathString += File.separator + subdir;
        path = getPath(pathString);

        // s3 storage
        if (DataManager.useS3){
            this.s3Bucket = DataManager.getConfigPropertyAsText("application.data.gtfs_s3_bucket");
            s3Client = new AmazonS3Client(getAWSCreds());
        }
    }

    private static File getPath (String pathString) {
        File path = new File(pathString);
        if (!path.exists() || !path.isDirectory()) {
            path = null;
            throw new IllegalArgumentException("Not a directory or not found: " + path.getAbsolutePath());
        }
        return path;
    }

    public List<String> getAllFeeds () {
        ArrayList<String> ret = new ArrayList<String>();
        // s3 storage
        if (DataManager.useS3) {
            // TODO: add method for retrieval of all s3 feeds
        }
        // local storage
        else {
            for (File file : path.listFiles()) {
                ret.add(file.getName());
            }
        }

        return ret;
    }

    public Long getFeedLastModified (String id) {
        // s3 storage
        if (DataManager.useS3){
            return s3Client.getObjectMetadata(s3Bucket, getS3Key(id)).getLastModified().getTime();
        }
        else {
            File feed = getFeed(id);
            return feed != null ? feed.lastModified() : null;
        }
    }

    public void deleteFeed (String id) {
        // s3 storage
        if (DataManager.useS3){
            s3Client.deleteObject(s3Bucket, getS3Key(id));
        }
        else {
            File feed = getFeed(id);
            if (feed != null && feed.exists())
                feed.delete();
        }
    }

    public Long getFeedSize (String id) {
        // s3 storage
        if (DataManager.useS3) {
            return s3Client.getObjectMetadata(s3Bucket, getS3Key(id)).getContentLength();
        }
        else {
            File feed = getFeed(id);
            return feed != null ? feed.length() : null;
        }
    }

    private AWSCredentials getAWSCreds () {
        if (this.s3CredentialsFilename != null) {
            return new ProfileCredentialsProvider(this.s3CredentialsFilename, "default").getCredentials();
        } else {
            // default credentials providers, e.g. IAM role
            return new DefaultAWSCredentialsProviderChain().getCredentials();
        }
    }

    private String getS3Key (String id) {
        return s3Prefix + id;
    }

    /**
     * Get the feed with the given ID.
     */
    public File getFeed (String id) {
        // local storage
        if (path != null) {
            File feed = new File(path, id);
            // don't let folks get feeds outside of the directory
            if (feed.getParentFile().equals(path) && feed.exists()) return feed;
        }
        // s3 storage
        else if (s3Bucket != null) {
            AWSCredentials creds = getAWSCreds();
            try {
                LOG.info("Downloading feed from s3");
                S3Object object = s3Client.getObject(
                        new GetObjectRequest(s3Bucket, getS3Key(id)));
                InputStream objectData = object.getObjectContent();

                return createTempFile(objectData);
            } catch (AmazonServiceException ase) {
                LOG.error("Error downloading from s3");
                ase.printStackTrace();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
        return null;
    }

    /**
     * Create a new feed with the given ID.
     */
    public File newFeed (String id, InputStream inputStream, FeedSource feedSource) {
        // s3 storage (store locally and let gtfsCache handle loading feed to s3)
        if (DataManager.useS3) {
            return storeFeedLocally(id, inputStream, feedSource);
        }
        // local storage
        else if (path != null) {
            return storeFeedLocally(id, inputStream, feedSource);
        }
        return null;
    }

    private File storeFeedLocally(String id, InputStream inputStream, FeedSource feedSource) {
        // store latest as feed-source-id.zip
        if (feedSource != null) {<commit_msg>store feed locally initially for both s3 and offline<commit_after>package com.conveyal.datatools.manager.persistence;

import java.io.*;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.util.ArrayList;
import java.util.List;

import com.amazonaws.AmazonClientException;
import com.amazonaws.AmazonServiceException;
import com.amazonaws.services.s3.model.*;
import com.amazonaws.services.s3.transfer.TransferManager;
import com.amazonaws.services.s3.transfer.Upload;
import com.conveyal.datatools.manager.DataManager;
import com.conveyal.datatools.manager.controllers.api.GtfsApiController;
import com.conveyal.datatools.manager.models.FeedSource;
import com.conveyal.gtfs.GTFSFeed;
import gnu.trove.list.TLongList;
import gnu.trove.list.array.TLongArrayList;
import org.apache.commons.io.IOUtils;

import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
import com.amazonaws.auth.profile.ProfileCredentialsProvider;
import com.amazonaws.services.s3.AmazonS3Client;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * Store a feed on the file system or s3
 * @author mattwigway
 *
 */
public class FeedStore {

    public static final Logger LOG = LoggerFactory.getLogger(FeedStore.class);

    /** Local file storage path if working offline */
    public static final File basePath = new File(DataManager.getConfigPropertyAsText("application.data.gtfs"));
    private final File path;
    /** An optional AWS S3 bucket to store the feeds */
    private String s3Bucket;

    public static final String s3Prefix = "gtfs/";

    public static AmazonS3Client s3Client;
    /** An AWS credentials file to use when uploading to S3 */
    private static final String s3CredentialsFilename = DataManager.getConfigPropertyAsText("application.data.s3_credentials_file");

    public FeedStore() {
        this(null);
    }

    /**
     * Allow construction of feed store in a subdirectory (e.g., "gtfsplus")
     * @param subdir
     */
    public FeedStore(String subdir) {

        // even with s3 storage, we keep a local copy, so we'll still set path.
        String pathString = basePath.getAbsolutePath();
        if (subdir != null) pathString += File.separator + subdir;
        path = getPath(pathString);

        // s3 storage
        if (DataManager.useS3){
            this.s3Bucket = DataManager.getConfigPropertyAsText("application.data.gtfs_s3_bucket");
            s3Client = new AmazonS3Client(getAWSCreds());
        }
    }

    private static File getPath (String pathString) {
        File path = new File(pathString);
        if (!path.exists() || !path.isDirectory()) {
            path = null;
            throw new IllegalArgumentException("Not a directory or not found: " + path.getAbsolutePath());
        }
        return path;
    }

    public List<String> getAllFeeds () {
        ArrayList<String> ret = new ArrayList<String>();
        // s3 storage
        if (DataManager.useS3) {
            // TODO: add method for retrieval of all s3 feeds
        }
        // local storage
        else {
            for (File file : path.listFiles()) {
                ret.add(file.getName());
            }
        }

        return ret;
    }

    public Long getFeedLastModified (String id) {
        // s3 storage
        if (DataManager.useS3){
            return s3Client.getObjectMetadata(s3Bucket, getS3Key(id)).getLastModified().getTime();
        }
        else {
            File feed = getFeed(id);
            return feed != null ? feed.lastModified() : null;
        }
    }

    public void deleteFeed (String id) {
        // s3 storage
        if (DataManager.useS3){
            s3Client.deleteObject(s3Bucket, getS3Key(id));
        }
        else {
            File feed = getFeed(id);
            if (feed != null && feed.exists())
                feed.delete();
        }
    }

    public Long getFeedSize (String id) {
        // s3 storage
        if (DataManager.useS3) {
            return s3Client.getObjectMetadata(s3Bucket, getS3Key(id)).getContentLength();
        }
        else {
            File feed = getFeed(id);
            return feed != null ? feed.length() : null;
        }
    }

    private AWSCredentials getAWSCreds () {
        if (this.s3CredentialsFilename != null) {
            return new ProfileCredentialsProvider(this.s3CredentialsFilename, "default").getCredentials();
        } else {
            // default credentials providers, e.g. IAM role
            return new DefaultAWSCredentialsProviderChain().getCredentials();
        }
    }

    private String getS3Key (String id) {
        return s3Prefix + id;
    }

    /**
     * Get the feed with the given ID.
     */
    public File getFeed (String id) {
        // local storage
        if (path != null) {
            File feed = new File(path, id);
            // don't let folks get feeds outside of the directory
            if (feed.getParentFile().equals(path) && feed.exists()) return feed;
        }
        // s3 storage
        else if (s3Bucket != null) {
            AWSCredentials creds = getAWSCreds();
            try {
                LOG.info("Downloading feed from s3");
                S3Object object = s3Client.getObject(
                        new GetObjectRequest(s3Bucket, getS3Key(id)));
                InputStream objectData = object.getObjectContent();

                return createTempFile(objectData);
            } catch (AmazonServiceException ase) {
                LOG.error("Error downloading from s3");
                ase.printStackTrace();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
        return null;
    }

    /**
     * Create a new feed with the given ID.
     */
    public File newFeed (String id, InputStream inputStream, FeedSource feedSource) {
        // For s3 storage (store locally and let gtfsCache handle loading feed to s3)
        return storeFeedLocally(id, inputStream, feedSource);
    }

    private File storeFeedLocally(String id, InputStream inputStream, FeedSource feedSource) {
        // store latest as feed-source-id.zip
        if (feedSource != null) {
{'fraction_non_alphanumeric': 0.09912467471019636, 'fraction_numerical': 0.008832111032252976, 'mean_word_length': 2.305186343497524, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6962480', 'n_tokens_mistral': 3584, 'n_tokens_neox': 3512, 'n_words': 921}