<commit_before>package com.conveyal.datatools.manager.models;

import com.conveyal.datatools.editor.datastore.FeedTx;
import com.conveyal.datatools.editor.datastore.GlobalTx;
import com.conveyal.datatools.editor.datastore.VersionedDataStore;
import com.conveyal.datatools.manager.DataManager;
import com.conveyal.datatools.manager.auth.Auth0UserProfile;
import com.conveyal.datatools.manager.jobs.NotifyUsersForSubscriptionJob;
import com.conveyal.datatools.manager.persistence.DataStore;
import com.conveyal.datatools.manager.utils.HashUtils;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonView;
import com.google.common.eventbus.EventBus;
import org.mapdb.Fun;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import spark.HaltException;

import java.io.File;
import java.io.IOException;
import java.io.InvalidClassException;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.*;
import java.util.stream.Collectors;

import static spark.Spark.halt;

/**
 * Created by demory on 3/22/16.
 */
@JsonIgnoreProperties(ignoreUnknown = true)
public class FeedSource extends Model implements Cloneable {
    private static final long serialVersionUID = 1L;

    public static final Logger LOG = LoggerFactory.getLogger(FeedSource.class);

    private static DataStore<FeedSource> sourceStore = new DataStore<FeedSource>("feedsources");

    /**
     * The collection of which this feed is a part
     */
    //@JsonView(JsonViews.DataDump.class)
    public String projectId;

    public String[] regions = {"1"};
    /**
     * Get the Project of which this feed is a part
     */
    @JsonIgnore
    public Project getProject () {
        return Project.get(projectId);
    }

    @JsonIgnore
    public List<Region> getRegionList () {
        return Region.getAll().stream().filter(r -> Arrays.asList(regions).contains(r.id)).collect(Collectors.toList());
    }

    public void setProject(Project proj) {
        this.projectId = proj.id;
        this.save();
        proj.save();
    }

    /** The name of this feed source, e.g. MTA New York City Subway */
    public String name;

    /** Is this feed public, i.e. should it be listed on the
     * public feeds page for download?
     */
    public boolean isPublic;

    /** Is this feed deployable? */
    public boolean deployable;

    /**
     * How do we receive this feed?
     */
    public FeedRetrievalMethod retrievalMethod;

    /**
     * When was this feed last fetched?
     */
    public Date lastFetched;

    /**
     * When was this feed last updated?
     */
    public transient Date lastUpdated;

    /**
     * From whence is this feed fetched?
     */
    public URL url;

    /**
     * What is the GTFS Editor snapshot for this feed?
     *
     * This is the String-formatted snapshot ID, which is the base64-encoded ID and the version number.
     */
    public String snapshotVersion;

    public String publishedVersionId;

    /**
     * Create a new feed.
     */
    public FeedSource (String name) {
        super();
        this.name = name;
        this.retrievalMethod = FeedRetrievalMethod.MANUALLY_UPLOADED;
    }

    /**
     * No-arg constructor to yield an uninitialized feed source, for dump/restore.
     * Should not be used in general code.
     */
    public FeedSource () {
        this(null);
    }

    /**
     * Fetch the latest version of the feed.
     */
    public FeedVersion fetch (EventBus eventBus, String fetchUser) {
        Map<String, Object> statusMap = new HashMap<>();
        statusMap.put("message", "Downloading file");
        statusMap.put("percentComplete", 20.0);
        statusMap.put("error", false);
        eventBus.post(statusMap);

        FeedVersion latest = getLatest();

        // We create a new FeedVersion now, so that the fetched date is (milliseconds) before
        // fetch occurs. That way, in the highly unlikely event that a feed is updated while we're
        // fetching it, we will not miss a new feed.
        FeedVersion version = new FeedVersion(this);

        // build the URL from which to fetch
        URL url = this.url;
        LOG.info("Fetching from {}", url.toString());

        // make the request, using the proper HTTP caching headers to prevent refetch, if applicable
        HttpURLConnection conn;
        try {
            conn = (HttpURLConnection) url.openConnection();
        } catch (IOException e) {
            String message = String.format("Unable to open connection to %s; not fetching feed %s", url, this.name);
            LOG.error(message);
            statusMap.put("message", message);
            statusMap.put("percentComplete", 0.0);
            statusMap.put("error", true);
            eventBus.post(statusMap);
            halt(400, message);
            return null;
        } catch (ClassCastException e) {
            String message = String.format("Unable to open connection to %s; not fetching %s feed", url, this.name);
            LOG.error(message);
            statusMap.put("message", message);
            statusMap.put("percentComplete", 0.0);
            statusMap.put("error", true);
            eventBus.post(statusMap);
            halt(400, message);
            return null;
        } catch (NullPointerException e) {
            String message = String.format("Unable to open connection to %s; not fetching %s feed", url, this.name);
            LOG.error(message);
            statusMap.put("message", message);
            statusMap.put("percentComplete", 0.0);
            statusMap.put("error", true);
            eventBus.post(statusMap);
            halt(400, message);
            return null;
        }

        conn.setDefaultUseCaches(true);

        // lastFetched is set to null when the URL changes and when latest feed version is deleted
        if (latest != null && this.lastFetched != null)
            conn.setIfModifiedSince(Math.min(latest.updated.getTime(), this.lastFetched.getTime()));

        File newGtfsFile;

        try {
            conn.connect();

            if (conn.getResponseCode() == HttpURLConnection.HTTP_NOT_MODIFIED) {
                String message = String.format("Feed %s has not been modified", this.name);
                LOG.info(message);
                statusMap.put("message", message);
                statusMap.put("percentComplete", 100.0);
                statusMap.put("error", true);
                eventBus.post(statusMap);
                halt(304, message);
                return null;
            }

            // TODO: redirects
            else if (conn.getResponseCode() == HttpURLConnection.HTTP_OK) {
                String message = String.format("Saving %s feed.", this.name);
                LOG.info(message);
                statusMap.put("message", message);<commit_msg>change log to warn if not modified<commit_after>package com.conveyal.datatools.manager.models;

import com.conveyal.datatools.editor.datastore.FeedTx;
import com.conveyal.datatools.editor.datastore.GlobalTx;
import com.conveyal.datatools.editor.datastore.VersionedDataStore;
import com.conveyal.datatools.manager.DataManager;
import com.conveyal.datatools.manager.auth.Auth0UserProfile;
import com.conveyal.datatools.manager.jobs.NotifyUsersForSubscriptionJob;
import com.conveyal.datatools.manager.persistence.DataStore;
import com.conveyal.datatools.manager.utils.HashUtils;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonView;
import com.google.common.eventbus.EventBus;
import org.mapdb.Fun;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import spark.HaltException;

import java.io.File;
import java.io.IOException;
import java.io.InvalidClassException;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.*;
import java.util.stream.Collectors;

import static spark.Spark.halt;

/**
 * Created by demory on 3/22/16.
 */
@JsonIgnoreProperties(ignoreUnknown = true)
public class FeedSource extends Model implements Cloneable {
    private static final long serialVersionUID = 1L;

    public static final Logger LOG = LoggerFactory.getLogger(FeedSource.class);

    private static DataStore<FeedSource> sourceStore = new DataStore<FeedSource>("feedsources");

    /**
     * The collection of which this feed is a part
     */
    //@JsonView(JsonViews.DataDump.class)
    public String projectId;

    public String[] regions = {"1"};
    /**
     * Get the Project of which this feed is a part
     */
    @JsonIgnore
    public Project getProject () {
        return Project.get(projectId);
    }

    @JsonIgnore
    public List<Region> getRegionList () {
        return Region.getAll().stream().filter(r -> Arrays.asList(regions).contains(r.id)).collect(Collectors.toList());
    }

    public void setProject(Project proj) {
        this.projectId = proj.id;
        this.save();
        proj.save();
    }

    /** The name of this feed source, e.g. MTA New York City Subway */
    public String name;

    /** Is this feed public, i.e. should it be listed on the
     * public feeds page for download?
     */
    public boolean isPublic;

    /** Is this feed deployable? */
    public boolean deployable;

    /**
     * How do we receive this feed?
     */
    public FeedRetrievalMethod retrievalMethod;

    /**
     * When was this feed last fetched?
     */
    public Date lastFetched;

    /**
     * When was this feed last updated?
     */
    public transient Date lastUpdated;

    /**
     * From whence is this feed fetched?
     */
    public URL url;

    /**
     * What is the GTFS Editor snapshot for this feed?
     *
     * This is the String-formatted snapshot ID, which is the base64-encoded ID and the version number.
     */
    public String snapshotVersion;

    public String publishedVersionId;

    /**
     * Create a new feed.
     */
    public FeedSource (String name) {
        super();
        this.name = name;
        this.retrievalMethod = FeedRetrievalMethod.MANUALLY_UPLOADED;
    }

    /**
     * No-arg constructor to yield an uninitialized feed source, for dump/restore.
     * Should not be used in general code.
     */
    public FeedSource () {
        this(null);
    }

    /**
     * Fetch the latest version of the feed.
     */
    public FeedVersion fetch (EventBus eventBus, String fetchUser) {
        Map<String, Object> statusMap = new HashMap<>();
        statusMap.put("message", "Downloading file");
        statusMap.put("percentComplete", 20.0);
        statusMap.put("error", false);
        eventBus.post(statusMap);

        FeedVersion latest = getLatest();

        // We create a new FeedVersion now, so that the fetched date is (milliseconds) before
        // fetch occurs. That way, in the highly unlikely event that a feed is updated while we're
        // fetching it, we will not miss a new feed.
        FeedVersion version = new FeedVersion(this);

        // build the URL from which to fetch
        URL url = this.url;
        LOG.info("Fetching from {}", url.toString());

        // make the request, using the proper HTTP caching headers to prevent refetch, if applicable
        HttpURLConnection conn;
        try {
            conn = (HttpURLConnection) url.openConnection();
        } catch (IOException e) {
            String message = String.format("Unable to open connection to %s; not fetching feed %s", url, this.name);
            LOG.error(message);
            statusMap.put("message", message);
            statusMap.put("percentComplete", 0.0);
            statusMap.put("error", true);
            eventBus.post(statusMap);
            halt(400, message);
            return null;
        } catch (ClassCastException e) {
            String message = String.format("Unable to open connection to %s; not fetching %s feed", url, this.name);
            LOG.error(message);
            statusMap.put("message", message);
            statusMap.put("percentComplete", 0.0);
            statusMap.put("error", true);
            eventBus.post(statusMap);
            halt(400, message);
            return null;
        } catch (NullPointerException e) {
            String message = String.format("Unable to open connection to %s; not fetching %s feed", url, this.name);
            LOG.error(message);
            statusMap.put("message", message);
            statusMap.put("percentComplete", 0.0);
            statusMap.put("error", true);
            eventBus.post(statusMap);
            halt(400, message);
            return null;
        }

        conn.setDefaultUseCaches(true);

        // lastFetched is set to null when the URL changes and when latest feed version is deleted
        if (latest != null && this.lastFetched != null)
            conn.setIfModifiedSince(Math.min(latest.updated.getTime(), this.lastFetched.getTime()));

        File newGtfsFile;

        try {
            conn.connect();

            if (conn.getResponseCode() == HttpURLConnection.HTTP_NOT_MODIFIED) {
                String message = String.format("Feed %s has not been modified", this.name);
                LOG.warn(message);
                statusMap.put("message", message);
                statusMap.put("percentComplete", 100.0);
                statusMap.put("error", true);
                eventBus.post(statusMap);
                halt(304, message);
                return null;
            }

            // TODO: redirects
            else if (conn.getResponseCode() == HttpURLConnection.HTTP_OK) {
                String message = String.format("Saving %s feed.", this.name);
                LOG.info(message);
                statusMap.put("message", message);
{'fraction_non_alphanumeric': 0.10483467828322503, 'fraction_numerical': 0.005284581875312433, 'mean_word_length': 2.475800446760983, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4824262', 'n_tokens_mistral': 3977, 'n_tokens_neox': 3804, 'n_words': 1077}