<commit_before>
package se.lagrummet.rinfo.collector.atom;

import java.io.IOException;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.URL;
import java.security.KeyManagementException;
import java.security.KeyStoreException;
import java.security.NoSuchAlgorithmException;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.DefaultHttpClient;

import org.apache.abdera.Abdera;
import org.apache.abdera.i18n.iri.IRI;
import org.apache.abdera.model.Feed;
import org.apache.abdera.model.Document;
import se.lagrummet.rinfo.collector.ParseFeedException;


public abstract class FeedArchiveReader {

    private final Logger logger = LoggerFactory.getLogger(FeedArchiveReader.class);

    public static final String LINK_NEXT_ARCHIVE_REL = "next-archive";
    public static final String LINK_PREV_ARCHIVE_REL = "prev-archive";

    private HttpClient httpClient;

    /**
     * This default constructor uses a new {@link DefaultHttpClient} with no
     * special settings.
     */
    public FeedArchiveReader() {
        this(new DefaultHttpClient());
    }

    public FeedArchiveReader(HttpClient httpClient) {
        this.httpClient = httpClient;
    }

    public final HttpClient getClient() {
        return httpClient;
    }

    /**
     * Called before {@link readFeed}.
     */
    public void initialize() {
    }

    /**
     * Always called when {@link readFeed} has completed. This shuts down the
     * HttpClient.
     */
    public void shutdown() {
        getClient().getConnectionManager().shutdown();
        logger.info("Closed down the HTTP client connection manager.");
    }

    /**
     * Starts the feed archive climbing.
     */
    public final void readFeed(URL url) throws Exception {
        initialize();
        try {
            beforeTraversal();
            URL followingUrl = url;
            while (followingUrl != null) {
                followingUrl = readFeedPage(followingUrl);
                if (followingUrl != null) {
                    logger.info(".. following: <"+followingUrl+">");
                }
            }
            afterTraversal();
            logger.info("Done.");
        } finally {
            shutdown();
        }
    }

    /**
     * Called before {@link readFeed} begins page traversal. Does nothing by
     * default.
     */
    public void beforeTraversal() throws Exception {
    }

    /**
     * Called when {@link readFeed} has (successfully) traversed all feed
     * pages. Does nothing by default.
     */
    public void afterTraversal() throws Exception {
    }

    /**
     * <p>Parses the feed and delegates to {@link processFeedPage} for handling.
     * Then looks for contained archive links to continue climbing (backwards
     * in time).</p>
     * <p>Only override this if you need to e.g. stop climbing at a certain
     * known archive URL.</p>
     *
     * @return the previous archive URL, or null to stop climbing.
     */
    public URL readFeedPage(URL url) throws IOException, ParseFeedException {
        logger.info(String.format("Reading Feed <%s> ...", url));
        Feed feed;
        URL followingUrl = null;
        /* TODO:IMPROVE:
            optional use of supplied "If-Modified-Since"/"ETag"..
            .. from template method(?): getFeedPageResponseCache().getLastReadTimeTag(url)..
         */
        InputStream inStream = getResponseAsInputStream(url);
        try {
            feed = parseFeed(inStream, url);

            if (processFeedPage(url, feed)) {
                IRI followingHref = feed.getLinkResolvedHref(getFollowingPageRel());
                if (followingHref != null) {
                    followingUrl = followingHref.toURL();
                }
            }
        } catch (ParseFeedException e) {
            logger.error(String.format("Error parsing feed: <%s>", url), e);
            throw e;
        } catch (Exception e) {
            logger.error(String.format("Error parsing feed: <%s>", url), e);
            throw new RuntimeException(e); /* TODO: stop on error?
            followingUrl = null; */
        } finally {
            inStream.close();
        }
        return followingUrl;
    }

    /**
     * Default method for getting the link relation name of the following page
     * to read from the current feed page.
     *
     * @return Defaults to {@link LINK_PREV_ARCHIVE_REL}.
     */
    public String getFollowingPageRel() {
        return LINK_PREV_ARCHIVE_REL;
    }

    /**
     * Template method intended for the feed processing.
     * @return whether to continue backwards in time or stop.
     */
    public abstract boolean processFeedPage(URL pageUrl, Feed feed) throws Exception;

    public static Feed parseFeed(InputStream inStream, URL baseUrl) throws ParseFeedException {
        Feed feed;

        try {
            Document document = Abdera.getInstance().getParser().parse(inStream, baseUrl.toString());
            feed = (Feed) document.getRoot();
        } catch (Exception e) {
            throw new ParseFeedException("Kan inte läsa källa: " + baseUrl);
        }

        return feed;
    }

    // TODO: define and make a default impl. (use a single dir, only latest feedUrl/feeId?)
    ///**
    // * Optional getter for a {@link FeedPageResponseCache}, used for
    // * conditional requests to optimize retrieval. See that interface for
    // * details.
    // * @return null by default.
    // */
    //public FeedPageResponseCache getFeedPageResponseCache() {
    //    return null;<commit_msg>Update error message (when feed can't be parsed). issue#48<commit_after>
package se.lagrummet.rinfo.collector.atom;

import java.io.IOException;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.URL;
import java.security.KeyManagementException;
import java.security.KeyStoreException;
import java.security.NoSuchAlgorithmException;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.DefaultHttpClient;

import org.apache.abdera.Abdera;
import org.apache.abdera.i18n.iri.IRI;
import org.apache.abdera.model.Feed;
import org.apache.abdera.model.Document;
import se.lagrummet.rinfo.collector.ParseFeedException;


public abstract class FeedArchiveReader {

    private final Logger logger = LoggerFactory.getLogger(FeedArchiveReader.class);

    public static final String LINK_NEXT_ARCHIVE_REL = "next-archive";
    public static final String LINK_PREV_ARCHIVE_REL = "prev-archive";

    private HttpClient httpClient;

    /**
     * This default constructor uses a new {@link DefaultHttpClient} with no
     * special settings.
     */
    public FeedArchiveReader() {
        this(new DefaultHttpClient());
    }

    public FeedArchiveReader(HttpClient httpClient) {
        this.httpClient = httpClient;
    }

    public final HttpClient getClient() {
        return httpClient;
    }

    /**
     * Called before {@link readFeed}.
     */
    public void initialize() {
    }

    /**
     * Always called when {@link readFeed} has completed. This shuts down the
     * HttpClient.
     */
    public void shutdown() {
        getClient().getConnectionManager().shutdown();
        logger.info("Closed down the HTTP client connection manager.");
    }

    /**
     * Starts the feed archive climbing.
     */
    public final void readFeed(URL url) throws Exception {
        initialize();
        try {
            beforeTraversal();
            URL followingUrl = url;
            while (followingUrl != null) {
                followingUrl = readFeedPage(followingUrl);
                if (followingUrl != null) {
                    logger.info(".. following: <"+followingUrl+">");
                }
            }
            afterTraversal();
            logger.info("Done.");
        } finally {
            shutdown();
        }
    }

    /**
     * Called before {@link readFeed} begins page traversal. Does nothing by
     * default.
     */
    public void beforeTraversal() throws Exception {
    }

    /**
     * Called when {@link readFeed} has (successfully) traversed all feed
     * pages. Does nothing by default.
     */
    public void afterTraversal() throws Exception {
    }

    /**
     * <p>Parses the feed and delegates to {@link processFeedPage} for handling.
     * Then looks for contained archive links to continue climbing (backwards
     * in time).</p>
     * <p>Only override this if you need to e.g. stop climbing at a certain
     * known archive URL.</p>
     *
     * @return the previous archive URL, or null to stop climbing.
     */
    public URL readFeedPage(URL url) throws IOException, ParseFeedException {
        logger.info(String.format("Reading Feed <%s> ...", url));
        Feed feed;
        URL followingUrl = null;
        /* TODO:IMPROVE:
            optional use of supplied "If-Modified-Since"/"ETag"..
            .. from template method(?): getFeedPageResponseCache().getLastReadTimeTag(url)..
         */
        InputStream inStream = getResponseAsInputStream(url);
        try {
            feed = parseFeed(inStream, url);

            if (processFeedPage(url, feed)) {
                IRI followingHref = feed.getLinkResolvedHref(getFollowingPageRel());
                if (followingHref != null) {
                    followingUrl = followingHref.toURL();
                }
            }
        } catch (ParseFeedException e) {
            logger.error(String.format("Error parsing feed: <%s>. Not a valid feed?", url), e);
            throw e;
        } catch (Exception e) {
            logger.error(String.format("Error parsing feed: <%s>", url), e);
            throw new RuntimeException(e); /* TODO: stop on error?
            followingUrl = null; */
        } finally {
            inStream.close();
        }
        return followingUrl;
    }

    /**
     * Default method for getting the link relation name of the following page
     * to read from the current feed page.
     *
     * @return Defaults to {@link LINK_PREV_ARCHIVE_REL}.
     */
    public String getFollowingPageRel() {
        return LINK_PREV_ARCHIVE_REL;
    }

    /**
     * Template method intended for the feed processing.
     * @return whether to continue backwards in time or stop.
     */
    public abstract boolean processFeedPage(URL pageUrl, Feed feed) throws Exception;

    public static Feed parseFeed(InputStream inStream, URL baseUrl) throws ParseFeedException {
        Feed feed;

        try {
            Document document = Abdera.getInstance().getParser().parse(inStream, baseUrl.toString());
            feed = (Feed) document.getRoot();
        } catch (Exception e) {
            throw new ParseFeedException("Kan inte läsa källa. Kontrollera att rätt URL är angiven och att källan är korrekt.");
        }

        return feed;
    }

    // TODO: define and make a default impl. (use a single dir, only latest feedUrl/feeId?)
    ///**
    // * Optional getter for a {@link FeedPageResponseCache}, used for
    // * conditional requests to optimize retrieval. See that interface for
    // * details.
    // * @return null by default.
    // */
    //public FeedPageResponseCache getFeedPageResponseCache() {
    //    return null;
{'fraction_non_alphanumeric': 0.10145052667933, 'fraction_numerical': 0.0008634087376964255, 'mean_word_length': 2.3603133159268928, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5112278', 'n_tokens_mistral': 3205, 'n_tokens_neox': 3061, 'n_words': 903}