<commit_before>/*
    ***** BEGIN LICENSE BLOCK *****
    
    Copyright © 2015 Center for History and New Media
                     George Mason University, Fairfax, Virginia, USA
                     http://zotero.org
    
    This file is part of Zotero.
    
    Zotero is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.
    
    Zotero is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.
    
    You should have received a copy of the GNU Affero General Public License
    along with Zotero.  If not, see <http://www.gnu.org/licenses/>.
    
    ***** END LICENSE BLOCK *****
*/


/**
 * Sample feeds:
 * 
 * http://cyber.law.harvard.edu/rss/examples/rss2sample.xml
 * http://feeds.feedburner.com/acs/acbcct
 * http://www.cell.com/molecular-cell/current.rss
 * http://ieeexplore.ieee.org/search/searchresult.jsp?searchField%3DSearch_All%26queryText%3Dwater&searchOrigin=saved_searches&rssFeed=true&rssFeedName=water
 * http://www.sciencemag.org/rss/current.xml
 * http://rss.sciencedirect.com/publication/science/20925212
 * http://www.ncbi.nlm.nih.gov/entrez/eutils/erss.cgi?rss_guid=1fmfIeN4X5Q8HemTZD5Rj6iu6-FQVCn7xc7_IPIIQtS1XiD9bf
 * http://export.arxiv.org/rss/astro-ph
 * http://fhs.dukejournals.org/rss_feeds/recent.xml
 */

/**
 * class Zotero.FeedReader
 * Asynchronously reads an ATOM/RSS feed
 *
 * @param {String} url URL of the feed
 *
 * @property {Zotero.Promise<Object>} feedProperties An object
 *   representing feed properties
 * @property {Zotero.Promise<FeedItem>*} ItemIterator Returns an iterator
 *   for feed items. The iterator returns FeedItem promises that have to be
 *   resolved before requesting the next promise. When all items are exhausted.
 *   the promise resolves to null.
 * @method {void} terminate Stops retrieving/parsing the feed. Data parsed up
 *   to this point is still available.
 */
Zotero.FeedReader = function(url) {
	if (!url) throw new Error("Feed URL must be supplied");

	
	this._url = url;
	this._feedItems = [Zotero.Promise.defer()];
	this._feedProcessed = Zotero.Promise.defer();

	let feedFetched = Zotero.Promise.defer();
	feedFetched.promise.then(function(feed) {
		let info = {};
		
		info.title = feed.title ? feed.title.plainText() : '';
		info.subtitle = feed.subtitle ? feed.subtitle.plainText() : '';
		
		if (feed.updated) info.updated = new Date(feed.updated);
		
		// categories: MDN says "not yet implemented"
		
		info.creators = Zotero.FeedReader._processCreators(feed, 'authors', 'author');
		
		// TODO: image as icon
		
		let publicationTitle = Zotero.FeedReader._getFeedField(feed, 'publicationName', 'prism')
			|| Zotero.FeedReader._getFeedField(feed, 'pubTitle');
		if (publicationTitle) info.publicationTitle = publicationTitle;
		
		let publisher = Zotero.FeedReader._getFeedField(feed, 'publisher', 'dc');
		if (publisher) info.publisher = publisher;
		
		let rights = (feed.rights && feed.rights.plainText())
			|| Zotero.FeedReader._getFeedField(feed, 'copyright', 'prism')
			|| Zotero.FeedReader._getFeedField(feed, 'rights', 'dc')
			|| Zotero.FeedReader._getFeedField(feed, 'copyright');
		if (rights) info.rights = rights;
		
		let issn = Zotero.FeedReader._getFeedField(feed, 'issn', 'prism');
		if (issn) info.ISSN = issn;
		
		let isbn = Zotero.FeedReader._getFeedField(feed, 'isbn', 'prism')
			|| Zotero.FeedReader._getFeedField(feed, 'isbn')
		if (isbn) info.ISBN = isbn;
		
		let language = Zotero.FeedReader._getFeedField(feed, 'language', 'dc')
			|| Zotero.FeedReader._getFeedField(feed, 'language');
		if (language) info.language = language;
		
		let ttl = Zotero.FeedReader._getFeedField(feed, 'ttl');
		if (ttl) info.ttl = ttl;
		
		this._feedProperties = info;
		this._feed = feed;
	}.bind(this)).then(function(){
		let items = this._feed.items;
		if (items && items.length) {
			for (let i=0; i<items.length; i++) {
				let item = items.queryElementAt(i, Components.interfaces.nsIFeedEntry);
				if (!item) continue;
				
				let feedItem = Zotero.FeedReader._getFeedItem(item, this._feedProperties);
				if (!feedItem) continue;
				
				let lastItem = this._feedItems[this._feedItems.length - 1];
				this._feedItems.push(Zotero.Promise.defer()); // Push a new deferred promise so an iterator has something to return
				lastItem.resolve(feedItem);
			}
		}
		this._feedProcessed.resolve();
	}.bind(this)).catch(function(e) {
		Zotero.debug("Feed processing failed " + e.message);
		this._feedProcessed.reject(e);
	}.bind(this)).finally(function() {
		// Make sure the last promise gets resolved to null
		let lastItem = this._feedItems[this._feedItems.length - 1];
		lastItem.resolve(null);
	}.bind(this));
	
	// Set up asynchronous feed processor
	let feedProcessor = Components.classes["@mozilla.org/feed-processor;1"]
		.createInstance(Components.interfaces.nsIFeedProcessor);

	let feedUrl = Services.io.newURI(url, null, null);
	feedProcessor.parseAsync(null, feedUrl);
	
	feedProcessor.listener = {
		/*
		 * MDN suggests that we could use nsIFeedProgressListener to handle the feed
		 * as it gets loaded, but this is actually not implemented (as of 32.0.3),
		 * so we have to load the whole feed and handle it in handleResult.
		 */
		handleResult: (result) => {
			if (!result.doc) {
				this.terminate("No Feed");
				return;
			}
			
			let newFeed = result.doc.QueryInterface(Components.interfaces.nsIFeed);
			feedFetched.resolve(newFeed);
		}
	};
	
	Zotero.debug("FeedReader: Fetching feed from " + feedUrl.spec);
	
	this._channel = Services.io.newChannelFromURI2(feedUrl, null, 
		Services.scriptSecurityManager.getSystemPrincipal(), null, 
		Ci.nsILoadInfo.SEC_NORMAL, Ci.nsIContentPolicy.TYPE_OTHER);
	this._channel.asyncOpen(feedProcessor, null); // Sends an HTTP request
}

/*
 * The constructor initiates async feed processing, but _feedProcessed
 * needs to be resolved before proceeding.
 */
Zotero.FeedReader.prototype.process = Zotero.Promise.coroutine(function* () {
	return this._feedProcessed.promise;
});

/*
 * Terminate feed processing at any given time
 * @param {String} status Reason for terminating processing
 */
Zotero.FeedReader.prototype.terminate = function(status) {
	Zotero.debug("FeedReader: Terminating feed reader (" + status + ")");
	
	// Reject feed promise if not resolved yet
	if (this._feedProcessed.promise.isPending()) {
		this._feedProcessed.reject(status);
	}
	
	// Reject feed item promise if not resolved yet
	let lastItem = this._feedItems[this._feedItems.length - 1];
	if (lastItem.promise.isPending()) {
		lastItem.reject(status);
	}
	
	// Close feed connection
	if (this._channel.isPending) {
		this._channel.cancel(Components.results.NS_BINDING_ABORTED);
	}<commit_msg>Closes #951. Bypasses cache when retrieving feeds<commit_after>/*
    ***** BEGIN LICENSE BLOCK *****
    
    Copyright © 2015 Center for History and New Media
                     George Mason University, Fairfax, Virginia, USA
                     http://zotero.org
    
    This file is part of Zotero.
    
    Zotero is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.
    
    Zotero is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.
    
    You should have received a copy of the GNU Affero General Public License
    along with Zotero.  If not, see <http://www.gnu.org/licenses/>.
    
    ***** END LICENSE BLOCK *****
*/


/**
 * Sample feeds:
 * 
 * http://cyber.law.harvard.edu/rss/examples/rss2sample.xml
 * http://feeds.feedburner.com/acs/acbcct
 * http://www.cell.com/molecular-cell/current.rss
 * http://ieeexplore.ieee.org/search/searchresult.jsp?searchField%3DSearch_All%26queryText%3Dwater&searchOrigin=saved_searches&rssFeed=true&rssFeedName=water
 * http://www.sciencemag.org/rss/current.xml
 * http://rss.sciencedirect.com/publication/science/20925212
 * http://www.ncbi.nlm.nih.gov/entrez/eutils/erss.cgi?rss_guid=1fmfIeN4X5Q8HemTZD5Rj6iu6-FQVCn7xc7_IPIIQtS1XiD9bf
 * http://export.arxiv.org/rss/astro-ph
 * http://fhs.dukejournals.org/rss_feeds/recent.xml
 */

/**
 * class Zotero.FeedReader
 * Asynchronously reads an ATOM/RSS feed
 *
 * @param {String} url URL of the feed
 *
 * @property {Zotero.Promise<Object>} feedProperties An object
 *   representing feed properties
 * @property {Zotero.Promise<FeedItem>*} ItemIterator Returns an iterator
 *   for feed items. The iterator returns FeedItem promises that have to be
 *   resolved before requesting the next promise. When all items are exhausted.
 *   the promise resolves to null.
 * @method {void} terminate Stops retrieving/parsing the feed. Data parsed up
 *   to this point is still available.
 */
Zotero.FeedReader = function(url) {
	if (!url) throw new Error("Feed URL must be supplied");

	
	this._url = url;
	this._feedItems = [Zotero.Promise.defer()];
	this._feedProcessed = Zotero.Promise.defer();

	let feedFetched = Zotero.Promise.defer();
	feedFetched.promise.then(function(feed) {
		let info = {};
		
		info.title = feed.title ? feed.title.plainText() : '';
		info.subtitle = feed.subtitle ? feed.subtitle.plainText() : '';
		
		if (feed.updated) info.updated = new Date(feed.updated);
		
		// categories: MDN says "not yet implemented"
		
		info.creators = Zotero.FeedReader._processCreators(feed, 'authors', 'author');
		
		// TODO: image as icon
		
		let publicationTitle = Zotero.FeedReader._getFeedField(feed, 'publicationName', 'prism')
			|| Zotero.FeedReader._getFeedField(feed, 'pubTitle');
		if (publicationTitle) info.publicationTitle = publicationTitle;
		
		let publisher = Zotero.FeedReader._getFeedField(feed, 'publisher', 'dc');
		if (publisher) info.publisher = publisher;
		
		let rights = (feed.rights && feed.rights.plainText())
			|| Zotero.FeedReader._getFeedField(feed, 'copyright', 'prism')
			|| Zotero.FeedReader._getFeedField(feed, 'rights', 'dc')
			|| Zotero.FeedReader._getFeedField(feed, 'copyright');
		if (rights) info.rights = rights;
		
		let issn = Zotero.FeedReader._getFeedField(feed, 'issn', 'prism');
		if (issn) info.ISSN = issn;
		
		let isbn = Zotero.FeedReader._getFeedField(feed, 'isbn', 'prism')
			|| Zotero.FeedReader._getFeedField(feed, 'isbn')
		if (isbn) info.ISBN = isbn;
		
		let language = Zotero.FeedReader._getFeedField(feed, 'language', 'dc')
			|| Zotero.FeedReader._getFeedField(feed, 'language');
		if (language) info.language = language;
		
		let ttl = Zotero.FeedReader._getFeedField(feed, 'ttl');
		if (ttl) info.ttl = ttl;
		
		this._feedProperties = info;
		this._feed = feed;
	}.bind(this)).then(function(){
		let items = this._feed.items;
		if (items && items.length) {
			for (let i=0; i<items.length; i++) {
				let item = items.queryElementAt(i, Components.interfaces.nsIFeedEntry);
				if (!item) continue;
				
				let feedItem = Zotero.FeedReader._getFeedItem(item, this._feedProperties);
				if (!feedItem) continue;
				
				let lastItem = this._feedItems[this._feedItems.length - 1];
				this._feedItems.push(Zotero.Promise.defer()); // Push a new deferred promise so an iterator has something to return
				lastItem.resolve(feedItem);
			}
		}
		this._feedProcessed.resolve();
	}.bind(this)).catch(function(e) {
		Zotero.debug("Feed processing failed " + e.message);
		this._feedProcessed.reject(e);
	}.bind(this)).finally(function() {
		// Make sure the last promise gets resolved to null
		let lastItem = this._feedItems[this._feedItems.length - 1];
		lastItem.resolve(null);
	}.bind(this));
	
	// Set up asynchronous feed processor
	let feedProcessor = Components.classes["@mozilla.org/feed-processor;1"]
		.createInstance(Components.interfaces.nsIFeedProcessor);

	let feedUrl = Services.io.newURI(url, null, null);
	feedProcessor.parseAsync(null, feedUrl);
	
	feedProcessor.listener = {
		/*
		 * MDN suggests that we could use nsIFeedProgressListener to handle the feed
		 * as it gets loaded, but this is actually not implemented (as of 32.0.3),
		 * so we have to load the whole feed and handle it in handleResult.
		 */
		handleResult: (result) => {
			if (!result.doc) {
				this.terminate("No Feed");
				return;
			}
			
			let newFeed = result.doc.QueryInterface(Components.interfaces.nsIFeed);
			feedFetched.resolve(newFeed);
		}
	};
	
	Zotero.debug("FeedReader: Fetching feed from " + feedUrl.spec);
	
	this._channel = Services.io.newChannelFromURI2(feedUrl, null, 
		Services.scriptSecurityManager.getSystemPrincipal(), null, 
		Ci.nsILoadInfo.SEC_NORMAL, Ci.nsIContentPolicy.TYPE_OTHER);
	this._channel.loadFlags |= Components.interfaces.nsIRequest.LOAD_BYPASS_CACHE;
	this._channel.asyncOpen(feedProcessor, null); // Sends an HTTP request
}

/*
 * The constructor initiates async feed processing, but _feedProcessed
 * needs to be resolved before proceeding.
 */
Zotero.FeedReader.prototype.process = Zotero.Promise.coroutine(function* () {
	return this._feedProcessed.promise;
});

/*
 * Terminate feed processing at any given time
 * @param {String} status Reason for terminating processing
 */
Zotero.FeedReader.prototype.terminate = function(status) {
	Zotero.debug("FeedReader: Terminating feed reader (" + status + ")");
	
	// Reject feed promise if not resolved yet
	if (this._feedProcessed.promise.isPending()) {
		this._feedProcessed.reject(status);
	}
	
	// Reject feed item promise if not resolved yet
	let lastItem = this._feedItems[this._feedItems.length - 1];
	if (lastItem.promise.isPending()) {
		lastItem.reject(status);
	}
	
	// Close feed connection
	if (this._channel.isPending()) {
		this._channel.cancel(Components.results.NS_BINDING_ABORTED);
	}
{'fraction_non_alphanumeric': 0.1220372460496614, 'fraction_numerical': 0.005713882618510158, 'mean_word_length': 3.890307002414626, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 8, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1677405', 'n_tokens_mistral': 4909, 'n_tokens_neox': 4444, 'n_words': 1124}