<commit_before>package com.noinnion.android.newsplus.extension.newsblurplus;

import java.io.IOException;
import java.sql.Time;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;

import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;

import android.content.Context;
import android.os.RemoteException;
import android.text.TextUtils;
import android.text.method.DateTimeKeyListener;

import com.androidquery.AQuery;
import com.androidquery.callback.AjaxCallback;
import com.androidquery.callback.AjaxStatus;
import com.androidquery.util.AQUtility;
import com.noinnion.android.reader.api.ReaderException;
import com.noinnion.android.reader.api.ReaderExtension;
import com.noinnion.android.reader.api.internal.IItemIdListHandler;
import com.noinnion.android.reader.api.internal.IItemListHandler;
import com.noinnion.android.reader.api.internal.ISubscriptionListHandler;
import com.noinnion.android.reader.api.internal.ITagListHandler;
import com.noinnion.android.reader.api.provider.IItem;
import com.noinnion.android.reader.api.provider.ISubscription;
import com.noinnion.android.reader.api.provider.ITag;

public class NewsBlurPlus extends ReaderExtension {
	@Override
	public void onCreate() {
		super.onCreate();	
		tags = new ArrayList<ITag>();
		feeds = new ArrayList<ISubscription>();	
		// Create the special starred tag
		starredTag = new ITag();
		starredTag.uid = "STAR:Starred items";
		starredTag.label = "Starred items";
		starredTag.type = ITag.TYPE_TAG_STARRED;
	}
	
	private List<ITag> tags;
	private List<ISubscription> feeds;
	private ITag starredTag;
	
	/*
	 * Get the categories (folders) and their feeds + handle reader list
	 * 
	 * API call: http://www.newsblur.com/reader/feeds
	 * Result: folders/0/Math/[ID] (ID = 1818)
	 */
	@Override
	public void handleReaderList(final ITagListHandler tagHandler, final ISubscriptionListHandler subHandler, long syncTime) throws IOException, ReaderException {
		AjaxCallback<JSONObject> cb = new AjaxCallback<JSONObject>() {
			@Override
			public void callback(String url, JSONObject json, AjaxStatus status) {
					if (APICalls.isJSONResponseValid(json, status)) {
						try {
							JSONObject json_feeds = json.getJSONObject("feeds");
							JSONObject json_folders = json.getJSONObject("flat_folders");
								
							Iterator<?> keys = json_folders.keys();
							while (keys.hasNext()) {
								String catName = ((String)keys.next());
								JSONArray feedsPerFolder = json_folders.getJSONArray(catName);
								catName = catName.trim();
								if (!TextUtils.isEmpty(catName)) {
									// Create the category
									ITag tag = new ITag();
									tag.label = catName;
									tag.uid = catName = ("FOL:" + catName);
									tag.type = ITag.TYPE_FOLDER;
									tags.add(tag);
								}
								// Add all feeds in this category
								for (int i=0; i<feedsPerFolder.length(); i++) {
									ISubscription sub = new ISubscription();
									String feedID = feedsPerFolder.getString(i);
									JSONObject f = json_feeds.getJSONObject(feedID);
										
									sub.uid = "FEED:" + APICalls.getFeedUrlFromFeedId(feedID);
									sub.title = f.getString("feed_title");
									sub.htmlUrl = f.getString("feed_link");
									sub.unreadCount = f.getInt("nt");
									if (!TextUtils.isEmpty(catName))
										sub.addCategory(catName);
									feeds.add(sub);
								}
							}
							if (feeds.size() > 0) {
								tags.add(starredTag);
								tagHandler.tags(tags);
								subHandler.subscriptions(feeds);
							}
						}
						catch (JSONException e) {
							AQUtility.report(e);
						}
						catch (RemoteException e) {
							AQUtility.report(e);			
						}	
					}
				}
			};
		final AQuery aq = new AQuery(this);
		final Context c = getApplicationContext();
		APICalls.wrapCallback(c, cb);
		aq.ajax(APICalls.API_URL_FOLDERS_AND_FEEDS, JSONObject.class, cb);
	}
	
	/*
	 * Handle a single item list (a feed or a folder).
	 * This functions calls the parseItemList function.
	 */
	@Override
	public void handleItemList(final IItemListHandler handler, long syncTime) throws IOException, ReaderException {
		try {
			String uid = handler.stream(); 
			if (uid.equals(ReaderExtension.STATE_READING_LIST)) {
				for (ISubscription sub : feeds) {
					if (sub.unreadCount > 0) {
						String url = sub.uid.replace("FEED:", "");
						parseItemList(url, handler, sub.uid);
					}
				}
			}
			else if (uid.startsWith("FOL:")) {
				for (ISubscription sub : feeds) {
					if (sub.getCategories().contains(uid)) {
						if (sub.unreadCount > 0) {
							String url = sub.uid.replace("FEED:", "");
							parseItemList(url, handler, sub.uid);
						}
					}
				}
			}
			else if (uid.startsWith("FEED:")) {
				String url = handler.stream().replace("FEED:", "");<commit_msg>Parse when the feed was last fetched by NewsBlur<commit_after>package com.noinnion.android.newsplus.extension.newsblurplus;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;

import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;

import android.content.Context;
import android.os.RemoteException;
import android.text.TextUtils;

import com.androidquery.AQuery;
import com.androidquery.callback.AjaxCallback;
import com.androidquery.callback.AjaxStatus;
import com.androidquery.util.AQUtility;
import com.noinnion.android.reader.api.ReaderException;
import com.noinnion.android.reader.api.ReaderExtension;
import com.noinnion.android.reader.api.internal.IItemIdListHandler;
import com.noinnion.android.reader.api.internal.IItemListHandler;
import com.noinnion.android.reader.api.internal.ISubscriptionListHandler;
import com.noinnion.android.reader.api.internal.ITagListHandler;
import com.noinnion.android.reader.api.provider.IItem;
import com.noinnion.android.reader.api.provider.ISubscription;
import com.noinnion.android.reader.api.provider.ITag;

public class NewsBlurPlus extends ReaderExtension {
	@Override
	public void onCreate() {
		super.onCreate();	
		tags = new ArrayList<ITag>();
		feeds = new ArrayList<ISubscription>();	
		// Create the special starred tag
		starredTag = new ITag();
		starredTag.uid = "STAR:Starred items";
		starredTag.label = "Starred items";
		starredTag.type = ITag.TYPE_TAG_STARRED;
	}
	
	private List<ITag> tags;
	private List<ISubscription> feeds;
	private ITag starredTag;
	
	/*
	 * Get the categories (folders) and their feeds + handle reader list
	 * 
	 * API call: http://www.newsblur.com/reader/feeds
	 * Result: folders/0/Math/[ID] (ID = 1818)
	 */
	@Override
	public void handleReaderList(final ITagListHandler tagHandler, final ISubscriptionListHandler subHandler, long syncTime) throws IOException, ReaderException {
		AjaxCallback<JSONObject> cb = new AjaxCallback<JSONObject>() {
			@Override
			public void callback(String url, JSONObject json, AjaxStatus status) {
					if (APICalls.isJSONResponseValid(json, status)) {
						try {
							JSONObject json_feeds = json.getJSONObject("feeds");
							JSONObject json_folders = json.getJSONObject("flat_folders");
							Iterator<?> keys = json_folders.keys();
							while (keys.hasNext()) {
								String catName = ((String)keys.next());
								JSONArray feedsPerFolder = json_folders.getJSONArray(catName);
								catName = catName.trim();
								if (!TextUtils.isEmpty(catName)) {
									// Create the category
									ITag tag = new ITag();
									tag.label = catName;
									tag.uid = catName = ("FOL:" + catName);
									tag.type = ITag.TYPE_FOLDER;
									tags.add(tag);
								}
								// Add all feeds in this category
								for (int i=0; i<feedsPerFolder.length(); i++) {
									ISubscription sub = new ISubscription();
									String feedID = feedsPerFolder.getString(i);
									JSONObject f = json_feeds.getJSONObject(feedID);
									Calendar updateTime = Calendar.getInstance();
									updateTime.add(Calendar.SECOND, f.getInt("updated_seconds_ago"));
									sub.newestItemTime = updateTime.getTimeInMillis();
									sub.uid = "FEED:" + APICalls.getFeedUrlFromFeedId(feedID);
									sub.title = f.getString("feed_title");
									sub.htmlUrl = f.getString("feed_link");
									sub.unreadCount = f.getInt("nt");
									if (!TextUtils.isEmpty(catName))
										sub.addCategory(catName);
									feeds.add(sub);
								}
							}
							if (feeds.size() > 0) {
								tags.add(starredTag);
								tagHandler.tags(tags);
								subHandler.subscriptions(feeds);
							}
						}
						catch (JSONException e) {
							AQUtility.report(e);
						}
						catch (RemoteException e) {
							AQUtility.report(e);			
						}	
					}
				}
			};
		final AQuery aq = new AQuery(this);
		final Context c = getApplicationContext();
		APICalls.wrapCallback(c, cb);
		aq.ajax(APICalls.API_URL_FOLDERS_AND_FEEDS, JSONObject.class, cb);
		cb.block();
	}
	
	/*
	 * Handle a single item list (a feed or a folder).
	 * This functions calls the parseItemList function.
	 */
	@Override
	public void handleItemList(final IItemListHandler handler, long syncTime) throws IOException, ReaderException {
		try {
			String uid = handler.stream(); 
			if (uid.equals(ReaderExtension.STATE_READING_LIST)) {
				for (ISubscription sub : feeds) {
					if (sub.unreadCount > 0) {
						String url = sub.uid.replace("FEED:", "");
						parseItemList(url, handler, sub.uid);
					}
				}
			}
			else if (uid.startsWith("FOL:")) {
				for (ISubscription sub : feeds) {
					if (sub.getCategories().contains(uid)) {
						if (sub.unreadCount > 0) {
							String url = sub.uid.replace("FEED:", "");
							parseItemList(url, handler, sub.uid);
						}
					}
				}
			}
			else if (uid.startsWith("FEED:")) {
				String url = handler.stream().replace("FEED:", "");
{'fraction_non_alphanumeric': 0.12089462018940157, 'fraction_numerical': 0.0018134193028410237, 'mean_word_length': 3.3123370981754996, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2503824', 'n_tokens_mistral': 4005, 'n_tokens_neox': 3100, 'n_words': 642}