<commit_before>package net.acuttone.reddimg.core;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.io.OutputStream;
import java.io.Serializable;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;

import net.acuttone.reddimg.views.GalleryActivity;

import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.NameValuePair;
import org.apache.http.client.CookieStore;
import org.apache.http.client.HttpClient;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.protocol.ClientContext;
import org.apache.http.cookie.Cookie;
import org.apache.http.impl.client.BasicCookieStore;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.impl.cookie.BasicClientCookie;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.protocol.BasicHttpContext;
import org.apache.http.protocol.HttpContext;
import org.apache.http.util.EntityUtils;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;

import android.content.SharedPreferences;
import android.content.SharedPreferences.Editor;
import android.graphics.Bitmap;
import android.text.Html;
import android.util.Log;

// TODO: encrypt communication, maybe not possible yet?
public class RedditClient {

	private static final String UH_KEY = "UH";

	public final static String UPVOTE = "1";
	public final static String DOWNVOTE = "-1";
	public final static String NO_VOTE = "0";

	private HttpClient httpclient;
	private String uh;
	private HttpContext localContext;
	private File cookiesCacheDir;
	private boolean cookiesFound;

	
	public RedditClient(File cacheDir) {
		httpclient = new DefaultHttpClient();
		httpclient.getParams().setParameter("http.useragent", "reddimg");
		CookieStore cookieStore = new BasicCookieStore();
		localContext = new BasicHttpContext();
		localContext.setAttribute(ClientContext.COOKIE_STORE, cookieStore);
		cookiesFound = false;
		this.cookiesCacheDir = new File(cacheDir, "cookies");
		if (cookiesCacheDir.exists()) {
			for (File cookieFile : cookiesCacheDir.listFiles()) {
				FileInputStream fis = null;
				ObjectInputStream in = null;
				try {
					fis = new FileInputStream(cookieFile);
					in = new ObjectInputStream(fis);
					SerializableCookie sc = (SerializableCookie) in.readObject();
					Cookie sessionCookie = SerializableCookie.toBasicClientCookie(sc);
					cookieStore.addCookie(sessionCookie);
					cookiesFound = true;
					Log.d(ReddimgApp.APP_NAME, "session cookie read successfully");
				} catch (Exception ex) {
					Log.e(ReddimgApp.APP_NAME, "error loading cookie: " + ex.toString());
				} finally {
					try {
						in.close();
					} catch (IOException ex) {
						Log.e(ReddimgApp.APP_NAME, "error loading cookie: " + ex.toString());
					}
				}
			}
		} else {
			Log.d(ReddimgApp.APP_NAME, "no session cookies found");
			cookiesCacheDir.mkdir();
		}

		uh = ReddimgApp.instance().getPrefs().getString(UH_KEY, "");
		if (!"".equals(uh)) {
			Log.d(ReddimgApp.APP_NAME, "UH found");
		} else {
			Log.d(ReddimgApp.APP_NAME, "UH not found");
		}
	}

	public boolean doLogin(String username, String password) {
		boolean success = true;
		HttpPost httppost = null;
		try {
			httppost = new HttpPost("https://ssl.reddit.com/api/login/" + username.trim());
		} catch(Exception e) {
			Log.e(ReddimgApp.APP_NAME, e.toString());
			return false;
		}
		CookieStore cookieStore = (CookieStore) localContext.getAttribute(ClientContext.COOKIE_STORE);
		cookieStore.clear();
		try {
			List<NameValuePair> nameValuePairs = new ArrayList<NameValuePair>(3);
			nameValuePairs.add(new BasicNameValuePair("user", username));
			nameValuePairs.add(new BasicNameValuePair("passwd", password));
			nameValuePairs.add(new BasicNameValuePair("api_type", "json"));
			httppost.setEntity(new UrlEncodedFormEntity(nameValuePairs));

			HttpResponse response = httpclient.execute(httppost, localContext);
			HttpEntity entity = response.getEntity();
			String result = EntityUtils.toString(entity);

			JSONObject jsonObject = new JSONObject(result);
			JSONObject jo = (JSONObject) jsonObject.get("json");
			JSONArray errors = (JSONArray) jo.get("errors");
			if (errors.length() == 0) {
				JSONObject data = (JSONObject) jo.get("data");
				uh = (String) data.get("modhash");
				saveLoginInfo(cookieStore.getCookies());
			} else {
				success = false;
			}
		} catch (Exception exc) {
			Log.e(ReddimgApp.APP_NAME, "error while logging in: " + exc.toString());
			success = false;
		}

		return success;
	}
	
	public String getLinks(List<RedditLink> newLinks, String subreddits, String lastT3) throws IOException {
		boolean showNSFW = ReddimgApp.instance().getPrefs().getBoolean("show_nsfw", false);
		BufferedReader in = null;
		try {
			HttpGet request = new HttpGet();
            request.setURI(new URI("http://www.reddit.com/r/" + subreddits + "/.json?after=t3_" + lastT3));
            HttpResponse response = httpclient.execute(request, localContext);
            in = new BufferedReader(new InputStreamReader(response.getEntity().getContent()));
			String inputLine;
			StringBuilder sb = new StringBuilder();
			while ((inputLine = in.readLine()) != null)
				sb.append(inputLine);
			in.close();

			JSONObject jsonObject = new JSONObject(sb.toString());
			JSONObject data = (JSONObject) jsonObject.get("data");
			JSONArray children = (JSONArray) data.get("children");
			if(children.length() == 0) { // no more children, start over
				lastT3 = "";
			}
			for (int j = 0; j < children.length(); j++) {
				JSONObject obj = (JSONObject) children.get(j);
				JSONObject cData = (JSONObject) obj.get("data");
				lastT3 = (String) cData.get("id");
				String url = (String) cData.get("url");
				if(isUrlValid(url) == false) {
					continue;
				}
				String thumbUrl = (String) cData.get("thumbnail");
				if(isUrlValid(thumbUrl) == false) {
					continue;
				}
				boolean isNSFW = cData.getBoolean("over_18");
				if (isNSFW && showNSFW == false) {
					continue;
				}
				String commentUrl = "http://www.reddit.com" + cData.get("permalink");<commit_msg>autofix imgur urls<commit_after>package net.acuttone.reddimg.core;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.io.OutputStream;
import java.io.Serializable;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;

import net.acuttone.reddimg.views.GalleryActivity;

import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.NameValuePair;
import org.apache.http.client.CookieStore;
import org.apache.http.client.HttpClient;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.protocol.ClientContext;
import org.apache.http.cookie.Cookie;
import org.apache.http.impl.client.BasicCookieStore;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.impl.cookie.BasicClientCookie;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.protocol.BasicHttpContext;
import org.apache.http.protocol.HttpContext;
import org.apache.http.util.EntityUtils;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;

import android.content.SharedPreferences;
import android.content.SharedPreferences.Editor;
import android.graphics.Bitmap;
import android.text.Html;
import android.util.Log;

// TODO: encrypt communication, maybe not possible yet?
public class RedditClient {

	private static final String UH_KEY = "UH";

	public final static String UPVOTE = "1";
	public final static String DOWNVOTE = "-1";
	public final static String NO_VOTE = "0";

	private HttpClient httpclient;
	private String uh;
	private HttpContext localContext;
	private File cookiesCacheDir;
	private boolean cookiesFound;

	
	public RedditClient(File cacheDir) {
		httpclient = new DefaultHttpClient();
		httpclient.getParams().setParameter("http.useragent", "reddimg");
		CookieStore cookieStore = new BasicCookieStore();
		localContext = new BasicHttpContext();
		localContext.setAttribute(ClientContext.COOKIE_STORE, cookieStore);
		cookiesFound = false;
		this.cookiesCacheDir = new File(cacheDir, "cookies");
		if (cookiesCacheDir.exists()) {
			for (File cookieFile : cookiesCacheDir.listFiles()) {
				FileInputStream fis = null;
				ObjectInputStream in = null;
				try {
					fis = new FileInputStream(cookieFile);
					in = new ObjectInputStream(fis);
					SerializableCookie sc = (SerializableCookie) in.readObject();
					Cookie sessionCookie = SerializableCookie.toBasicClientCookie(sc);
					cookieStore.addCookie(sessionCookie);
					cookiesFound = true;
					Log.d(ReddimgApp.APP_NAME, "session cookie read successfully");
				} catch (Exception ex) {
					Log.e(ReddimgApp.APP_NAME, "error loading cookie: " + ex.toString());
				} finally {
					try {
						in.close();
					} catch (IOException ex) {
						Log.e(ReddimgApp.APP_NAME, "error loading cookie: " + ex.toString());
					}
				}
			}
		} else {
			Log.d(ReddimgApp.APP_NAME, "no session cookies found");
			cookiesCacheDir.mkdir();
		}

		uh = ReddimgApp.instance().getPrefs().getString(UH_KEY, "");
		if (!"".equals(uh)) {
			Log.d(ReddimgApp.APP_NAME, "UH found");
		} else {
			Log.d(ReddimgApp.APP_NAME, "UH not found");
		}
	}

	public boolean doLogin(String username, String password) {
		boolean success = true;
		HttpPost httppost = null;
		try {
			httppost = new HttpPost("https://ssl.reddit.com/api/login/" + username.trim());
		} catch(Exception e) {
			Log.e(ReddimgApp.APP_NAME, e.toString());
			return false;
		}
		CookieStore cookieStore = (CookieStore) localContext.getAttribute(ClientContext.COOKIE_STORE);
		cookieStore.clear();
		try {
			List<NameValuePair> nameValuePairs = new ArrayList<NameValuePair>(3);
			nameValuePairs.add(new BasicNameValuePair("user", username));
			nameValuePairs.add(new BasicNameValuePair("passwd", password));
			nameValuePairs.add(new BasicNameValuePair("api_type", "json"));
			httppost.setEntity(new UrlEncodedFormEntity(nameValuePairs));

			HttpResponse response = httpclient.execute(httppost, localContext);
			HttpEntity entity = response.getEntity();
			String result = EntityUtils.toString(entity);

			JSONObject jsonObject = new JSONObject(result);
			JSONObject jo = (JSONObject) jsonObject.get("json");
			JSONArray errors = (JSONArray) jo.get("errors");
			if (errors.length() == 0) {
				JSONObject data = (JSONObject) jo.get("data");
				uh = (String) data.get("modhash");
				saveLoginInfo(cookieStore.getCookies());
			} else {
				success = false;
			}
		} catch (Exception exc) {
			Log.e(ReddimgApp.APP_NAME, "error while logging in: " + exc.toString());
			success = false;
		}

		return success;
	}
	
	public String getLinks(List<RedditLink> newLinks, String subreddits, String lastT3) throws IOException {
		boolean showNSFW = ReddimgApp.instance().getPrefs().getBoolean("show_nsfw", false);
		BufferedReader in = null;
		try {
			HttpGet request = new HttpGet();
            request.setURI(new URI("http://www.reddit.com/r/" + subreddits + "/.json?after=t3_" + lastT3));
            HttpResponse response = httpclient.execute(request, localContext);
            in = new BufferedReader(new InputStreamReader(response.getEntity().getContent()));
			String inputLine;
			StringBuilder sb = new StringBuilder();
			while ((inputLine = in.readLine()) != null)
				sb.append(inputLine);
			in.close();

			JSONObject jsonObject = new JSONObject(sb.toString());
			JSONObject data = (JSONObject) jsonObject.get("data");
			JSONArray children = (JSONArray) data.get("children");
			if(children.length() == 0) { // no more children, start over
				lastT3 = "";
			}
			for (int j = 0; j < children.length(); j++) {
				JSONObject obj = (JSONObject) children.get(j);
				JSONObject cData = (JSONObject) obj.get("data");
				lastT3 = (String) cData.get("id");
				String url = (String) cData.get("url");
				if(isUrlValid(url) == false) {
					if(url.contains("imgur.com")) {
						url = url + ".jpg";
					} else {
						continue;
					}
				}
				String thumbUrl = (String) cData.get("thumbnail");
				if(isUrlValid(thumbUrl) == false) {
					continue;
				}
				boolean isNSFW = cData.getBoolean("over_18");
				if (isNSFW && showNSFW == false) {
					continue;
				}
				String commentUrl = "http://www.reddit.com" + cData.get("permalink");
{'fraction_non_alphanumeric': 0.12766276627662768, 'fraction_numerical': 0.0021002100210021, 'mean_word_length': 4.417716375457132, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7245466', 'n_tokens_mistral': 4673, 'n_tokens_neox': 4126, 'n_words': 886}