<commit_before>package com.leocardz.link.preview.library;

import android.os.AsyncTask;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;

public class TextCrawler {

	public static final int ALL = -1;
	public static final int NONE = -2;

	private final String HTTP_PROTOCOL = "http://";
	private final String HTTPS_PROTOCOL = "https://";

	private LinkPreviewCallback callback;

	private AsyncTask getCodeTask;

	public TextCrawler() {
	}

	public void makePreview(LinkPreviewCallback callback, String url) {
		makePreview(callback, url, ALL);
	}

	public void makePreview(LinkPreviewCallback callback, String url,
							int imageQuantity) {
		this.callback = callback;
		cancel();
		getCodeTask = new GetCode(imageQuantity).execute(url);
	}

	public void cancel(){
		if(getCodeTask != null){
			getCodeTask.cancel(true);
		}
	}


	/** Get html code */
	public class GetCode extends AsyncTask<String, Void, Void> {

		private SourceContent sourceContent = new SourceContent();
		private int imageQuantity;
		private ArrayList<String> urls;

		public GetCode(int imageQuantity) {
			this.imageQuantity = imageQuantity;
		}

		@Override
		protected void onPreExecute() {
			if (callback != null) {
				callback.onPre();
			}
			super.onPreExecute();
		}

		@Override
		protected void onPostExecute(Void result) {
			if (callback != null) {
				callback.onPos(sourceContent, isNull());
			}
			super.onPostExecute(result);
		}

		@Override
		protected void onCancelled() {
			super.onCancelled();
		}

		@Override
		protected Void doInBackground(String... params) {
			// Don't forget the http:// or https://
			urls = SearchUrls.matches(params[0]);

			if (urls.size() > 0)
				sourceContent
						.setFinalUrl(unshortenUrl(extendedTrim(urls.get(0))));
			else
				sourceContent.setFinalUrl("");

			if (!sourceContent.getFinalUrl().equals("")) {
				if (isImage(sourceContent.getFinalUrl())
						&& !sourceContent.getFinalUrl().contains("dropbox")) {
					sourceContent.setSuccess(true);

					sourceContent.getImages().add(sourceContent.getFinalUrl());

					sourceContent.setTitle("");
					sourceContent.setDescription("");

				} else {
					try {
						Document doc = Jsoup
								.connect(sourceContent.getFinalUrl())
								.userAgent("Mozilla").get();

						sourceContent.setHtmlCode(extendedTrim(doc.toString()));

						HashMap<String, String> metaTags = getMetaTags(sourceContent
								.getHtmlCode());

						sourceContent.setMetaTags(metaTags);

						sourceContent.setTitle(metaTags.get("title"));
						sourceContent.setDescription(metaTags
								.get("description"));

						if (sourceContent.getTitle().equals("")) {
							String matchTitle = Regex.pregMatch(
									sourceContent.getHtmlCode(),
									Regex.TITLE_PATTERN, 2);

							if (!matchTitle.equals(""))
								sourceContent.setTitle(htmlDecode(matchTitle));
						}

						if (sourceContent.getDescription().equals(""))
							sourceContent
									.setDescription(crawlCode(sourceContent
											.getHtmlCode()));

						sourceContent.setDescription(sourceContent
								.getDescription().replaceAll(
										Regex.SCRIPT_PATTERN, ""));

						if (imageQuantity != NONE) {
							if (!metaTags.get("image").equals(""))
								sourceContent.getImages().add(
										metaTags.get("image"));
							else {
								sourceContent.setImages(getImages(doc,
										imageQuantity));
							}
						}

						sourceContent.setSuccess(true);
					} catch (Exception e) {
						sourceContent.setSuccess(false);
					}
				}
			}

			String[] finalLinkSet = sourceContent.getFinalUrl().split("&");
			sourceContent.setUrl(finalLinkSet[0]);

			sourceContent.setCannonicalUrl(cannonicalPage(sourceContent
					.getFinalUrl()));
			sourceContent.setDescription(stripTags(sourceContent
					.getDescription()));

			return null;
		}
<commit_msg>Catch Exceptions in text crawl background task<commit_after>package com.leocardz.link.preview.library;

import android.os.AsyncTask;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;

public class TextCrawler {

	public static final int ALL = -1;
	public static final int NONE = -2;

	private final String HTTP_PROTOCOL = "http://";
	private final String HTTPS_PROTOCOL = "https://";

	private LinkPreviewCallback callback;

	private AsyncTask getCodeTask;

	public TextCrawler() {
	}

	public void makePreview(LinkPreviewCallback callback, String url) {
		makePreview(callback, url, ALL);
	}

	public void makePreview(LinkPreviewCallback callback, String url,
							int imageQuantity) {
		this.callback = callback;
		cancel();
		getCodeTask = new GetCode(imageQuantity).execute(url);
	}

	public void cancel(){
		if(getCodeTask != null){
			getCodeTask.cancel(true);
		}
	}


	/** Get html code */
	public class GetCode extends AsyncTask<String, Void, Void> {

		private SourceContent sourceContent = new SourceContent();
		private int imageQuantity;
		private ArrayList<String> urls;

		public GetCode(int imageQuantity) {
			this.imageQuantity = imageQuantity;
		}

		@Override
		protected void onPreExecute() {
			if (callback != null) {
				callback.onPre();
			}
			super.onPreExecute();
		}

		@Override
		protected void onPostExecute(Void result) {
			if (callback != null) {
				callback.onPos(sourceContent, isNull());
			}
			super.onPostExecute(result);
		}

		@Override
		protected void onCancelled() {
			super.onCancelled();
			if(callback != null){
				callback.onPos(null, true);
			}
		}

		@Override
		protected Void doInBackground(String... params) {
			try {
				// Don't forget the http:// or https://
				urls = SearchUrls.matches(params[0]);

				if (urls.size() > 0)
					sourceContent
							.setFinalUrl(unshortenUrl(extendedTrim(urls.get(0))));
				else
					sourceContent.setFinalUrl("");

				if (!sourceContent.getFinalUrl().equals("")) {
					if (isImage(sourceContent.getFinalUrl())
							&& !sourceContent.getFinalUrl().contains("dropbox")) {
						sourceContent.setSuccess(true);

						sourceContent.getImages().add(sourceContent.getFinalUrl());

						sourceContent.setTitle("");
						sourceContent.setDescription("");

					} else {
						try {
							Document doc = Jsoup
									.connect(sourceContent.getFinalUrl())
									.userAgent("Mozilla").get();

							sourceContent.setHtmlCode(extendedTrim(doc.toString()));

							HashMap<String, String> metaTags = getMetaTags(sourceContent
									.getHtmlCode());

							sourceContent.setMetaTags(metaTags);

							sourceContent.setTitle(metaTags.get("title"));
							sourceContent.setDescription(metaTags
									.get("description"));

							if (sourceContent.getTitle().equals("")) {
								String matchTitle = Regex.pregMatch(
										sourceContent.getHtmlCode(),
										Regex.TITLE_PATTERN, 2);

								if (!matchTitle.equals(""))
									sourceContent.setTitle(htmlDecode(matchTitle));
							}

							if (sourceContent.getDescription().equals(""))
								sourceContent
										.setDescription(crawlCode(sourceContent
												.getHtmlCode()));

							sourceContent.setDescription(sourceContent
									.getDescription().replaceAll(
											Regex.SCRIPT_PATTERN, ""));

							if (imageQuantity != NONE) {
								if (!metaTags.get("image").equals(""))
									sourceContent.getImages().add(
											metaTags.get("image"));
								else {
									sourceContent.setImages(getImages(doc,
											imageQuantity));
								}
							}

							sourceContent.setSuccess(true);
						} catch (Exception e) {
							sourceContent.setSuccess(false);
						}
					}
				}

				String[] finalLinkSet = sourceContent.getFinalUrl().split("&");
				sourceContent.setUrl(finalLinkSet[0]);

				sourceContent.setCannonicalUrl(cannonicalPage(sourceContent
						.getFinalUrl()));
				sourceContent.setDescription(stripTags(sourceContent
						.getDescription()));

				return null;
			} catch (Exception e){
				return null;
			}
		}

{'fraction_non_alphanumeric': 0.12532637075718014, 'fraction_numerical': 0.001661523854735343, 'mean_word_length': 3.339340885684861, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '377211', 'n_tokens_mistral': 3336, 'n_tokens_neox': 2471, 'n_words': 474}