<commit_before>package com.adsdk.sdk.nativeads;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import java.net.HttpURLConnection;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;

import org.apache.http.HttpResponse;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.params.HttpConnectionParams;
import org.apache.http.params.HttpProtocolParams;
import org.json.JSONException;
import org.json.JSONObject;

import com.adsdk.sdk.Const;
import com.adsdk.sdk.Log;
import com.adsdk.sdk.RequestException;
import com.adsdk.sdk.nativeads.NativeAd.ImageAsset;
import com.adsdk.sdk.nativeads.NativeAd.TextAsset;
import com.adsdk.sdk.nativeads.NativeAd.Tracker;

public class RequestNativeAd {

	public NativeAd sendRequest(NativeAdRequest request) throws RequestException {
		String url = request.toString();
		Log.d("Ad RequestPerform HTTP Get Url: " + url);
		DefaultHttpClient client = new DefaultHttpClient();
		HttpConnectionParams.setSoTimeout(client.getParams(), Const.SOCKET_TIMEOUT);
		HttpConnectionParams.setConnectionTimeout(client.getParams(), Const.CONNECTION_TIMEOUT);
		HttpProtocolParams.setUserAgent(client.getParams(), request.getUserAgent());
		HttpGet get = new HttpGet(url);
		get.setHeader("User-Agent", System.getProperty("http.agent"));
		HttpResponse response;
		try {
			response = client.execute(get);
			int responseCode = response.getStatusLine().getStatusCode();
			if (responseCode == HttpURLConnection.HTTP_OK) {
				return parse(response.getEntity().getContent());
			} else {
				throw new RequestException("Server Error. Response code:" + responseCode);
			}
		} catch (RequestException e) {
			throw e;
		} catch (ClientProtocolException e) {
			throw new RequestException("Error in HTTP request", e);
		} catch (IOException e) {
			throw new RequestException("Error in HTTP request", e);
		} catch (Throwable t) {
			throw new RequestException("Error in HTTP request", t);
		}
	}

	protected NativeAd parse(final InputStream inputStream) throws RequestException {

		final NativeAd response = new NativeAd();
		List<TextAsset> textAssets = new ArrayList<NativeAd.TextAsset>();
		List<ImageAsset> imageAssets = new ArrayList<NativeAd.ImageAsset>();
		List<Tracker> trackers = new ArrayList<NativeAd.Tracker>();

		// TODO: fill native ad fields
		try {
			BufferedReader reader;
			reader = new BufferedReader(new InputStreamReader(inputStream, "UTF-8"), 8);
			StringBuilder sb = new StringBuilder();

			String line = null;
			while ((line = reader.readLine()) != null) {
				sb.append(line + "\n");
			}
			String result = sb.toString();
			JSONObject mainObject = new JSONObject(result);
			JSONObject imageAssetsObject = mainObject.getJSONObject("imageassets");
			if (imageAssetsObject != null) {
				Iterator<String> keys = imageAssetsObject.keys();

				while (keys.hasNext()) {
					ImageAsset asset = new ImageAsset();
					String type = keys.next();
					asset.type = type;
					JSONObject assetObject = imageAssetsObject.getJSONObject(type);
					asset.url = assetObject.getString("url");
					asset.width = assetObject.getInt("width");
					asset.height = assetObject.getInt("height");
					response.getImageAssets().add(asset);
				}
			}

			JSONObject textAssetsObject = mainObject.getJSONObject("textassets");
			if (textAssetsObject != null) {
				Iterator<String> keys = textAssetsObject.keys();
				while (keys.hasNext()) {
					TextAsset asset = new TextAsset();
					String type = keys.next();
					asset.type = type;
					asset.text = textAssetsObject.getString(type);
					response.getTextAssets().add(asset);
				}
			}
			
			//TODO: tracking events

		} catch (UnsupportedEncodingException e) {
			throw new RequestException("Cannot parse Response", e);
		} catch (IOException e) {
			throw new RequestException("Cannot parse Response", e);
		} catch (JSONException e) {
			throw new RequestException("Cannot parse Response", e);
		}

		return response;
	}

}<commit_msg>Finish parsing native ads response.<commit_after>package com.adsdk.sdk.nativeads;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import java.net.HttpURLConnection;
import java.util.Iterator;

import org.apache.http.HttpResponse;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.params.HttpConnectionParams;
import org.apache.http.params.HttpProtocolParams;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;

import com.adsdk.sdk.Const;
import com.adsdk.sdk.Log;
import com.adsdk.sdk.RequestException;
import com.adsdk.sdk.nativeads.NativeAd.ImageAsset;
import com.adsdk.sdk.nativeads.NativeAd.TextAsset;
import com.adsdk.sdk.nativeads.NativeAd.Tracker;

public class RequestNativeAd {

	public NativeAd sendRequest(NativeAdRequest request) throws RequestException {
		String url = request.toString();
		Log.d("Ad RequestPerform HTTP Get Url: " + url);
		DefaultHttpClient client = new DefaultHttpClient();
		HttpConnectionParams.setSoTimeout(client.getParams(), Const.SOCKET_TIMEOUT);
		HttpConnectionParams.setConnectionTimeout(client.getParams(), Const.CONNECTION_TIMEOUT);
		HttpProtocolParams.setUserAgent(client.getParams(), request.getUserAgent());
		HttpGet get = new HttpGet(url);
		get.setHeader("User-Agent", System.getProperty("http.agent"));
		HttpResponse response;
		try {
			response = client.execute(get);
			int responseCode = response.getStatusLine().getStatusCode();
			if (responseCode == HttpURLConnection.HTTP_OK) {
				return parse(response.getEntity().getContent());
			} else {
				throw new RequestException("Server Error. Response code:" + responseCode);
			}
		} catch (RequestException e) {
			throw e;
		} catch (ClientProtocolException e) {
			throw new RequestException("Error in HTTP request", e);
		} catch (IOException e) {
			throw new RequestException("Error in HTTP request", e);
		} catch (Throwable t) {
			throw new RequestException("Error in HTTP request", t);
		}
	}

	protected NativeAd parse(final InputStream inputStream) throws RequestException {

		final NativeAd response = new NativeAd();

		try {
			BufferedReader reader;
			reader = new BufferedReader(new InputStreamReader(inputStream, "UTF-8"), 8);
			StringBuilder sb = new StringBuilder();

			String line = null;
			while ((line = reader.readLine()) != null) {
				sb.append(line + "\n");
			}
			String result = sb.toString();
			JSONObject mainObject = new JSONObject(result);
			JSONObject imageAssetsObject = mainObject.optJSONObject("imageassets");
			if (imageAssetsObject != null) {
				@SuppressWarnings("unchecked")
				Iterator<String> keys = imageAssetsObject.keys();

				while (keys.hasNext()) {
					ImageAsset asset = new ImageAsset();
					String type = keys.next();
					asset.type = type;
					JSONObject assetObject = imageAssetsObject.getJSONObject(type);
					asset.url = assetObject.getString("url");
					asset.width = assetObject.getInt("width");
					asset.height = assetObject.getInt("height");
					response.getImageAssets().add(asset);
				}
			}

			JSONObject textAssetsObject = mainObject.optJSONObject("textassets");
			if (textAssetsObject != null) {
				@SuppressWarnings("unchecked")
				Iterator<String> keys = textAssetsObject.keys();
				while (keys.hasNext()) {
					TextAsset asset = new TextAsset();
					String type = keys.next();
					asset.type = type;
					asset.text = textAssetsObject.getString(type);
					response.getTextAssets().add(asset);
				}
			}

			response.setClickUrl(mainObject.optString("click_url", null));

			JSONArray trackersArray = mainObject.optJSONArray("trackers");
			if (trackersArray != null) {
				for (int i = 0; i < trackersArray.length(); i++) {
					JSONObject trackerObject = trackersArray.optJSONObject(i);
					if(trackerObject != null) {
						Tracker tracker = new Tracker();
						tracker.type = trackerObject.getString("type");
						tracker.url = trackerObject.getString("url");
						response.getTrackers().add(tracker);
					}
				}
			}

		} catch (UnsupportedEncodingException e) {
			throw new RequestException("Cannot parse Response", e);
		} catch (IOException e) {
			throw new RequestException("Cannot parse Response", e);
		} catch (JSONException e) {
			throw new RequestException("Cannot parse Response", e);
		}

		return response;
	}

}
{'fraction_non_alphanumeric': 0.11920302301614566, 'fraction_numerical': 0.0005725409366769724, 'mean_word_length': 4.563057324840765, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2644344', 'n_tokens_mistral': 2889, 'n_tokens_neox': 2511, 'n_words': 575}