<commit_before>package net.lightbody.bmp.proxy;

import net.lightbody.bmp.core.har.Har;
import net.lightbody.bmp.core.har.HarContent;
import net.lightbody.bmp.core.har.HarEntry;
import net.lightbody.bmp.core.har.HarLog;
import net.lightbody.bmp.core.har.HarNameValuePair;
import net.lightbody.bmp.core.har.HarNameVersion;
import net.lightbody.bmp.core.har.HarPage;
import net.lightbody.bmp.core.har.HarPageTimings;
import net.lightbody.bmp.core.har.HarPostData;
import net.lightbody.bmp.core.har.HarRequest;
import net.lightbody.bmp.core.har.HarResponse;
import net.lightbody.bmp.core.har.HarTimings;
import net.lightbody.bmp.proxy.util.IOUtils;
import org.apache.http.HttpEntity;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.StringEntity;
import org.apache.http.message.BasicNameValuePair;
import org.junit.Assert;
import org.junit.Test;
import org.openqa.selenium.Proxy;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.firefox.FirefoxDriver;
import org.openqa.selenium.remote.CapabilityType;
import org.openqa.selenium.remote.DesiredCapabilities;

import java.io.ByteArrayOutputStream;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Random;
import java.util.zip.GZIPInputStream;

public class HarTest extends DummyServerTest {
    @Test
    public void testRequestAndResponseSizesAreSet() throws Exception {

        // see https://github.com/lightbody/browsermob-proxy/pull/36 for history

        proxy.setCaptureContent(true);
        proxy.newHar("test");

        HttpGet get = new HttpGet("http://127.0.0.1:8080/a.txt");
        client.execute(get);

        Har har = proxy.getHar();
        HarLog log = har.getLog();
        List<HarEntry> entries = log.getEntries();
        HarEntry entry = entries.get(0);

        /*
        Request headers should be something like this:

        Host: 127.0.0.1:8080
        User-Agent: bmp.lightbody.net/2.0-beta-10-SNAPSHOT
         */
        Assert.assertTrue("Minimum header size not seen", entry.getRequest().getHeadersSize() > 70);
        Assert.assertEquals(0, entry.getRequest().getBodySize());

        /*
        Response headers should be something like this:

        Date: Sun, 31 Aug 2014 16:08:44 GMT
        Server: Jetty/5.1.x (Mac OS X/10.9.4 x86_64 java/1.7.0_09
        Content-Type: text/plain
        Content-Length: 13
        Last-Modified: Sun, 17 Nov 2013 05:37:58 GMT
        Accept-Ranges: bytes
         */
        Assert.assertTrue("Minimum header size not seen", entry.getResponse().getHeadersSize() > 200);
        Assert.assertEquals(13, entry.getResponse().getBodySize());
    }
    
	@Test
	public void testHarContainsUserAgent() {
		ProxyServer server = new ProxyServer(0);
		server.start();
		
		WebDriver driver = null;
		try {
			server.setCaptureHeaders(true);
			server.newHar("testHarContainsUserAgent");
			
			Proxy proxy = server.seleniumProxy();
			DesiredCapabilities capabilities = new DesiredCapabilities();
			
			capabilities.setCapability(CapabilityType.PROXY, proxy);
			
			driver = new FirefoxDriver(capabilities);
			
			driver.get("http://www.msn.com");
			
			Har har = server.getHar();
			Assert.assertNotNull("Har is null", har);
			HarLog log = har.getLog();
			Assert.assertNotNull("Log is null", log);
			HarNameVersion harNameVersion = log.getBrowser();
			Assert.assertNotNull("HarNameVersion is null", harNameVersion);
			
			Assert.assertEquals("Expected browser to be Firefox", "Firefox", harNameVersion.getName());
			Assert.assertNotNull("browser version is null", harNameVersion.getVersion());
		} finally {
			server.stop();
			if (driver != null) {
				driver.quit();
			}
		}
	}

	@Test
	public void testThatProxyCanCaptureBodyInHar() throws IOException, InterruptedException {
		proxy.setCaptureContent(true);
		proxy.newHar("Test");

		String body = IOUtils.readFully(client.execute(new HttpGet("http://127.0.0.1:8080/a.txt")).getEntity().getContent());
		System.out.println("Done with request");

		Assert.assertTrue(body.contains("this is a.txt"));

		Har har = proxy.getHar();
		Assert.assertNotNull("Har is null", har);
		HarLog log = har.getLog();
		Assert.assertNotNull("Log is null", log);
		List<HarEntry> entries = log.getEntries();
		Assert.assertNotNull("Entries are null", entries);
		HarEntry entry = entries.get(0);
		Assert.assertNotNull("No entry found", entry);
		HarResponse response = entry.getResponse();
		Assert.assertNotNull("Response is null", response);
		HarContent content = response.getContent();
		Assert.assertNotNull("Content is null", content);
		String mime = content.getMimeType();
		Assert.assertEquals("Mime not matched", "text/plain", mime);
		String encoding = content.getEncoding();
		Assert.assertEquals("Encoding not matched", null, encoding);
		String text = content.getText();
		Assert.assertEquals("Text not matched", "this is a.txt", text);
	}

	@Test
	public void testThatProxyCanCaptureJsonRpc() throws IOException, InterruptedException {
		proxy.setCaptureContent(true);
		proxy.newHar("Test");

		HttpPost post = new HttpPost("http://127.0.0.1:8080/jsonrpc/");
		HttpEntity entity = new StringEntity("{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"test\",\"params\":{}}");
		post.setEntity(entity);
		post.addHeader("Accept", "application/json-rpc");
		post.addHeader("Content-Type", "application/json; charset=UTF-8");

		String body = IOUtils.readFully(client.execute(post).getEntity().getContent());

		Assert.assertTrue(body.contains("{\"jsonrpc\":\"2.0\",\"id\":1,\"result\":{}}"));

		Har har = proxy.getHar();
		Assert.assertNotNull("Har is null", har);
		HarLog log = har.getLog();
		Assert.assertNotNull("Log is null", log);
		List<HarEntry> entries = log.getEntries();
		Assert.assertNotNull("Entries are null", entries);
		HarEntry entry = entries.get(0);
		Assert.assertNotNull("No entry found", entry);
		HarResponse response = entry.getResponse();
		Assert.assertNotNull("Response is null", response);
		HarRequest request = entry.getRequest();
		Assert.assertNotNull("Request is null", request);
		HarPostData postdata = request.getPostData();
		Assert.assertNotNull("PostData is null", postdata);
		Assert.assertTrue(postdata.getText().contains("{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"test\",\"params\":{}}"));
	}

	@Test
	public void testThatTraditionalPostParamsAreCaptured() throws IOException, InterruptedException {
		proxy.setCaptureContent(true);
		proxy.newHar("Test");

		HttpPost post = new HttpPost("http://127.0.0.1:8080/jsonrpc/");
		post.setEntity(new UrlEncodedFormEntity(Collections.singletonList(new BasicNameValuePair("foo", "bar"))));

		IOUtils.readFully(client.execute(post).getEntity().getContent());

		Har har = proxy.getHar();
		Assert.assertNotNull("Har is null", har);
		HarLog log = har.getLog();
		Assert.assertNotNull("Log is null", log);
		List<HarEntry> entries = log.getEntries();
		Assert.assertNotNull("Entries are null", entries);
		HarEntry entry = entries.get(0);
		Assert.assertNotNull("No entry found", entry);
		HarResponse response = entry.getResponse();
		Assert.assertNotNull("Response is null", response);
		HarRequest request = entry.getRequest();
		Assert.assertNotNull("Request is null", request);
		HarPostData postdata = request.getPostData();
		Assert.assertNotNull("PostData is null", postdata);
		Assert.assertEquals("application/x-www-form-urlencoded", postdata.getMimeType());
		Assert.assertEquals(1, postdata.getParams().size());
		Assert.assertEquals("foo", postdata.getParams().get(0).getName());
		Assert.assertEquals("bar", postdata.getParams().get(0).getValue());
	}

	@Test
	public void testThatImagesAreCapturedAsBase64EncodedContent() throws IOException, InterruptedException {
		proxy.setCaptureContent(true);
		proxy.newHar("Test");

		InputStream is1 = client.execute(new HttpGet("http://127.0.0.1:8080/c.png")).getEntity().getContent();
		ByteArrayOutputStream o1 = new ByteArrayOutputStream();
		IOUtils.copy(is1, o1);
		ByteArrayOutputStream o2 = new ByteArrayOutputStream();
		IOUtils.copy(new FileInputStream("src/test/dummy-server/c.png"), o2);

		Assert.assertTrue("Image does not match file system", Arrays.equals(o1.toByteArray(), o2.toByteArray()));

		Har har = proxy.getHar();
		Assert.assertNotNull("Har is null", har);
		HarLog log = har.getLog();
		Assert.assertNotNull("Log is null", log);
		List<HarEntry> entries = log.getEntries();
		Assert.assertNotNull("Entries are null", entries);
		HarEntry entry = entries.get(0);
		Assert.assertNotNull("No entry found", entry);
		HarResponse response = entry.getResponse();
		Assert.assertNotNull("Response is null", response);
		HarContent content = response.getContent();
		Assert.assertNotNull("Content is null", content);
		String mime = content.getMimeType();
		Assert.assertEquals("Mime not matched", "image/png", mime);
		String encoding = content.getEncoding();
		Assert.assertEquals("Encoding not matched", "base64", encoding);
		String text = content.getText();
		String base64 = "<KEY>;
		Assert.assertEquals("Base64 not correct", base64, text);
	}

	@Test
	public void testThatUrlEncodedQueryStringIsParsedCorrecty() throws IOException, InterruptedException {
		proxy.setCaptureContent(true);
		proxy.newHar("Test");

		HttpGet get = new HttpGet("http://1172.16.17.32:8080/a.txt?foo=bar&a=1%262");
		client.execute(get);

		Har har = proxy.getHar();
		Assert.assertNotNull("Har is null", har);
		HarLog log = har.getLog();
		Assert.assertNotNull("Log is null", log);
		List<HarEntry> entries = log.getEntries();
		Assert.assertNotNull("Entries are null", entries);
		HarEntry entry = entries.get(0);
		Assert.assertNotNull("No entry found", entry);
		HarRequest req = entry.getRequest();
		Assert.assertNotNull("No request found", req);
		// the HAR spec is not clear on what order the parameters should show up in. intuitively, since getQueryString()
		// returns a List, the order should match the query string itself, but this is not technically required.
		boolean sawFoo = false;
		boolean sawA = false;
		for (HarNameValuePair queryStringParam : req.getQueryString()) {
			if (queryStringParam.getName().equals("foo")) {
				Assert.assertEquals("expected 'foo' query param's value to be 'bar'", "bar", queryStringParam.getValue());
				sawFoo = true;
			} else if (queryStringParam.getName().equals("a")) {
				Assert.assertEquals("expected 'a' query param's value to be '1&2'", "1&2", queryStringParam.getValue());
				sawA = true;
			} else {
				Assert.fail("Unexpected query param: " + queryStringParam.getName() + ", value: " + queryStringParam.getValue());
			}
		}

		Assert.assertTrue("did not find query param 'foo'", sawFoo);
		Assert.assertTrue("did not find query param 'a'", sawA);
	}

	@Test
	public void testThatGzippedContentIsProperlyCapturedInHar() throws IOException, InterruptedException {
		proxy.setCaptureContent(true);
		proxy.newHar("Test");

		// gzip all requests
		dummy.getHandler().setMinGzipLength(1);


		HttpGet get = new HttpGet("http://127.0.0.1:8080/a.txt");
		get.addHeader("Accept-Encoding", "gzip");
		String body = IOUtils.readFully(new GZIPInputStream(client.execute(get).getEntity().getContent()));
		System.out.println("Done with request");

		Assert.assertTrue(body.contains("this is a.txt"));

		Har har = proxy.getHar();
		Assert.assertNotNull("Har is null", har);
		HarLog log = har.getLog();
		Assert.assertNotNull("Log is null", log);
		List<HarEntry> entries = log.getEntries();
		Assert.assertNotNull("Entries are null", entries);
		HarEntry entry = entries.get(0);
		Assert.assertNotNull("No entry found", entry);
		HarResponse response = entry.getResponse();
		Assert.assertNotNull("Response is null", response);
		HarContent content = response.getContent();
		Assert.assertNotNull("Content is null", content);
		String mime = content.getMimeType();
		Assert.assertEquals("Mime not matched", "text/plain", mime);
		String text = content.getText();
		Assert.assertEquals("Text not matched", "this is a.txt", text);
	}

	@Test
	public void testHarTimingsPopulated() {
		ProxyServer server = new ProxyServer(0);
		server.start();

		WebDriver driver = null;
		try {
			server.setCaptureHeaders(true);
			server.newHar("testHarContainsUserAgent");

			Proxy proxy = server.seleniumProxy();
			DesiredCapabilities capabilities = new DesiredCapabilities();

			capabilities.setCapability(CapabilityType.PROXY, proxy);

			driver = new FirefoxDriver(capabilities);

			driver.get("http://www.msn.com");

			Har har = server.getHar();
			Assert.assertNotNull("Har is null", har);
			HarLog log = har.getLog();
			Assert.assertNotNull("Log is null", log);

			Assert.assertNotNull("No log entries", log.getEntries());
			Assert.assertFalse("No log entries", log.getEntries().isEmpty());

			HarEntry firstEntry = log.getEntries().get(0);
			HarTimings timings = firstEntry.getTimings();

			Assert.assertNotNull("No har timings", timings);
			Assert.assertNotNull("blocked timing is null", timings.getBlocked());
			Assert.assertNotNull("dns timing is null", timings.getDns());
			Assert.assertNotNull("connect timing is null", timings.getConnect());
			Assert.assertNotEquals("connect timing should be greater than 0", 0L, timings.getConnect().longValue());

			Assert.assertNotEquals("wait timing should be greater than 0", 0L, timings.getWait());
			Assert.assertNotEquals("receive timing should be greater than 0", 0L, timings.getReceive());
		} finally {
			server.stop();
			if (driver != null) {
				driver.quit();
			}
		}
	}

	@Test
	public void testSendTimingPopulated() throws IOException {
		proxy.setCaptureContent(true);
		proxy.newHar("testSendTimingPopulated");

		// using this POST dumping ground so that we get a "reasonable" send time. using the server at localhost
		// may not actually take more than 1ms. that would cause the send time to be 0ms, which would be indistinguishable
		// for testing purposes from a failed-to--populate-send-time error condition.
		// thanks to <NAME> for creating this POST testing website!
		HttpPost post = new HttpPost("http://posttestserver.com/");

		// fill the POST data with some random ascii text
		Random random = new Random();
		StringBuilder lengthyPost = new StringBuilder(30000);
		for (int i = 0; i < 30000; i++) {
			lengthyPost.append((char)(random.nextInt(94) + 32));
		}

		HttpEntity entity = new StringEntity(lengthyPost.toString());
		post.setEntity(entity);
		post.addHeader("Content-Type", "text/unknown; charset=UTF-8");

		String body = IOUtils.readFully(client.execute(post).getEntity().getContent());

		Har har = proxy.getHar();
		Assert.assertNotNull("Har is null", har);
		HarLog log = har.getLog();
		Assert.assertNotNull("Log is null", log);

		Assert.assertNotNull("No log entries", log.getEntries());
		Assert.assertFalse("No log entries", log.getEntries().isEmpty());

		HarEntry firstEntry = log.getEntries().get(0);
		HarTimings timings = firstEntry.getTimings();

		HarResponse response = firstEntry.getResponse();
		Assert.assertNotNull("Response is null", response);
		HarRequest request = firstEntry.getRequest();
		Assert.assertNotNull("Request is null", request);
		HarPostData postdata = request.getPostData();
		Assert.assertNotNull("PostData is null", postdata);

		Assert.assertNotNull("No har timings", timings);

		Assert.assertNotEquals("send timing should be greater than 0", 0L, timings.getSend());
	}

	@Test
	public void testHarPagesPopulated() throws IOException {
		proxy.newHar("testpage1");

		HttpGet get = new HttpGet("http://127.0.0.1:8080/a.txt");
		IOUtils.readFully(client.execute(get).getEntity().getContent());

		proxy.endPage();

		proxy.newPage("testpage2");

		IOUtils.readFully(client.execute(get).getEntity().getContent());
		IOUtils.readFully(client.execute(get).getEntity().getContent());

		proxy.endPage();

		Har har = proxy.getHar();
		Assert.assertNotNull("Har is null", har);
		HarLog log = har.getLog();
		Assert.assertNotNull("Log is null", log);

		List<HarEntry> entries = log.getEntries();
		Assert.assertNotNull("Entries are null", entries);

		Assert.assertNotNull("har pages are null", log.getPages());
		Assert.assertEquals("expected 2 har pages", 2, log.getPages().size());

		HarPage page1 = log.getPages().get(0);
		Assert.assertEquals("incorrect har page id", "testpage1", page1.getId());
		Assert.assertNotNull("har page timings are null", page1.getPageTimings());

		HarPageTimings timings1 = page1.getPageTimings();
		Assert.assertNotNull("har page onLoad timing is null", timings1.getOnLoad());
		Assert.assertNotEquals("har page onLoad timing should be greater than 0", timings1.getOnLoad().longValue(), 0L);

		HarPage page2 = log.getPages().get(1);
		Assert.assertEquals("incorrect har page id", "testpage2", page2.getId());
		Assert.assertNotNull("har page timings are null", page2.getPageTimings());
		HarPageTimings timings2 = page2.getPageTimings();
		Assert.assertNotNull("har page onLoad timing is null", timings2.getOnLoad());
		Assert.assertNotEquals("har page onLoad timing should be greater than 0", timings2.getOnLoad().longValue(), 0L);
	}

	@Test
	public void testEntryFieldsPopulated() throws IOException {
		proxy.newHar("testEntryTimePopulated");

		HttpGet get = new HttpGet("http://127.0.0.1:8080/a.txt");
		IOUtils.readFully(client.execute(get).getEntity().getContent());

		proxy.endPage();

		Har har = proxy.getHar();
		Assert.assertNotNull("Har is null", har);
		HarLog log = har.getLog();
		Assert.assertNotNull("Log is null", log);

		List<HarEntry> entries = log.getEntries();
		Assert.assertNotNull("Entries are null", entries);
		Assert.assertFalse("Entries are empty", entries.isEmpty());

		HarEntry entry = log.getEntries().get(0);
		Assert.assertNotEquals("entry time should be greater than 0 but was " + entry.getTime(), entry.getTime(), 0L);
		Assert.assertNotNull("entry startedDateTime is null", entry.getStartedDateTime());

		Assert.assertEquals("entry pageref is incorrect", "testEntryTimePopulated", entry.getPageref());

		//TODO this does not currently work -- it is not capturing the server IP address if it is not resolving a hostname
		//Assert.assertEquals("entry ip address is incorrect", "127.0.0.1", entry.getServerIPAddress());
	}
}<commit_msg>Added additional har tests<commit_after>package net.lightbody.bmp.proxy;

import net.lightbody.bmp.core.har.Har;
import net.lightbody.bmp.core.har.HarContent;
import net.lightbody.bmp.core.har.HarEntry;
import net.lightbody.bmp.core.har.HarLog;
import net.lightbody.bmp.core.har.HarNameValuePair;
import net.lightbody.bmp.core.har.HarNameVersion;
import net.lightbody.bmp.core.har.HarPage;
import net.lightbody.bmp.core.har.HarPageTimings;
import net.lightbody.bmp.core.har.HarPostData;
import net.lightbody.bmp.core.har.HarRequest;
import net.lightbody.bmp.core.har.HarResponse;
import net.lightbody.bmp.core.har.HarTimings;
import net.lightbody.bmp.proxy.util.IOUtils;
import org.apache.http.HttpEntity;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.StringEntity;
import org.apache.http.message.BasicNameValuePair;
import org.junit.Assert;
import org.junit.Ignore;
import org.junit.Test;
import org.openqa.selenium.Proxy;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.firefox.FirefoxDriver;
import org.openqa.selenium.remote.CapabilityType;
import org.openqa.selenium.remote.DesiredCapabilities;

import java.io.ByteArrayOutputStream;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Random;
import java.util.zip.GZIPInputStream;

public class HarTest extends DummyServerTest {
    @Test
    public void testRequestAndResponseSizesAreSet() throws Exception {

        // see https://github.com/lightbody/browsermob-proxy/pull/36 for history

        proxy.setCaptureContent(true);
        proxy.newHar("test");

        HttpGet get = new HttpGet("http://127.0.0.1:8080/a.txt");
        client.execute(get);

        Har har = proxy.getHar();
        HarLog log = har.getLog();
        List<HarEntry> entries = log.getEntries();
        HarEntry entry = entries.get(0);

        /*
        Request headers should be something like this:

        Host: 127.0.0.1:8080
        User-Agent: bmp.lightbody.net/2.0-beta-10-SNAPSHOT
         */
        Assert.assertTrue("Minimum header size not seen", entry.getRequest().getHeadersSize() > 70);
        Assert.assertEquals(0, entry.getRequest().getBodySize());

        /*
        Response headers should be something like this:

        Date: Sun, 31 Aug 2014 16:08:44 GMT
        Server: Jetty/5.1.x (Mac OS X/10.9.4 x86_64 java/1.7.0_09
        Content-Type: text/plain
        Content-Length: 13
        Last-Modified: Sun, 17 Nov 2013 05:37:58 GMT
        Accept-Ranges: bytes
         */
        Assert.assertTrue("Minimum header size not seen", entry.getResponse().getHeadersSize() > 200);
        Assert.assertEquals(13, entry.getResponse().getBodySize());
    }
    
	@Test
	public void testHarContainsUserAgent() {
		ProxyServer server = new ProxyServer(0);
		server.start();
		
		WebDriver driver = null;
		try {
			server.setCaptureHeaders(true);
			server.newHar("testHarContainsUserAgent");
			
			Proxy proxy = server.seleniumProxy();
			DesiredCapabilities capabilities = new DesiredCapabilities();
			
			capabilities.setCapability(CapabilityType.PROXY, proxy);
			
			driver = new FirefoxDriver(capabilities);
			
			driver.get("http://www.msn.com");
			
			Har har = server.getHar();
			Assert.assertNotNull("Har is null", har);
			HarLog log = har.getLog();
			Assert.assertNotNull("Log is null", log);
			HarNameVersion harNameVersion = log.getBrowser();
			Assert.assertNotNull("HarNameVersion is null", harNameVersion);
			
			Assert.assertEquals("Expected browser to be Firefox", "Firefox", harNameVersion.getName());
			Assert.assertNotNull("browser version is null", harNameVersion.getVersion());
		} finally {
			server.stop();
			if (driver != null) {
				driver.quit();
			}
		}
	}

	@Test
	public void testThatProxyCanCaptureBodyInHar() throws IOException, InterruptedException {
		proxy.setCaptureContent(true);
		proxy.newHar("Test");

		String body = IOUtils.readFully(client.execute(new HttpGet("http://127.0.0.1:8080/a.txt")).getEntity().getContent());
		System.out.println("Done with request");

		Assert.assertTrue(body.contains("this is a.txt"));

		Har har = proxy.getHar();
		Assert.assertNotNull("Har is null", har);
		HarLog log = har.getLog();
		Assert.assertNotNull("Log is null", log);
		List<HarEntry> entries = log.getEntries();
		Assert.assertNotNull("Entries are null", entries);
		HarEntry entry = entries.get(0);
		Assert.assertNotNull("No entry found", entry);
		HarResponse response = entry.getResponse();
		Assert.assertNotNull("Response is null", response);
		HarContent content = response.getContent();
		Assert.assertNotNull("Content is null", content);
		String mime = content.getMimeType();
		Assert.assertEquals("Mime not matched", "text/plain", mime);
		String encoding = content.getEncoding();
		Assert.assertEquals("Encoding not matched", null, encoding);
		String text = content.getText();
		Assert.assertEquals("Text not matched", "this is a.txt", text);
	}

	@Test
	public void testThatProxyCanCaptureJsonRpc() throws IOException, InterruptedException {
		proxy.setCaptureContent(true);
		proxy.newHar("Test");

		HttpPost post = new HttpPost("http://127.0.0.1:8080/jsonrpc/");
		HttpEntity entity = new StringEntity("{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"test\",\"params\":{}}");
		post.setEntity(entity);
		post.addHeader("Accept", "application/json-rpc");
		post.addHeader("Content-Type", "application/json; charset=UTF-8");

		String body = IOUtils.readFully(client.execute(post).getEntity().getContent());

		Assert.assertTrue(body.contains("{\"jsonrpc\":\"2.0\",\"id\":1,\"result\":{}}"));

		Har har = proxy.getHar();
		Assert.assertNotNull("Har is null", har);
		HarLog log = har.getLog();
		Assert.assertNotNull("Log is null", log);
		List<HarEntry> entries = log.getEntries();
		Assert.assertNotNull("Entries are null", entries);
		HarEntry entry = entries.get(0);
		Assert.assertNotNull("No entry found", entry);
		HarResponse response = entry.getResponse();
		Assert.assertNotNull("Response is null", response);
		HarRequest request = entry.getRequest();
		Assert.assertNotNull("Request is null", request);
		HarPostData postdata = request.getPostData();
		Assert.assertNotNull("PostData is null", postdata);
		Assert.assertTrue(postdata.getText().contains("{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"test\",\"params\":{}}"));
	}

	@Test
	public void testThatTraditionalPostParamsAreCaptured() throws IOException, InterruptedException {
		proxy.setCaptureContent(true);
		proxy.newHar("Test");

		HttpPost post = new HttpPost("http://127.0.0.1:8080/jsonrpc/");
		post.setEntity(new UrlEncodedFormEntity(Collections.singletonList(new BasicNameValuePair("foo", "bar"))));

		IOUtils.readFully(client.execute(post).getEntity().getContent());

		Har har = proxy.getHar();
		Assert.assertNotNull("Har is null", har);
		HarLog log = har.getLog();
		Assert.assertNotNull("Log is null", log);
		List<HarEntry> entries = log.getEntries();
		Assert.assertNotNull("Entries are null", entries);
		HarEntry entry = entries.get(0);
		Assert.assertNotNull("No entry found", entry);
		HarResponse response = entry.getResponse();
		Assert.assertNotNull("Response is null", response);
		HarRequest request = entry.getRequest();
		Assert.assertNotNull("Request is null", request);
		HarPostData postdata = request.getPostData();
		Assert.assertNotNull("PostData is null", postdata);
		Assert.assertEquals("application/x-www-form-urlencoded", postdata.getMimeType());
		Assert.assertEquals(1, postdata.getParams().size());
		Assert.assertEquals("foo", postdata.getParams().get(0).getName());
		Assert.assertEquals("bar", postdata.getParams().get(0).getValue());
	}

	@Test
	public void testThatImagesAreCapturedAsBase64EncodedContent() throws IOException, InterruptedException {
		proxy.setCaptureContent(true);
		proxy.newHar("Test");

		InputStream is1 = client.execute(new HttpGet("http://127.0.0.1:8080/c.png")).getEntity().getContent();
		ByteArrayOutputStream o1 = new ByteArrayOutputStream();
		IOUtils.copy(is1, o1);
		ByteArrayOutputStream o2 = new ByteArrayOutputStream();
		IOUtils.copy(new FileInputStream("src/test/dummy-server/c.png"), o2);

		Assert.assertTrue("Image does not match file system", Arrays.equals(o1.toByteArray(), o2.toByteArray()));

		Har har = proxy.getHar();
		Assert.assertNotNull("Har is null", har);
		HarLog log = har.getLog();
		Assert.assertNotNull("Log is null", log);
		List<HarEntry> entries = log.getEntries();
		Assert.assertNotNull("Entries are null", entries);
		HarEntry entry = entries.get(0);
		Assert.assertNotNull("No entry found", entry);
		HarResponse response = entry.getResponse();
		Assert.assertNotNull("Response is null", response);
		HarContent content = response.getContent();
		Assert.assertNotNull("Content is null", content);
		String mime = content.getMimeType();
		Assert.assertEquals("Mime not matched", "image/png", mime);
		String encoding = content.getEncoding();
		Assert.assertEquals("Encoding not matched", "base64", encoding);
		String text = content.getText();
		String base64 = "<KEY>;
		Assert.assertEquals("Base64 not correct", base64, text);
	}

	@Test
	public void testThatUrlEncodedQueryStringIsParsedCorrecty() throws IOException, InterruptedException {
		proxy.setCaptureContent(true);
		proxy.newHar("Test");

		HttpGet get = new HttpGet("http://127.0.0.1:8080/a.txt?foo=bar&a=1%262");
		client.execute(get);

		Har har = proxy.getHar();
		Assert.assertNotNull("Har is null", har);
		HarLog log = har.getLog();
		Assert.assertNotNull("Log is null", log);
		List<HarEntry> entries = log.getEntries();
		Assert.assertNotNull("Entries are null", entries);
		HarEntry entry = entries.get(0);
		Assert.assertNotNull("No entry found", entry);
		HarRequest req = entry.getRequest();
		Assert.assertNotNull("No request found", req);
		// the HAR spec is not clear on what order the parameters should show up in. intuitively, since getQueryString()
		// returns a List, the order should match the query string itself, but this is not technically required.
		boolean sawFoo = false;
		boolean sawA = false;
		for (HarNameValuePair queryStringParam : req.getQueryString()) {
			if (queryStringParam.getName().equals("foo")) {
				Assert.assertEquals("expected 'foo' query param's value to be 'bar'", "bar", queryStringParam.getValue());
				sawFoo = true;
			} else if (queryStringParam.getName().equals("a")) {
				Assert.assertEquals("expected 'a' query param's value to be '1&2'", "1&2", queryStringParam.getValue());
				sawA = true;
			} else {
				Assert.fail("Unexpected query param: " + queryStringParam.getName() + ", value: " + queryStringParam.getValue());
			}
		}

		Assert.assertTrue("did not find query param 'foo'", sawFoo);
		Assert.assertTrue("did not find query param 'a'", sawA);
	}

	@Test
	public void testThatGzippedContentIsProperlyCapturedInHar() throws IOException, InterruptedException {
		proxy.setCaptureContent(true);
		proxy.newHar("Test");

		// gzip all requests
		dummy.getHandler().setMinGzipLength(1);


		HttpGet get = new HttpGet("http://127.0.0.1:8080/a.txt");
		get.addHeader("Accept-Encoding", "gzip");
		String body = IOUtils.readFully(new GZIPInputStream(client.execute(get).getEntity().getContent()));
		System.out.println("Done with request");

		Assert.assertTrue(body.contains("this is a.txt"));

		Har har = proxy.getHar();
		Assert.assertNotNull("Har is null", har);
		HarLog log = har.getLog();
		Assert.assertNotNull("Log is null", log);
		List<HarEntry> entries = log.getEntries();
		Assert.assertNotNull("Entries are null", entries);
		HarEntry entry = entries.get(0);
		Assert.assertNotNull("No entry found", entry);
		HarResponse response = entry.getResponse();
		Assert.assertNotNull("Response is null", response);
		HarContent content = response.getContent();
		Assert.assertNotNull("Content is null", content);
		String mime = content.getMimeType();
		Assert.assertEquals("Mime not matched", "text/plain", mime);
		String text = content.getText();
		Assert.assertEquals("Text not matched", "this is a.txt", text);
	}

	@Test
	public void testHarTimingsPopulated() {
		ProxyServer server = new ProxyServer(0);
		server.start();

		WebDriver driver = null;
		try {
			server.setCaptureHeaders(true);
			server.newHar("testHarContainsUserAgent");

			Proxy proxy = server.seleniumProxy();
			DesiredCapabilities capabilities = new DesiredCapabilities();

			capabilities.setCapability(CapabilityType.PROXY, proxy);

			driver = new FirefoxDriver(capabilities);

			driver.get("http://www.msn.com");

			Har har = server.getHar();
			Assert.assertNotNull("Har is null", har);
			HarLog log = har.getLog();
			Assert.assertNotNull("Log is null", log);

			Assert.assertNotNull("No log entries", log.getEntries());
			Assert.assertFalse("No log entries", log.getEntries().isEmpty());

			HarEntry firstEntry = log.getEntries().get(0);
			HarTimings timings = firstEntry.getTimings();

			Assert.assertNotNull("No har timings", timings);
			Assert.assertNotNull("blocked timing is null", timings.getBlocked());
			Assert.assertNotNull("dns timing is null", timings.getDns());
			Assert.assertNotNull("connect timing is null", timings.getConnect());
			Assert.assertNotEquals("connect timing should be greater than 0", 0L, timings.getConnect().longValue());

			// we can't guarantee that wait timing will be greater than 0
			//Assert.assertNotEquals("wait timing should be greater than 0", 0L, timings.getWait());

			Assert.assertNotEquals("receive timing should be greater than 0", 0L, timings.getReceive());
		} finally {
			server.stop();
			if (driver != null) {
				driver.quit();
			}
		}
	}

	@Test
	public void testSendTimingPopulated() throws IOException {
		proxy.setCaptureContent(true);
		proxy.newHar("testSendTimingPopulated");

		// using this POST dumping ground so that we get a "reasonable" send time. using the server at localhost
		// may not actually take more than 1ms. that would cause the send time to be 0ms, which would be indistinguishable
		// for testing purposes from a failed-to--populate-send-time error condition.
		// thanks to <NAME> for creating this POST testing website!
		HttpPost post = new HttpPost("http://posttestserver.com/");

		// fill the POST data with some random ascii text
		Random random = new Random();
		StringBuilder lengthyPost = new StringBuilder(30000);
		for (int i = 0; i < 30000; i++) {
			lengthyPost.append((char)(random.nextInt(94) + 32));
		}

		HttpEntity entity = new StringEntity(lengthyPost.toString());
		post.setEntity(entity);
		post.addHeader("Content-Type", "text/unknown; charset=UTF-8");

		String body = IOUtils.readFully(client.execute(post).getEntity().getContent());

		Har har = proxy.getHar();
		Assert.assertNotNull("Har is null", har);
		HarLog log = har.getLog();
		Assert.assertNotNull("Log is null", log);

		Assert.assertNotNull("No log entries", log.getEntries());
		Assert.assertFalse("No log entries", log.getEntries().isEmpty());

		HarEntry firstEntry = log.getEntries().get(0);
		HarTimings timings = firstEntry.getTimings();

		HarResponse response = firstEntry.getResponse();
		Assert.assertNotNull("Response is null", response);
		HarRequest request = firstEntry.getRequest();
		Assert.assertNotNull("Request is null", request);
		HarPostData postdata = request.getPostData();
		Assert.assertNotNull("PostData is null", postdata);

		Assert.assertNotNull("No har timings", timings);

		Assert.assertNotEquals("send timing should be greater than 0", 0L, timings.getSend());
	}

	@Test
	public void testHarPagesPopulated() throws IOException {
		proxy.newHar("testpage1");

		HttpGet get = new HttpGet("http://1172.16.17.32:8080/a.txt");
		IOUtils.readFully(client.execute(get).getEntity().getContent());

		proxy.endPage();

		proxy.newPage("testpage2");

		IOUtils.readFully(client.execute(get).getEntity().getContent());
		IOUtils.readFully(client.execute(get).getEntity().getContent());

		proxy.endPage();

		Har har = proxy.getHar();
		Assert.assertNotNull("Har is null", har);
		HarLog log = har.getLog();
		Assert.assertNotNull("Log is null", log);

		List<HarEntry> entries = log.getEntries();
		Assert.assertNotNull("Entries are null", entries);

		Assert.assertNotNull("har pages are null", log.getPages());
		Assert.assertEquals("expected 2 har pages", 2, log.getPages().size());

		HarPage page1 = log.getPages().get(0);
		Assert.assertEquals("incorrect har page id", "testpage1", page1.getId());
		Assert.assertNotNull("har page timings are null", page1.getPageTimings());

		HarPageTimings timings1 = page1.getPageTimings();
		Assert.assertNotNull("har page onLoad timing is null", timings1.getOnLoad());
		Assert.assertNotEquals("har page onLoad timing should be greater than 0", timings1.getOnLoad().longValue(), 0L);

		HarPage page2 = log.getPages().get(1);
		Assert.assertEquals("incorrect har page id", "testpage2", page2.getId());
		Assert.assertNotNull("har page timings are null", page2.getPageTimings());
		HarPageTimings timings2 = page2.getPageTimings();
		Assert.assertNotNull("har page onLoad timing is null", timings2.getOnLoad());
		Assert.assertNotEquals("har page onLoad timing should be greater than 0", timings2.getOnLoad().longValue(), 0L);
	}

	@Test
	public void testEntryFieldsPopulated() throws IOException {
		proxy.newHar("testEntryTimePopulated");

		// not using localhost so we get >0ms timing
		HttpGet get = new HttpGet("http://www.msn.com");
		IOUtils.readFully(client.execute(get).getEntity().getContent());

		proxy.endPage();

		Har har = proxy.getHar();
		Assert.assertNotNull("Har is null", har);
		HarLog log = har.getLog();
		Assert.assertNotNull("Log is null", log);

		List<HarEntry> entries = log.getEntries();
		Assert.assertNotNull("Entries are null", entries);
		Assert.assertFalse("Entries are empty", entries.isEmpty());

		HarEntry entry = log.getEntries().get(0);
		Assert.assertNotEquals("entry time should be greater than 0 but was " + entry.getTime(), entry.getTime(), 0L);
		Assert.assertNotNull("entry startedDateTime is null", entry.getStartedDateTime());

		Assert.assertEquals("entry pageref is incorrect", "testEntryTimePopulated", entry.getPageref());

		Assert.assertNotNull("entry ip address is not populated", entry.getServerIPAddress());
	}

	@Test
	@Ignore
	public void testIpAddressPopulatedForLocalhost() throws IOException {
		proxy.newHar("testIpAddressPopulated");

		HttpGet get = new HttpGet("http://localhost:8080/a.txt");
		IOUtils.readFully(client.execute(get).getEntity().getContent());

		proxy.endPage();

		Har har = proxy.getHar();
		Assert.assertNotNull("Har is null", har);
		HarLog log = har.getLog();
		Assert.assertNotNull("Log is null", log);

		List<HarEntry> entries = log.getEntries();
		Assert.assertNotNull("Entries are null", entries);
		Assert.assertFalse("Entries are empty", entries.isEmpty());

		HarEntry entry = log.getEntries().get(0);
		Assert.assertNotNull("entry startedDateTime is null", entry.getStartedDateTime());

		Assert.assertEquals("entry pageref is incorrect", "testIpAddressPopulated", entry.getPageref());

		//TODO: this does not currently work when resolving localhost
		Assert.assertEquals("entry ip address is not correct", "127.0.0.1", entry.getServerIPAddress());
	}

	@Test
	@Ignore
	public void testIpAddressPopulatedForIpAddressUrl() throws IOException {
		proxy.newHar("testIpAddressPopulatedForIpAddressUrl");

		HttpGet get = new HttpGet("http://127.0.0.1:8080/a.txt");
		IOUtils.readFully(client.execute(get).getEntity().getContent());

		proxy.endPage();

		Har har = proxy.getHar();
		Assert.assertNotNull("Har is null", har);
		HarLog log = har.getLog();
		Assert.assertNotNull("Log is null", log);

		List<HarEntry> entries = log.getEntries();
		Assert.assertNotNull("Entries are null", entries);
		Assert.assertFalse("Entries are empty", entries.isEmpty());

		HarEntry entry = log.getEntries().get(0);
		Assert.assertNotNull("entry startedDateTime is null", entry.getStartedDateTime());

		Assert.assertEquals("entry pageref is incorrect", "testIpAddressPopulatedForIpAddressUrl", entry.getPageref());

		//TODO: this does not currently work when resolving 127.0.0.1
		Assert.assertEquals("entry ip address is not correct", "127.0.0.1", entry.getServerIPAddress());
	}
}
{'fraction_non_alphanumeric': 0.14247851371519737, 'fraction_numerical': 0.01446142761566569, 'mean_word_length': 4.79266656729766, 'pattern_counts': {'":': 22, '<': 29, '<?xml version=': 0, '>': 32, 'https://': 2, 'lorem ipsum': 0, 'www.': 7, 'xml': 0}, 'pii_count': 24, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1784669', 'n_tokens_mistral': 12828, 'n_tokens_neox': 11594, 'n_words': 2739}