<commit_before>/*
 * Zed Attack Proxy (ZAP) and its related class files.
 * 
 * ZAP is an HTTP/HTTPS proxy for assessing web application security.
 * 
 * Licensed under the Apache License, Version 2.0 (the "License"); 
 * you may not use this file except in compliance with the License. 
 * You may obtain a copy of the License at 
 * 
 *   http://www.apache.org/licenses/LICENSE-2.0 
 *   
 * Unless required by applicable law or agreed to in writing, software 
 * distributed under the License is distributed on an "AS IS" BASIS, 
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 
 * See the License for the specific language governing permissions and 
 * limitations under the License. 
 */
package org.zaproxy.zap.extension.spiderAjax;

import java.awt.EventQueue;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.regex.Pattern;

import javax.inject.Inject;
import javax.inject.Provider;

import org.apache.commons.httpclient.URI;
import org.apache.commons.httpclient.URIException;
import org.apache.log4j.Logger;
import org.openqa.selenium.chrome.ChromeDriver;
import org.openqa.selenium.chrome.ChromeOptions;
import org.openqa.selenium.firefox.FirefoxDriver;
import org.openqa.selenium.firefox.FirefoxProfile;
import org.openqa.selenium.htmlunit.HtmlUnitDriver;
import org.parosproxy.paros.core.proxy.ProxyServer;
import org.parosproxy.paros.core.proxy.OverrideMessageProxyListener;
import org.parosproxy.paros.model.HistoryReference;
import org.parosproxy.paros.model.Session;
import org.parosproxy.paros.network.HttpHeader;
import org.parosproxy.paros.network.HttpMalformedHeaderException;
import org.parosproxy.paros.network.HttpMessage;
import org.parosproxy.paros.network.HttpResponseHeader;
import org.zaproxy.zap.extension.spiderAjax.AjaxSpiderParam.Browser;
import org.zaproxy.zap.network.HttpResponseBody;

import com.crawljax.browser.EmbeddedBrowser;
import com.crawljax.browser.WebDriverBackedEmbeddedBrowser;
import com.crawljax.core.CrawljaxRunner;
import com.crawljax.core.configuration.BrowserConfiguration;
import com.crawljax.core.configuration.CrawlRules.CrawlRulesBuilder;
import com.crawljax.core.configuration.CrawljaxConfiguration;
import com.crawljax.core.configuration.CrawljaxConfiguration.CrawljaxConfigurationBuilder;
import com.crawljax.core.configuration.ProxyConfiguration;
import com.crawljax.core.configuration.ProxyConfiguration.ProxyType;
import com.crawljax.core.plugin.Plugins;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableSortedSet;

public class SpiderThread implements Runnable {

	private static final String LOCAL_PROXY_IP = "127.0.0.1";

	// crawljax config
	private static final boolean RAND_INPUT_FORMS = true;
	private static final int MAX_DEPTH = 10;	// TODO - make this configurable by the user
	private String url = null;
	private ExtensionAjax extension = null;
	private CrawljaxRunner crawljax;
	private final boolean spiderInScope;
	private boolean running;
	private final Session session;
	private static final Logger logger = Logger.getLogger(SpiderThread.class);

	private HttpResponseHeader outOfScopeResponseHeader;
	private HttpResponseBody outOfScopeResponseBody;
	private List<SpiderListener> spiderListeners;
	private final List<String> exclusionList;
	private final String targetHost;
	private ProxyServer proxy;
	private int proxyPort;

	/**
	 * 
	 * @param url
	 * @param extension
	 * @param inScope
	 */
	SpiderThread(String url, ExtensionAjax extension, boolean inScope, SpiderListener spiderListener) throws URIException {
		this.url = url;
		this.extension = extension;
		this.spiderInScope = inScope;
		this.running = false;
		spiderListeners = new ArrayList<>(2);
		spiderListeners.add(spiderListener);
		this.session = extension.getModel().getSession();
		this.exclusionList = session.getExcludeFromSpiderRegexs();
		this.targetHost = new URI(url, true).getHost();

		createOutOfScopeResponse(extension.getMessages().getString("spiderajax.outofscope.response"));

		proxy = new ProxyServer();
		proxy.setConnectionParam(extension.getModel().getOptionsParam().getConnectionParam());
		proxy.addOverrideMessageProxyListener(new SpiderProxyListener());
	}

	private void createOutOfScopeResponse(String response) {
		outOfScopeResponseBody = new HttpResponseBody();
		outOfScopeResponseBody.setBody(response.getBytes(StandardCharsets.UTF_8));

		final StringBuilder strBuilder = new StringBuilder(150);
		final String crlf = HttpHeader.CRLF;
		strBuilder.append("HTTP/1.1 403 Forbidden").append(crlf);
		strBuilder.append(HttpHeader.PRAGMA).append(": ").append("no-cache").append(crlf);
		strBuilder.append(HttpHeader.CACHE_CONTROL).append(": ").append("no-cache").append(crlf);
		strBuilder.append(HttpHeader.CONTENT_TYPE).append(": ").append("text/plain; charset=UTF-8").append(crlf);
		strBuilder.append(HttpHeader.CONTENT_LENGTH).append(": ").append(outOfScopeResponseBody.length()).append(crlf);

		HttpResponseHeader responseHeader;
		try {
			responseHeader = new HttpResponseHeader(strBuilder.toString());
		} catch (HttpMalformedHeaderException e) {
			logger.error("Failed to create a valid! response header: ", e);
			responseHeader = new HttpResponseHeader();
		}
		outOfScopeResponseHeader = responseHeader;
	}

	/**
	 * 
	 * @return the SpiderThread object
	 */
	public SpiderThread getSpiderThread() {
		return this;
	}
	
	/**
	 * 
	 * @return the SpiderThread object
	 */
	public boolean isRunning() {
		return this.running;
	}
	
	public CrawljaxConfiguration createCrawljaxConfiguration() {
		CrawljaxConfigurationBuilder configurationBuilder = CrawljaxConfiguration.builderFor(this.url);

		configurationBuilder.setProxyConfig(ProxyConfiguration.manualProxyOn(LOCAL_PROXY_IP, proxyPort));

		configurationBuilder.setBrowserConfig(new BrowserConfiguration(
				com.crawljax.browser.EmbeddedBrowser.BrowserType.FIREFOX,
				this.extension.getAjaxSpiderParam().getNumberOfBrowsers(),
				new AjaxSpiderBrowserBuilder(extension.getAjaxSpiderParam().getBrowser())));

		if (this.extension.getAjaxSpiderParam().isCrawlInDepth()) {
			CrawlRulesBuilder crawlRulesBuilder = configurationBuilder.crawlRules();
			crawlRulesBuilder.click("a");
			crawlRulesBuilder.click("button");
			crawlRulesBuilder.click("td");
			crawlRulesBuilder.click("span");
			crawlRulesBuilder.click("div");
			crawlRulesBuilder.click("tr");
			crawlRulesBuilder.click("ol");
			crawlRulesBuilder.click("li");
			crawlRulesBuilder.click("radio");
			crawlRulesBuilder.click("non");
			crawlRulesBuilder.click("meta");
			crawlRulesBuilder.click("refresh");
			crawlRulesBuilder.click("xhr");
			crawlRulesBuilder.click("relative");
			crawlRulesBuilder.click("link");
			crawlRulesBuilder.click("self");
			crawlRulesBuilder.click("form");
			crawlRulesBuilder.click("select");
			crawlRulesBuilder.click("input");
			crawlRulesBuilder.click("option");
			crawlRulesBuilder.click("img");
			crawlRulesBuilder.click("p");
		}

		configurationBuilder.crawlRules().insertRandomDataInInputForms(RAND_INPUT_FORMS);
		// TODO - make this configurable by the user
		configurationBuilder.crawlRules().waitAfterEvent(1000, TimeUnit.MILLISECONDS);

		// TODO - make this configurable by the user
		configurationBuilder.setUnlimitedStates();
		configurationBuilder.setMaximumDepth(MAX_DEPTH);

		return configurationBuilder.build();
	}
	
	/**
	 * Instantiates the crawljax classes. 
	 */
	@Override
	public void run() {
		logger.info("Running crawljax targeting " + this.url );
		this.running = true;<commit_msg>Issue 945 - In addition to Maximum crawl depth and Event wait time, added Max Crawl states, max duration, reload wait time and 'click elements once' as options to be configurable by a user. Added more clickable elements: abbr, address, area, article,	aside, audio, canvas, details, footer, header, label, nav, section, summary, table, textarea, th, ul,video. And removed unclickable elements: non, meta, refresh, xhr, relative, link, self.<commit_after>/*
 * Zed Attack Proxy (ZAP) and its related class files.
 * 
 * ZAP is an HTTP/HTTPS proxy for assessing web application security.
 * 
 * Licensed under the Apache License, Version 2.0 (the "License"); 
 * you may not use this file except in compliance with the License. 
 * You may obtain a copy of the License at 
 * 
 *   http://www.apache.org/licenses/LICENSE-2.0 
 *   
 * Unless required by applicable law or agreed to in writing, software 
 * distributed under the License is distributed on an "AS IS" BASIS, 
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 
 * See the License for the specific language governing permissions and 
 * limitations under the License. 
 */
package org.zaproxy.zap.extension.spiderAjax;

import java.awt.EventQueue;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.regex.Pattern;

import javax.inject.Inject;
import javax.inject.Provider;

import org.apache.commons.httpclient.URI;
import org.apache.commons.httpclient.URIException;
import org.apache.log4j.Logger;
import org.openqa.selenium.chrome.ChromeDriver;
import org.openqa.selenium.chrome.ChromeOptions;
import org.openqa.selenium.firefox.FirefoxDriver;
import org.openqa.selenium.firefox.FirefoxProfile;
import org.openqa.selenium.htmlunit.HtmlUnitDriver;
import org.parosproxy.paros.core.proxy.ProxyServer;
import org.parosproxy.paros.core.proxy.OverrideMessageProxyListener;
import org.parosproxy.paros.model.HistoryReference;
import org.parosproxy.paros.model.Session;
import org.parosproxy.paros.network.HttpHeader;
import org.parosproxy.paros.network.HttpMalformedHeaderException;
import org.parosproxy.paros.network.HttpMessage;
import org.parosproxy.paros.network.HttpResponseHeader;
import org.zaproxy.zap.extension.spiderAjax.AjaxSpiderParam.Browser;
import org.zaproxy.zap.network.HttpResponseBody;

import com.crawljax.browser.EmbeddedBrowser;
import com.crawljax.browser.WebDriverBackedEmbeddedBrowser;
import com.crawljax.core.CrawljaxRunner;
import com.crawljax.core.configuration.BrowserConfiguration;
import com.crawljax.core.configuration.CrawlRules.CrawlRulesBuilder;
import com.crawljax.core.configuration.CrawljaxConfiguration;
import com.crawljax.core.configuration.CrawljaxConfiguration.CrawljaxConfigurationBuilder;
import com.crawljax.core.configuration.ProxyConfiguration;
import com.crawljax.core.configuration.ProxyConfiguration.ProxyType;
import com.crawljax.core.plugin.Plugins;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableSortedSet;

public class SpiderThread implements Runnable {

	private static final String LOCAL_PROXY_IP = "127.0.0.1";

	// crawljax config
	private static final boolean RAND_INPUT_FORMS = true;
	private String url = null;
	private ExtensionAjax extension = null;
	private CrawljaxRunner crawljax;
	private final boolean spiderInScope;
	private boolean running;
	private final Session session;
	private static final Logger logger = Logger.getLogger(SpiderThread.class);

	private HttpResponseHeader outOfScopeResponseHeader;
	private HttpResponseBody outOfScopeResponseBody;
	private List<SpiderListener> spiderListeners;
	private final List<String> exclusionList;
	private final String targetHost;
	private ProxyServer proxy;
	private int proxyPort;

	/**
	 * 
	 * @param url
	 * @param extension
	 * @param inScope
	 */
	SpiderThread(String url, ExtensionAjax extension, boolean inScope, SpiderListener spiderListener) throws URIException {
		this.url = url;
		this.extension = extension;
		this.spiderInScope = inScope;
		this.running = false;
		spiderListeners = new ArrayList<>(2);
		spiderListeners.add(spiderListener);
		this.session = extension.getModel().getSession();
		this.exclusionList = session.getExcludeFromSpiderRegexs();
		this.targetHost = new URI(url, true).getHost();

		createOutOfScopeResponse(extension.getMessages().getString("spiderajax.outofscope.response"));

		proxy = new ProxyServer();
		proxy.setConnectionParam(extension.getModel().getOptionsParam().getConnectionParam());
		proxy.addOverrideMessageProxyListener(new SpiderProxyListener());
	}

	private void createOutOfScopeResponse(String response) {
		outOfScopeResponseBody = new HttpResponseBody();
		outOfScopeResponseBody.setBody(response.getBytes(StandardCharsets.UTF_8));

		final StringBuilder strBuilder = new StringBuilder(150);
		final String crlf = HttpHeader.CRLF;
		strBuilder.append("HTTP/1.1 403 Forbidden").append(crlf);
		strBuilder.append(HttpHeader.PRAGMA).append(": ").append("no-cache").append(crlf);
		strBuilder.append(HttpHeader.CACHE_CONTROL).append(": ").append("no-cache").append(crlf);
		strBuilder.append(HttpHeader.CONTENT_TYPE).append(": ").append("text/plain; charset=UTF-8").append(crlf);
		strBuilder.append(HttpHeader.CONTENT_LENGTH).append(": ").append(outOfScopeResponseBody.length()).append(crlf);

		HttpResponseHeader responseHeader;
		try {
			responseHeader = new HttpResponseHeader(strBuilder.toString());
		} catch (HttpMalformedHeaderException e) {
			logger.error("Failed to create a valid! response header: ", e);
			responseHeader = new HttpResponseHeader();
		}
		outOfScopeResponseHeader = responseHeader;
	}

	/**
	 * 
	 * @return the SpiderThread object
	 */
	public SpiderThread getSpiderThread() {
		return this;
	}
	
	/**
	 * 
	 * @return the SpiderThread object
	 */
	public boolean isRunning() {
		return this.running;
	}
	
	public CrawljaxConfiguration createCrawljaxConfiguration() {
		CrawljaxConfigurationBuilder configurationBuilder = CrawljaxConfiguration.builderFor(this.url);

		configurationBuilder.setProxyConfig(ProxyConfiguration.manualProxyOn(LOCAL_PROXY_IP, proxyPort));

		configurationBuilder.setBrowserConfig(new BrowserConfiguration(
				com.crawljax.browser.EmbeddedBrowser.BrowserType.FIREFOX,
				this.extension.getAjaxSpiderParam().getNumberOfBrowsers(),
				new AjaxSpiderBrowserBuilder(extension.getAjaxSpiderParam().getBrowser())));

		if (this.extension.getAjaxSpiderParam().isCrawlInDepth()) {
			CrawlRulesBuilder crawlRulesBuilder = configurationBuilder.crawlRules();
			crawlRulesBuilder.click("a");
			crawlRulesBuilder.click("button");
			crawlRulesBuilder.click("td");
			crawlRulesBuilder.click("span");
			crawlRulesBuilder.click("div");
			crawlRulesBuilder.click("tr");
			crawlRulesBuilder.click("ol");
			crawlRulesBuilder.click("li");
			crawlRulesBuilder.click("radio");
			crawlRulesBuilder.click("form");
			crawlRulesBuilder.click("select");
			crawlRulesBuilder.click("input");
			crawlRulesBuilder.click("option");
			crawlRulesBuilder.click("img");
			crawlRulesBuilder.click("p");
			crawlRulesBuilder.click("abbr");
			crawlRulesBuilder.click("address");
			crawlRulesBuilder.click("area");
			crawlRulesBuilder.click("article");
			crawlRulesBuilder.click("aside");
			crawlRulesBuilder.click("audio");
			crawlRulesBuilder.click("canvas");
			crawlRulesBuilder.click("details");
			crawlRulesBuilder.click("footer");
			crawlRulesBuilder.click("header");
			crawlRulesBuilder.click("label");
			crawlRulesBuilder.click("nav");
			crawlRulesBuilder.click("section");
			crawlRulesBuilder.click("summary");
			crawlRulesBuilder.click("table");
			crawlRulesBuilder.click("textarea");
			crawlRulesBuilder.click("th");
			crawlRulesBuilder.click("ul");
			crawlRulesBuilder.click("video");
		}

		configurationBuilder.crawlRules().insertRandomDataInInputForms(RAND_INPUT_FORMS);
		
		configurationBuilder.crawlRules().waitAfterEvent(this.extension.getAjaxSpiderParam().getEventWait(),TimeUnit.MILLISECONDS);
		configurationBuilder.crawlRules().waitAfterReloadUrl(this.extension.getAjaxSpiderParam().getReloadWait(),TimeUnit.MILLISECONDS);

		if (this.extension.getAjaxSpiderParam().getMaxCrawlStates() == 0) {
			configurationBuilder.setUnlimitedStates();
		} else {
			configurationBuilder.setMaximumStates(this.extension.getAjaxSpiderParam().getMaxCrawlStates());
		}
				
		configurationBuilder.setMaximumDepth(this.extension.getAjaxSpiderParam().getMaxCrawlDepth());
		configurationBuilder.setMaximumRunTime(this.extension.getAjaxSpiderParam().getMaxDuration(),TimeUnit.MINUTES);
		configurationBuilder.crawlRules().clickOnce(this.extension.getAjaxSpiderParam().isClickOnce());
				
		return configurationBuilder.build();
	}
	
	/**
	 * Instantiates the crawljax classes. 
	 */
	@Override
	public void run() {
		logger.info("Running crawljax targeting " + this.url );
		this.running = true;
{'fraction_non_alphanumeric': 0.10757813442741644, 'fraction_numerical': 0.003258115120067576, 'mean_word_length': 5.877593360995851, 'pattern_counts': {'":': 8, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6612003', 'n_tokens_mistral': 5342, 'n_tokens_neox': 4777, 'n_words': 950}