<commit_before>/*
 * 
 * ZAP is an HTTP/HTTPS proxy for assessing web application security.
 * 
 * Copyright 2010 <EMAIL>
 * 
 * Licensed under the Apache License, Version 2.0 (the "License"); 
 * you may not use this file except in compliance with the License. 
 * You may obtain a copy of the License at 
 * 
 *   http://www.apache.org/licenses/LICENSE-2.0 
 *   
 * Unless required by applicable law or agreed to in writing, software 
 * distributed under the License is distributed on an "AS IS" BASIS, 
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 
 * See the License for the specific language governing permissions and 
 * limitations under the License. 
 */
package org.zaproxy.zap.extension.spider;

import java.awt.EventQueue;
import java.util.Collections;
import java.util.Date;
import java.util.Enumeration;
import java.util.LinkedList;
import java.util.List;

import javax.swing.DefaultListModel;

import org.apache.commons.httpclient.URI;
import org.apache.log4j.Logger;
import org.parosproxy.paros.model.HistoryReference;
import org.parosproxy.paros.model.Model;
import org.parosproxy.paros.model.SiteNode;
import org.parosproxy.paros.network.HttpMessage;
import org.parosproxy.paros.view.View;
import org.zaproxy.zap.model.Context;
import org.zaproxy.zap.model.ScanListenner;
import org.zaproxy.zap.model.ScanThread;
import org.zaproxy.zap.model.SessionStructure;
import org.zaproxy.zap.model.TechSet;
import org.zaproxy.zap.spider.Spider;
import org.zaproxy.zap.spider.SpiderListener;
import org.zaproxy.zap.spider.SpiderParam;
import org.zaproxy.zap.spider.filters.FetchFilter;
import org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus;
import org.zaproxy.zap.spider.filters.ParseFilter;
import org.zaproxy.zap.spider.parser.SpiderParser;
import org.zaproxy.zap.users.User;

/**
 * The Class SpiderThread that controls the spidering process on a particular site. Being a
 * ScanThread, it also handles the update of the graphical UI and any other "extension-level"
 * required actions.
 */
public class SpiderThread extends ScanThread implements SpiderListener {

	/** Whether the scanning process has stopped (either completed, either by user request). */
	private boolean stopScan = false;

	/** Whether the scanning process is paused. */
	private boolean isPaused = false;

	/** Whether the scanning process is running. */
	private boolean isAlive = false;

	/** The related extension. */
	private ExtensionSpider extension;

	/** The spider. */
	private Spider spider = null;

	/** The pending spider listeners which will be added to the Spider as soon at is initialized. */
	private List<SpiderListener> pendingSpiderListeners;

	/** The spider done. */
	private int spiderDone = 0;

	/** The spider todo. It will be updated by the "spiderProgress()" method. */
	private int spiderTodo = 1;

	/** The Constant log used for logging. */
	private static final Logger log = Logger.getLogger(SpiderThread.class);

	/** The just scan in scope. */
	private boolean justScanInScope = false;

	/** The scan children. */
	private boolean scanChildren = false;

	/** The scan context. */
	private Context scanContext = null;

	/** The scan user. */
	private User scanUser = null;

	/** The results model. */
	private SpiderPanelTableModel resultsModel;

	/** The start uri. */
	private URI startURI = null;
	
	private SpiderParam spiderParams;
	
	private List<SpiderParser> customSpiderParsers = null;

	private List<FetchFilter> customFetchFilters = null;

	private List<ParseFilter> customParseFilters = null;

	private final String id;

	/**
	 * Constructs a {@code SpiderThread} with the given data.
	 * 
	 * @param extension the extension to obtain configurations and notify the view
	 * @param spiderParams the spider options
	 * @param site the name that identifies the target site
	 * @param listenner the scan listener
	 * @deprecated (TODO add version) Use {@link #SpiderThread(String, ExtensionSpider, SpiderParam, String, ScanListenner)}
	 */
	@Deprecated
	public SpiderThread(ExtensionSpider extension, SpiderParam spiderParams, String site, ScanListenner listenner) {
		this("?", extension, spiderParams, site, listenner);
	}

	/**
	 * Constructs a {@code SpiderThread} with the given data.
	 * 
	 * @param id the ID of the spider, usually a unique integer
	 * @param extension the extension to obtain configurations and notify the view
	 * @param spiderParams the spider options
	 * @param site the name that identifies the target site
	 * @param listenner the scan listener
	 * @since TODO add version
	 */
	public SpiderThread(String id, ExtensionSpider extension, SpiderParam spiderParams, String site, ScanListenner listenner) {
		super(site, listenner);
		log.debug("Initializing spider thread for site: " + site);
		this.id = id;
		this.extension = extension;
		this.site = site;
		this.pendingSpiderListeners = new LinkedList<>();
		this.resultsModel = extension.getView() != null ? new SpiderPanelTableModel() : null;
		this.spiderParams = spiderParams;

		setName("ZAP-SpiderInitThread-"+ id);
	}

	@Override
	public void run() {
		runScan();
	}

	/**
	 * Runs the scan.
	 */
	private void runScan() {
		// Do the scan
		spiderDone = 0;
		Date start = new Date();
		log.info("Starting spidering scan on " + site + " at " + start);<commit_msg>Stop the spider scan if failed to properly start<commit_after>/*
 * 
 * ZAP is an HTTP/HTTPS proxy for assessing web application security.
 * 
 * Copyright 2010 <EMAIL>
 * 
 * Licensed under the Apache License, Version 2.0 (the "License"); 
 * you may not use this file except in compliance with the License. 
 * You may obtain a copy of the License at 
 * 
 *   http://www.apache.org/licenses/LICENSE-2.0 
 *   
 * Unless required by applicable law or agreed to in writing, software 
 * distributed under the License is distributed on an "AS IS" BASIS, 
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 
 * See the License for the specific language governing permissions and 
 * limitations under the License. 
 */
package org.zaproxy.zap.extension.spider;

import java.awt.EventQueue;
import java.util.Collections;
import java.util.Date;
import java.util.Enumeration;
import java.util.LinkedList;
import java.util.List;

import javax.swing.DefaultListModel;

import org.apache.commons.httpclient.URI;
import org.apache.log4j.Logger;
import org.parosproxy.paros.model.HistoryReference;
import org.parosproxy.paros.model.Model;
import org.parosproxy.paros.model.SiteNode;
import org.parosproxy.paros.network.HttpMessage;
import org.parosproxy.paros.view.View;
import org.zaproxy.zap.model.Context;
import org.zaproxy.zap.model.ScanListenner;
import org.zaproxy.zap.model.ScanThread;
import org.zaproxy.zap.model.SessionStructure;
import org.zaproxy.zap.model.TechSet;
import org.zaproxy.zap.spider.Spider;
import org.zaproxy.zap.spider.SpiderListener;
import org.zaproxy.zap.spider.SpiderParam;
import org.zaproxy.zap.spider.filters.FetchFilter;
import org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus;
import org.zaproxy.zap.spider.filters.ParseFilter;
import org.zaproxy.zap.spider.parser.SpiderParser;
import org.zaproxy.zap.users.User;

/**
 * The Class SpiderThread that controls the spidering process on a particular site. Being a
 * ScanThread, it also handles the update of the graphical UI and any other "extension-level"
 * required actions.
 */
public class SpiderThread extends ScanThread implements SpiderListener {

	/** Whether the scanning process has stopped (either completed, either by user request). */
	private boolean stopScan = false;

	/** Whether the scanning process is paused. */
	private boolean isPaused = false;

	/** Whether the scanning process is running. */
	private boolean isAlive = false;

	/** The related extension. */
	private ExtensionSpider extension;

	/** The spider. */
	private Spider spider = null;

	/** The pending spider listeners which will be added to the Spider as soon at is initialized. */
	private List<SpiderListener> pendingSpiderListeners;

	/** The spider done. */
	private int spiderDone = 0;

	/** The spider todo. It will be updated by the "spiderProgress()" method. */
	private int spiderTodo = 1;

	/** The Constant log used for logging. */
	private static final Logger log = Logger.getLogger(SpiderThread.class);

	/** The just scan in scope. */
	private boolean justScanInScope = false;

	/** The scan children. */
	private boolean scanChildren = false;

	/** The scan context. */
	private Context scanContext = null;

	/** The scan user. */
	private User scanUser = null;

	/** The results model. */
	private SpiderPanelTableModel resultsModel;

	/** The start uri. */
	private URI startURI = null;
	
	private SpiderParam spiderParams;
	
	private List<SpiderParser> customSpiderParsers = null;

	private List<FetchFilter> customFetchFilters = null;

	private List<ParseFilter> customParseFilters = null;

	private final String id;

	/**
	 * Constructs a {@code SpiderThread} with the given data.
	 * 
	 * @param extension the extension to obtain configurations and notify the view
	 * @param spiderParams the spider options
	 * @param site the name that identifies the target site
	 * @param listenner the scan listener
	 * @deprecated (TODO add version) Use {@link #SpiderThread(String, ExtensionSpider, SpiderParam, String, ScanListenner)}
	 */
	@Deprecated
	public SpiderThread(ExtensionSpider extension, SpiderParam spiderParams, String site, ScanListenner listenner) {
		this("?", extension, spiderParams, site, listenner);
	}

	/**
	 * Constructs a {@code SpiderThread} with the given data.
	 * 
	 * @param id the ID of the spider, usually a unique integer
	 * @param extension the extension to obtain configurations and notify the view
	 * @param spiderParams the spider options
	 * @param site the name that identifies the target site
	 * @param listenner the scan listener
	 * @since TODO add version
	 */
	public SpiderThread(String id, ExtensionSpider extension, SpiderParam spiderParams, String site, ScanListenner listenner) {
		super(site, listenner);
		log.debug("Initializing spider thread for site: " + site);
		this.id = id;
		this.extension = extension;
		this.site = site;
		this.pendingSpiderListeners = new LinkedList<>();
		this.resultsModel = extension.getView() != null ? new SpiderPanelTableModel() : null;
		this.spiderParams = spiderParams;

		setName("ZAP-SpiderInitThread-"+ id);
	}

	@Override
	public void run() {
		try {
			runScan();
		} catch (Exception e) {
			log.error("An error occurred while starting the spider:", e);
			stopScan();
		}
	}

	/**
	 * Runs the scan.
	 */
	private void runScan() {
		// Do the scan
		spiderDone = 0;
		Date start = new Date();
		log.info("Starting spidering scan on " + site + " at " + start);
{'fraction_non_alphanumeric': 0.09658563740953795, 'fraction_numerical': 0.002226758211170904, 'mean_word_length': 4.152485659655832, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4260191', 'n_tokens_mistral': 3340, 'n_tokens_neox': 3063, 'n_words': 1080}