<commit_before>/*
 * Zed Attack Proxy (ZAP) and its related class files.
 * 
 * ZAP is an HTTP/HTTPS proxy for assessing web application security.
 * 
 * Copyright 2011 The Zed Attack Proxy Project
 * 
 * Licensed under the Apache License, Version 2.0 (the "License"); 
 * you may not use this file except in compliance with the License. 
 * You may obtain a copy of the License at 
 * 
 *   http://www.apache.org/licenses/LICENSE-2.0 
 *   
 * Unless required by applicable law or agreed to in writing, software 
 * distributed under the License is distributed on an "AS IS" BASIS, 
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 
 * See the License for the specific language governing permissions and 
 * limitations under the License. 
 */
package org.zaproxy.zap.extension.autoupdate;
import java.awt.EventQueue;
import java.awt.Toolkit;
import java.awt.Window;
import java.awt.event.ActionEvent;
import java.awt.event.ActionListener;
import java.awt.event.KeyEvent;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.StringReader;
import java.lang.reflect.InvocationTargetException;
import java.net.MalformedURLException;
import java.net.URL;
import java.nio.file.FileAlreadyExistsException;
import java.nio.file.Path;
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import java.util.Vector;

import javax.net.ssl.SSLHandshakeException;
import javax.swing.ImageIcon;
import javax.swing.JButton;
import javax.swing.JCheckBox;
import javax.swing.JFileChooser;
import javax.swing.JOptionPane;
import javax.swing.KeyStroke;
import javax.swing.SwingWorker;
import javax.swing.filechooser.FileFilter;

import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.XMLPropertiesConfiguration;
import org.apache.commons.httpclient.URI;
import org.apache.commons.io.FileUtils;
import org.apache.log4j.Logger;
import org.parosproxy.paros.CommandLine;
import org.parosproxy.paros.Constant;
import org.parosproxy.paros.control.Control;
import org.parosproxy.paros.extension.CommandLineArgument;
import org.parosproxy.paros.extension.CommandLineListener;
import org.parosproxy.paros.extension.Extension;
import org.parosproxy.paros.extension.ExtensionAdaptor;
import org.parosproxy.paros.extension.ExtensionHook;
import org.parosproxy.paros.model.FileCopier;
import org.parosproxy.paros.model.Model;
import org.parosproxy.paros.network.HttpMessage;
import org.parosproxy.paros.network.HttpSender;
import org.parosproxy.paros.network.HttpStatusCode;
import org.parosproxy.paros.view.View;
import org.zaproxy.zap.ZAP;
import org.zaproxy.zap.control.AddOn;
import org.zaproxy.zap.control.AddOn.AddOnRunRequirements;
import org.zaproxy.zap.control.AddOnCollection;
import org.zaproxy.zap.control.AddOnCollection.Platform;
import org.zaproxy.zap.control.AddOnRunIssuesUtils;
import org.zaproxy.zap.control.AddOnUninstallationProgressCallback;
import org.zaproxy.zap.control.ExtensionFactory;
import org.zaproxy.zap.control.ZapRelease;
import org.zaproxy.zap.extension.autoupdate.AddOnDependencyChecker.AddOnChangesResult;
import org.zaproxy.zap.extension.autoupdate.AddOnDependencyChecker.UninstallationResult;
import org.zaproxy.zap.extension.autoupdate.UninstallationProgressDialogue.AddOnUninstallListener;
import org.zaproxy.zap.extension.autoupdate.UninstallationProgressDialogue.UninstallationProgressEvent;
import org.zaproxy.zap.extension.autoupdate.UninstallationProgressDialogue.UninstallationProgressHandler;
import org.zaproxy.zap.extension.log4j.ExtensionLog4j;
import org.zaproxy.zap.utils.ZapXmlConfiguration;
import org.zaproxy.zap.view.ScanStatus;
import org.zaproxy.zap.view.ZapMenuItem;

public class ExtensionAutoUpdate extends ExtensionAdaptor implements CheckForUpdateCallback, CommandLineListener {
	
	private static final String NAME = "ExtensionAutoUpdate";
	
	// The short URL means that the number of checkForUpdates can be tracked - see https://bitly.com/u/psiinon
	// Note that URLs must now use https (unless you change the code;)
    
    private static final String ZAP_VERSIONS_REL_XML_SHORT = "https://bit.ly/owaspzap-2-6-0";
    private static final String ZAP_VERSIONS_REL_XML_FULL = "https://raw.githubusercontent.com/zaproxy/zap-admin/master/ZapVersions-2.6.xml";
    
    private static final String ZAP_VERSIONS_DEV_XML_SHORT = "https://bit.ly/owaspzap-dev";
    private static final String ZAP_VERSIONS_DEV_XML_FULL = "https://raw.githubusercontent.com/zaproxy/zap-admin/master/ZapVersions-dev.xml";
    private static final String ZAP_VERSIONS_WEEKLY_XML_SHORT = "https://bit.ly/owaspzap-devw";

	// URLs for use when testing locally ;)
	//private static final String ZAP_VERSIONS_XML_SHORT = "https://localhost:8080/zapcfu/ZapVersions.xml";
	//private static final String ZAP_VERSIONS_XML_FULL = "https://localhost:8080/zapcfu/ZapVersions.xml";

	private static final String VERSION_FILE_NAME = "ZapVersions.xml";

	private ZapMenuItem menuItemCheckUpdate = null;
	private ZapMenuItem menuItemLoadAddOn = null;
    
    private static final Logger logger = Logger.getLogger(ExtensionAutoUpdate.class);
    
	private HttpSender httpSender = null;

    private DownloadManager downloadManager = null;
	private ManageAddOnsDialog addonsDialog = null;
	//private UpdateDialog updateDialog = null;
	private Thread downloadProgressThread = null;
	private Thread remoteCallThread = null; 
	private ScanStatus scanStatus = null;
	private JButton addonsButton = null;
    private JButton outOfDateButton = null;

	private AddOnCollection latestVersionInfo = null;
	private AddOnCollection localVersionInfo = null;
	private AddOnCollection previousVersionInfo = null;

    private AutoUpdateAPI api = null;

    private boolean oldZapAlertAdded = false;
    private boolean noCfuAlertAdded = false;

    // Files currently being downloaded
	private List<Downloader> downloadFiles = new ArrayList<>();

    private static final int ARG_CFU_INSTALL_IDX = 0;
    private static final int ARG_CFU_INSTALL_ALL_IDX = 1;
    private static final int ARG_CFU_UNINSTALL_IDX = 2;
    private static final int ARG_CFU_UPDATE_IDX = 3;
    private static final int ARG_CFU_LIST_IDX = 4;
    private static final int[] ARG_IDXS = {
    							ARG_CFU_INSTALL_IDX,
    							ARG_CFU_INSTALL_ALL_IDX,
    							ARG_CFU_UNINSTALL_IDX,
    							ARG_CFU_UPDATE_IDX,
    							ARG_CFU_LIST_IDX};
	private CommandLineArgument[] arguments = new CommandLineArgument[ARG_IDXS.length];

    public ExtensionAutoUpdate() {
        super();
 		initialize();
   }   

	/**
	 * This method initializes this
	 */
	private void initialize() {
        this.setName(NAME);
        this.setOrder(1);	// High order so that cmdline updates are installed asap
        this.downloadManager = new DownloadManager(Model.getSingleton().getOptionsParam().getConnectionParam());
        this.downloadManager.start();
        // Do this before it can get overwritten by the latest one
        this.getPreviousVersionInfo();
	}

	@Override
	public String getUIName() {
		return Constant.messages.getString("autoupdate.name");
	}
	
	@Override
	public void postInit() {
		switch (ZAP.getProcessType()) {
		case cmdline:
		case daemon:
		case zaas:
			this.warnIfOutOfDate();
			break;
		case desktop:
		default:
			break;
		}
	}

	/**
	 * This method initializes menuItemEncoder	
	 * 	
	 * @return javax.swing.JMenuItem	
	 */    
	private ZapMenuItem getMenuItemCheckUpdate() {
		if (menuItemCheckUpdate == null) {
			menuItemCheckUpdate = new ZapMenuItem("cfu.help.menu.check", 
					KeyStroke.getKeyStroke(KeyEvent.VK_U, Toolkit.getDefaultToolkit().getMenuShortcutKeyMask(), false));
			menuItemCheckUpdate.setText(Constant.messages.getString("cfu.help.menu.check"));
			menuItemCheckUpdate.addActionListener(new java.awt.event.ActionListener() { 
				@Override
				public void actionPerformed(java.awt.event.ActionEvent e) {    
					getAddOnsDialog().setVisible(true);
					getAddOnsDialog().checkForUpdates();
				}

			});
		}
		return menuItemCheckUpdate;
	}

	private ZapMenuItem getMenuItemLoadAddOn() {
		if (menuItemLoadAddOn == null) {
			menuItemLoadAddOn = new ZapMenuItem("cfu.file.menu.loadaddon", 
					KeyStroke.getKeyStroke(KeyEvent.VK_L, Toolkit.getDefaultToolkit().getMenuShortcutKeyMask(), false));
			menuItemLoadAddOn.addActionListener(new java.awt.event.ActionListener() { 
				@Override
				public void actionPerformed(java.awt.event.ActionEvent e) {
					try {
						JFileChooser chooser = new JFileChooser(Model.getSingleton().getOptionsParam().getUserDirectory());
						File file = null;
						chooser.setFileFilter(new FileFilter() {
						       @Override
						       public boolean accept(File file) {
						            if (file.isDirectory()) {
						                return true;
						            } else if (file.isFile() && AddOn.isAddOnFileName(file.getName())) {
						                return true;
						            }
						            return false;
						        }
						       @Override
						       public String getDescription() {
						           return Constant.messages.getString("file.format.zap.addon");
						       }
						});
						int rc = chooser.showOpenDialog(View.getSingleton().getMainFrame());
						if(rc == JFileChooser.APPROVE_OPTION) {
							file = chooser.getSelectedFile();
							if (file == null) {
								return;
							}
							installLocalAddOn(file.toPath());
						}
					} catch (Exception e1) {
						logger.error(e1.getMessage(), e1);
					}
				}
			});
		}
		return menuItemLoadAddOn;
	}
	
	private void installLocalAddOn(Path file) throws Exception {
		if (!AddOn.isAddOn(file)) {
			showWarningMessageInvalidAddOnFile();
			return;
		}

		AddOn ao;
		try {
			ao = new AddOn(file);
		} catch (Exception e) {
			showWarningMessageInvalidAddOnFile();
			return;
		}

		if (!ao.canLoadInCurrentVersion()) {
			showWarningMessageCantLoadAddOn(ao);
			return;
		}

		AddOnDependencyChecker dependencyChecker = new AddOnDependencyChecker(getLocalVersionInfo(), latestVersionInfo == null
				? getLocalVersionInfo()
				: latestVersionInfo);

		boolean update = false;
		AddOnChangesResult result;
		AddOn installedAddOn = getLocalVersionInfo().getAddOn(ao.getId());
		if (installedAddOn != null) {
			if (!ao.isUpdateTo(installedAddOn)) {
				View.getSingleton().showWarningDialog(
						MessageFormat.format(
								Constant.messages.getString("cfu.warn.addOnOlderVersion"),
								installedAddOn.getVersion(),
								View.getSingleton().getStatusUI(installedAddOn.getStatus()).toString(),
								ao.getVersion(),
								View.getSingleton().getStatusUI(ao.getStatus()).toString()));
				return;
			}

			result = dependencyChecker.calculateUpdateChanges(ao);
			update = true;
		} else {
			result = dependencyChecker.calculateInstallChanges(ao);
		}

		if (result.getOldVersions().isEmpty() && result.getUninstalls().isEmpty()) {
			AddOnRunRequirements reqs = ao.calculateRunRequirements(getLocalVersionInfo().getAddOns());
			if (!reqs.isRunnable()) {
				if (!AddOnRunIssuesUtils.askConfirmationAddOnNotRunnable(
						Constant.messages.getString("cfu.warn.addOnNotRunnable.message"),
						Constant.messages.getString("cfu.warn.addOnNotRunnable.question"),
						getLocalVersionInfo(),
						ao)) {
					return;
				}
			}
			
			installLocalAddOn(ao);
			return;
		}

		if (update) {
			if (!dependencyChecker.confirmUpdateChanges(getView().getMainFrame(), result)) {
				return;
			}
			// The new version of the add-on is installed manually
			result.getNewVersions().remove(ao);
		} else {
			if (!dependencyChecker.confirmInstallChanges(getView().getMainFrame(), result)) {
				return;
			}
			// The add-on is installed manually
			result.getInstalls().remove(ao);
		}
		
		processAddOnChanges(getView().getMainFrame(), result);
		installLocalAddOn(ao);
	}

	private void installLocalAddOn(AddOn ao) {
		File addOnFile;
		try {
			addOnFile = copyAddOnFileToLocalPluginFolder(ao);
		} catch (FileAlreadyExistsException e) {
			showWarningMessageAddOnFileAlreadyExists(e.getFile(), e.getOtherFile());
			logger.warn("Unable to copy add-on, a file with the same name already exists.", e);
			return;
		} catch (IOException e) {
			showWarningMessageUnableToCopyAddOnFile();
			logger.warn("Unable to copy add-on to local plugin folder.", e);
			return;
		}

		ao.setFile(addOnFile);

		install(ao);
	}

	private void showWarningMessageInvalidAddOnFile() {
		View.getSingleton().showWarningDialog(Constant.messages.getString("cfu.warn.invalidAddOn"));
	}

	private void showWarningMessageCantLoadAddOn(AddOn ao) {
		String message = MessageFormat.format(
				Constant.messages.getString("cfu.warn.cantload"),
				ao.getNotBeforeVersion(),
				ao.getNotFromVersion());
		View.getSingleton().showWarningDialog(message);
	}

	private static File copyAddOnFileToLocalPluginFolder(AddOn addOn) throws IOException {
		if (isFileInLocalPluginFolder(addOn.getFile())) {
			return addOn.getFile();
		}

		File targetFile = new File(Constant.FOLDER_LOCAL_PLUGIN, addOn.getNormalisedFileName());
		if (targetFile.exists()) {
			throw new FileAlreadyExistsException(addOn.getFile().getAbsolutePath(), targetFile.getAbsolutePath(), "");
		}

		FileCopier fileCopier = new FileCopier();
		fileCopier.copy(addOn.getFile(), targetFile);

		return targetFile;
	}

	private static boolean isFileInLocalPluginFolder(File file) {
		File fileLocalPluginFolder = new File(Constant.FOLDER_LOCAL_PLUGIN, file.getName());
		if (fileLocalPluginFolder.getAbsolutePath().equals(file.getAbsolutePath())) {
			return true;
		}
		return false;
	}

	private static void showWarningMessageAddOnFileAlreadyExists(String file, String targetFile) {
		String message = MessageFormat.format(Constant.messages.getString("cfu.warn.addOnAlreadExists"), file, targetFile);
		View.getSingleton().showWarningDialog(message);
	}

	private static void showWarningMessageUnableToCopyAddOnFile() {
		String pathPluginFolder = new File(Constant.FOLDER_LOCAL_PLUGIN).getAbsolutePath();
		String message = MessageFormat.format(Constant.messages.getString("cfu.warn.unableToCopyAddOn"), pathPluginFolder);
		View.getSingleton().showWarningDialog(message);
	}
	

	private synchronized ManageAddOnsDialog getAddOnsDialog() {
		if (addonsDialog == null) {
			addonsDialog = new ManageAddOnsDialog(this, this.getCurrentVersion(), getLocalVersionInfo());
			if (this.previousVersionInfo != null) {
				addonsDialog.setPreviousVersionInfo(this.previousVersionInfo);
			}
			if (this.latestVersionInfo != null) {
				addonsDialog.setLatestVersionInfo(this.latestVersionInfo);
			}
		}
		return addonsDialog;
	}
	
	private void downloadFile (URL url, File targetFile, long size, String hash) {
		if (View.isInitialised()) {
			// Report info to the Output tab
			View.getSingleton().getOutputPanel().append(
					MessageFormat.format(
							Constant.messages.getString("cfu.output.downloading") + "\n", 
							url.toString(),
							targetFile.getAbsolutePath()));
		}
		this.downloadFiles.add(this.downloadManager.downloadFile(url, targetFile, size, hash));
		if (View.isInitialised()) {
			// Means we do have a UI
			if (this.downloadProgressThread != null && ! this.downloadProgressThread.isAlive()) {
				this.downloadProgressThread = null;
			}
			if (this.downloadProgressThread == null) {
				this.downloadProgressThread = new Thread() {
					@Override
					public void run() {
						while (downloadManager.getCurrentDownloadCount() > 0) {
							getScanStatus().setScanCount(downloadManager.getCurrentDownloadCount());
							if (addonsDialog != null && addonsDialog.isVisible()) {
								addonsDialog.showProgress();
							}
							try {
								sleep(100);
							} catch (InterruptedException e) {
								// Ignore
							}
						}
						// Complete download progress
						if (addonsDialog != null) {
							addonsDialog.showProgress();
						}
						getScanStatus().setScanCount(0);
						installNewExtensions();
					}
				};
				this.downloadProgressThread.start();
			}
		}
	}
	
	public void installNewExtensions() {
    	final OptionsParamCheckForUpdates options = getModel().getOptionsParam().getCheckForUpdatesParam();
		List<Downloader> handledFiles = new ArrayList<>();
		
		for (Downloader dl : downloadFiles) {
			if (dl.getFinished() == null) {
				continue;
			}
			handledFiles.add(dl);
			try {
				if (!dl.isValidated()) {
					logger.debug("Ignoring unvalidated download: " + dl.getUrl());
					if (addonsDialog != null) {
						addonsDialog.notifyAddOnDownloadFailed(dl.getUrl().toString());
					} else {
						String url = dl.getUrl().toString();
						for (AddOn addOn : latestVersionInfo.getAddOns()) {
							if (url.equals(addOn.getUrl().toString())) {
								addOn.setInstallationStatus(AddOn.InstallationStatus.AVAILABLE);
								break;
							}
						}
					}
				} else if (AddOn.isAddOn(dl.getTargetFile().toPath())) {
					File f = dl.getTargetFile();
					if (! options.getDownloadDirectory().equals(dl.getTargetFile().getParentFile())) {
						// Move the file to the specified directory - we do this after its been downloaded
						// as these directories can be shared, and other ZAP instances could get incomplete
						// add-ons
						try {
							f = new File(options.getDownloadDirectory(), dl.getTargetFile().getName());
							logger.info("Moving downloaded add-on from " + dl.getTargetFile().getAbsolutePath() +
									" to " + f.getAbsolutePath());
							FileUtils.moveFile(dl.getTargetFile(), f);
						} catch (Exception e) {
							if (!f.exists() && dl.getTargetFile().exists()) {
								logger.error("Failed to move downloaded add-on from " + dl.getTargetFile().getAbsolutePath() +
										" to " + f.getAbsolutePath() + " - left at original location", e);
								f = dl.getTargetFile();
							} else {
								logger.error("Failed to move downloaded add-on from " + dl.getTargetFile().getAbsolutePath() +
										" to " + f.getAbsolutePath() + " - skipping", e);
								continue;
							}
						}
					}
					
					AddOn ao = new AddOn(f.toPath());
					if (ao.canLoadInCurrentVersion()) {
						install(ao);
					} else {
			    		logger.info("Cant load add-on " + ao.getName() + 
			    				" Not before=" + ao.getNotBeforeVersion() + " Not from=" + ao.getNotFromVersion() + 
			    				" Version=" + Constant.PROGRAM_VERSION);
					}
				}
			} catch (Exception e) {
				logger.error(e.getMessage(), e);
			}
		}
		
		for (Downloader dl : handledFiles) {
			// Cant remove in loop above as we're iterating through the list
			this.downloadFiles.remove(dl);
		}
	}
	
	public int getDownloadProgressPercent(URL url) throws Exception {
		return this.downloadManager.getProgressPercent(url);
	}
	
	public int getCurrentDownloadCount() {
		return this.downloadManager.getCurrentDownloadCount();
	}

	@Override
	public void hook(ExtensionHook extensionHook) {
	    super.hook(extensionHook);
	    if (getView() != null) {
	        extensionHook.getHookMenu().addHelpMenuItem(getMenuItemCheckUpdate());
	        extensionHook.getHookMenu().addFileMenuItem(getMenuItemLoadAddOn());
	        
			View.getSingleton().addMainToolbarButton(getAddonsButton());

			View.getSingleton().getMainFrame().getMainFooterPanel().addFooterToolbarRightLabel(getScanStatus().getCountLabel());
	    }
	    extensionHook.addCommandLine(getCommandLineArguments());
        this.api = new AutoUpdateAPI(this);
        this.api.addApiOptions(getModel().getOptionsParam().getCheckForUpdatesParam());
        extensionHook.addApiImplementor(this.api);
	}
	
	private ScanStatus getScanStatus() {
		if (scanStatus == null) {
	        scanStatus = new ScanStatus(
					new ImageIcon(
							ExtensionLog4j.class.getResource("/resource/icon/fugue/download.png")),
						Constant.messages.getString("cfu.downloads.icon.title"));
		}
		return scanStatus;
	}
    

	private JButton getAddonsButton() {
		if (addonsButton == null) {
			addonsButton = new JButton();
			addonsButton.setIcon(new ImageIcon(ExtensionAutoUpdate.class.getResource("/resource/icon/fugue/block.png")));
			addonsButton.setToolTipText(Constant.messages.getString("cfu.button.addons.browse"));
			addonsButton.setEnabled(true);
			addonsButton.addActionListener(new java.awt.event.ActionListener() { 
				@Override
				public void actionPerformed(java.awt.event.ActionEvent e) {
					getAddOnsDialog().setVisible(true);
				}
			});

		}
		return this.addonsButton;
	}
	
	@Override
	public String getAuthor() {
		return Constant.ZAP_TEAM;
	}

	@Override
	public String getDescription() {
		return Constant.messages.getString("autoupdate.desc");
	}

	@Override
	public URL getURL() {
		try {
			return new URL(Constant.ZAP_HOMEPAGE);
		} catch (MalformedURLException e) {
			return null;
		}
	}
	
	@Override
	public void destroy() {
		this.downloadManager.shutdown(true);
	}
	
    private HttpSender getHttpSender() {
        if (httpSender == null) {
            httpSender = new HttpSender(Model.getSingleton().getOptionsParam().getConnectionParam(), true, 
            		HttpSender.CHECK_FOR_UPDATES_INITIATOR);
        }
        return httpSender;
    }
    
    private int dayDiff(Date d1, Date d2) {
    	long diff = d1.getTime() - d2.getTime();
    	return (int) (diff / (1000 * 60 * 60 * 24));
    }
    
    public void alertIfNewVersions() {
    	// Kicks off a thread and pops up a window if there are new versions.
    	// Depending on the options the user has chosen.
    	// Only expect this to be called on startup and in desktop mode
    	
    	final OptionsParamCheckForUpdates options = getModel().getOptionsParam().getCheckForUpdatesParam();
    	
		if (View.isInitialised()) {
			if (options.isCheckOnStartUnset()) {
				// First time in
                int result = getView().showConfirmDialog(
                		Constant.messages.getString("cfu.confirm.startCheck"));
                if (result == JOptionPane.OK_OPTION) {
                	options.setCheckOnStart(true);
                	options.setCheckAddonUpdates(true);
                	options.setDownloadNewRelease(true);
                } else {
                	options.setCheckOnStart(false);
                }
                // Save
			    try {
			    	this.getModel().getOptionsParam().getConfig().save();
	            } catch (ConfigurationException ce) {
	            	logger.error(ce.getMessage(), ce);
	                getView().showWarningDialog(
	                		Constant.messages.getString("cfu.confirm.error"));
	                return;
	            }
			}
			if (! options.isCheckOnStart()) {
				alertIfOutOfDate(false);
				return;
			}
		}
    	
		if (! options.checkOnStart()) {
			// Top level option not set, dont do anything, unless already downloaded last release
			if (View.isInitialised() && this.getPreviousVersionInfo() != null) {
				ZapRelease rel = this.getPreviousVersionInfo().getZapRelease();
				if (rel != null && rel.isNewerThan(this.getCurrentVersion())) {
					File f = new File(Constant.FOLDER_LOCAL_PLUGIN, rel.getFileName());
					if (f.exists() && f.length() >= rel.getSize()) {
						// Already downloaded, prompt to install and exit
						this.promptToLaunchReleaseAndClose(rel.getVersion(), f);
					}
				}
			}
			return;
		}
		// Handle the response in a callback
		this.getLatestVersionInfo(this);
    }
    
    private void warnIfOutOfDate() {
    	final OptionsParamCheckForUpdates options = getModel().getOptionsParam().getCheckForUpdatesParam();
		Date today = new Date();
		Date releaseCreated = Constant.getReleaseCreateDate();
		if (releaseCreated != null) {
			// Should only be null for dev builds
			if (dayDiff(today, releaseCreated) > 365) {
				// Oh no, its more than a year old!
				if (ZAP.getProcessType().equals(ZAP.ProcessType.cmdline)) {
					CommandLine.error("This ZAP installation is over a year old - its probably very out of date");
				} else {
					logger.warn("This ZAP installation is over a year old - its probably very out of date");
				}
				return;
			}
		}
		
		Date lastChecked = options.getDayLastChecked();
		Date installDate = Constant.getInstallDate();
		if (installDate == null || dayDiff(today, installDate) < 90) {
			// Dont warn if installed in the last 3 months
		} else if (lastChecked == null || dayDiff(today, lastChecked) > 90) {
			// Not checked for update in 3 months :(
			if (ZAP.getProcessType().equals(ZAP.ProcessType.cmdline)) {
				CommandLine.error("No check for updates for over 3 month - add-ons may well be out of date");
			} else {
				logger.warn("No check for updates for over 3 month - add-ons may well be out of date");
			}
		}
    }
    

    
    private void alertIfOutOfDate(boolean alwaysPrompt) {
    	final OptionsParamCheckForUpdates options = getModel().getOptionsParam().getCheckForUpdatesParam();
		Date today = new Date();
		Date releaseCreated = Constant.getReleaseCreateDate();
		Date lastInstallWarning = options.getDayLastInstallWarned();
        int result = -1;
        logger.debug("Install created " + releaseCreated);
		if (releaseCreated != null) {
			// Should only be null for dev builds
			int daysOld = dayDiff(today, releaseCreated);
	        logger.debug("Install is " + daysOld + " days old");
			if (daysOld > 365) {
				// Oh no, its more than a year old!
				boolean setCfuOnStart = false;
				
				if (alwaysPrompt || lastInstallWarning == null || dayDiff(today, lastInstallWarning) > 30) {
					JCheckBox cfuOnStart = new JCheckBox(Constant.messages.getString("cfu.label.cfuonstart"));
					cfuOnStart.setSelected(true);
					String msg = Constant.messages.getString("cfu.label.oldzap"); 
					
					result = View.getSingleton().showYesNoDialog(
							View.getSingleton().getMainFrame(), new Object[]{msg, cfuOnStart}						);
					setCfuOnStart = cfuOnStart.isSelected();
					
				}
				options.setDayLastInstallWarned();
				
                if (result == JOptionPane.OK_OPTION) {
    				if (setCfuOnStart) {
    					options.setCheckOnStart(true);
    				}
					getAddOnsDialog().setVisible(true);
					getAddOnsDialog().checkForUpdates();
					
                } else if (!oldZapAlertAdded){
                	JButton button = new JButton(Constant.messages.getString("cfu.label.outofdatezap"));
                	button.setIcon(new ImageIcon(
                			ExtensionAutoUpdate.class.getResource("/resource/icon/16/050.png"))); // Alert triangle
                	button.addActionListener(new ActionListener() {
						@Override
						public void actionPerformed(ActionEvent e) {
							alertIfOutOfDate(true);
						}
                	});
                	
        			View.getSingleton().getMainFrame().getMainFooterPanel().addFooterToolbarLeftComponent(button );
        			oldZapAlertAdded = true;
                }
            	return;
			}
		}
		
		Date lastChecked = options.getDayLastChecked();
		Date lastUpdateWarning = options.getDayLastUpdateWarned();
		Date installDate = Constant.getInstallDate();
		if (installDate == null || dayDiff(today, installDate) < 90) {
			// Dont warn if installed in the last 3 months
		} else if (lastChecked == null || dayDiff(today, lastChecked) > 90) {
			// Not checked for updates in 3 months :(
			boolean setCfuOnStart = false;
			
			if (alwaysPrompt || lastUpdateWarning == null || dayDiff(today, lastUpdateWarning) > 30) {
				JCheckBox cfuOnStart = new JCheckBox(Constant.messages.getString("cfu.label.cfuonstart"));
				cfuOnStart.setSelected(true);
				String msg = Constant.messages.getString("cfu.label.norecentcfu");
				
				result = View.getSingleton().showYesNoDialog(
						View.getSingleton().getMainFrame(), new Object[]{msg, cfuOnStart});
				setCfuOnStart = cfuOnStart.isSelected();
				
			}
			options.setDayLastUpdateWarned();
			
            if (result == JOptionPane.OK_OPTION) {
				if (setCfuOnStart) {
					options.setCheckOnStart(true);
				}
				getAddOnsDialog().setVisible(true);
				getAddOnsDialog().checkForUpdates();
				if (noCfuAlertAdded) {
	    			View.getSingleton().getMainFrame().getMainFooterPanel().
    					removeFooterToolbarLeftComponent(getOutOfDateButton());
				}
				
            } else if (!noCfuAlertAdded){
    			View.getSingleton().getMainFrame().getMainFooterPanel().
    				addFooterToolbarLeftComponent(getOutOfDateButton());
    			noCfuAlertAdded = true;
            }
		}
    }
    
    private JButton getOutOfDateButton() {
    	if (outOfDateButton == null) {
    		outOfDateButton = new JButton(Constant.messages.getString("cfu.label.outofdateaddons"));
    		outOfDateButton.setIcon(new ImageIcon(
	    			ExtensionAutoUpdate.class.getResource("/resource/icon/16/050.png"))); // Alert triangle
    		outOfDateButton.addActionListener(new ActionListener() {
				@Override
				public void actionPerformed(ActionEvent e) {
					alertIfOutOfDate(true);
				}
	    	});
		}
    	return outOfDateButton;	
    }
    
    protected AddOnCollection getLocalVersionInfo () {
    	if (localVersionInfo == null) {
    		localVersionInfo = ExtensionFactory.getAddOnLoader().getAddOnCollection(); 
    	}
    	return localVersionInfo;
    }

    private ZapXmlConfiguration getRemoteConfigurationUrl(String url) throws 
    		IOException, ConfigurationException, InvalidCfuUrlException {
        HttpMessage msg = new HttpMessage(new URI(url, true), 
        		Model.getSingleton().getOptionsParam().getConnectionParam());
        getHttpSender().sendAndReceive(msg,true);
        if (msg.getResponseHeader().getStatusCode() != HttpStatusCode.OK) {
            throw new IOException();
        }
        if (! msg.getRequestHeader().isSecure()) {
        	// Only access the cfu page over https
            throw new InvalidCfuUrlException(msg.getRequestHeader().getURI().toString());
        }
        
    	ZapXmlConfiguration config = new ZapXmlConfiguration();
    	config.setDelimiterParsingDisabled(true);
    	config.load(new StringReader(msg.getResponseBody().toString()));

        // Save version file so we can report new addons next time
		File f = new File(Constant.FOLDER_LOCAL_PLUGIN, VERSION_FILE_NAME);
    	FileWriter out = null;
	    try {
	    	out = new FileWriter(f);
	    	out.write(msg.getResponseBody().toString());
		} catch (Exception e) {
			logger.error(e.getMessage(), e);
	    } finally {
	    	try {
				if (out != null) {
					out.close();
				}
			} catch (IOException e) {
				// Ignore
			}
		}

    	
    	return config;
    }

    protected String getLatestVersionNumber() {
    	if (this.getLatestVersionInfo() == null ||
    			this.getLatestVersionInfo().getZapRelease() == null) {
    		return null;
    	}
    	return this.getLatestVersionInfo().getZapRelease().getVersion();
    }
    
    protected boolean isLatestVersion() {
    	if (this.getLatestVersionInfo() == null ||
    			this.getLatestVersionInfo().getZapRelease() == null) {
    		return true;
    	}
		return  ! this.getLatestVersionInfo().getZapRelease().isNewerThan(this.getCurrentVersion());
    }
    
    protected boolean downloadLatestRelease() {
    	if (Constant.isKali()) {
    		if (View.isInitialised()) {
	    		// Just tell the user to use one of the Kali options
	    		View.getSingleton().showMessageDialog(this.getAddOnsDialog(), Constant.messages.getString("cfu.kali.options"));
    		}
    		return false;
    	}
    	if (this.getLatestVersionInfo() == null ||
    			this.getLatestVersionInfo().getZapRelease() == null) {
    		return false;
    	}
    	ZapRelease latestRelease = this.getLatestVersionInfo().getZapRelease();
		if (latestRelease.isNewerThan(this.getCurrentVersion())) {
			File f = new File(Constant.FOLDER_LOCAL_PLUGIN, latestRelease.getFileName());
			downloadFile(latestRelease.getUrl(), f, latestRelease.getSize(), latestRelease.getHash());
			return true;
		}
		return false;
    }
    
	private AddOnCollection getPreviousVersionInfo() {
		if (this.previousVersionInfo == null) {
			File f = new File(Constant.FOLDER_LOCAL_PLUGIN, VERSION_FILE_NAME);
			if (f.exists()) {
				try {
					this.previousVersionInfo = new AddOnCollection(new ZapXmlConfiguration(f), this.getPlatform());
				} catch (ConfigurationException e) {
					logger.error(e.getMessage(), e);
				} 
			}
		}
		return this.previousVersionInfo;
	}
    
	protected List<Downloader> getAllDownloadsProgress() {
		return this.downloadManager.getProgress();
	}

    
	protected List<AddOn> getUpdatedAddOns() {
    	return getLocalVersionInfo().getUpdatedAddOns(this.getLatestVersionInfo());
    }
    
	protected List<AddOn> getNewAddOns() {
    	if (this.getPreviousVersionInfo() != null) {
    		return this.getPreviousVersionInfo().getNewAddOns(this.getLatestVersionInfo());
    	}
    	return getLocalVersionInfo().getNewAddOns(this.getLatestVersionInfo());
    }

	protected AddOn getAddOn(String id) {
        AddOnCollection aoc = getLatestVersionInfo();
        if (aoc != null) {
            return aoc.getAddOn(id);
        }
        return null;
    }
    
	protected List<AddOn> getInstalledAddOns() {
        return getLocalVersionInfo().getInstalledAddOns();
    }

	protected List<AddOn> getMarketplaceAddOns() {
        AddOnCollection aoc = this.getLatestVersionInfo();
        if (aoc != null) {
            return aoc.getAddOns();
        }
        return Collections.<AddOn>emptyList();
    }

    protected AddOnCollection getLatestVersionInfo () {
    	return getLatestVersionInfo(null);
    }
    
    protected AddOnCollection getLatestVersionInfo (final CheckForUpdateCallback callback) {
    	if (latestVersionInfo == null) {
    		
    		if (this.remoteCallThread == null || !this.remoteCallThread.isAlive()) {
    			this.remoteCallThread = new Thread() {
    			
	    			@Override
	    			public void run() {
	    				// Using a thread as the first call could timeout
	    				// and we dont want the ui to hang in the meantime
	    				this.setName("ZAP-cfu");
						String shortUrl;
						String longUrl;
						if (Constant.isDevBuild()) {
							shortUrl = ZAP_VERSIONS_DEV_XML_SHORT;
							longUrl = ZAP_VERSIONS_DEV_XML_FULL;
						} else if (Constant.isDailyBuild()) {
							shortUrl = ZAP_VERSIONS_WEEKLY_XML_SHORT;
							longUrl = ZAP_VERSIONS_DEV_XML_FULL;
						} else {
							shortUrl = ZAP_VERSIONS_REL_XML_SHORT;
							longUrl = ZAP_VERSIONS_REL_XML_FULL;
						}
						logger.debug("Getting latest version info from " + shortUrl);
			    		try {
							latestVersionInfo = new AddOnCollection(getRemoteConfigurationUrl(shortUrl), getPlatform(), false);
						} catch (Exception e1) {
							logger.debug("Failed to access " + shortUrl, e1);
							logger.debug("Getting latest version info from " + longUrl);
				    		try {
				    			latestVersionInfo = new AddOnCollection(getRemoteConfigurationUrl(longUrl), getPlatform(), false);
				    		} catch (SSLHandshakeException e2) {
					    		if (callback != null) {
					    			callback.insecureUrl(longUrl, e2);
					    		}
							} catch (InvalidCfuUrlException e2) {
					    		if (callback != null) {
					    			callback.insecureUrl(longUrl, e2);
					    		}
							} catch (Exception e2) {
								logger.debug("Failed to access " + longUrl, e2);
							}
						}
			    		if (callback != null && latestVersionInfo != null) {
							logger.debug("Calling callback with  " + latestVersionInfo);
			    			callback.gotLatestData(latestVersionInfo);
			    		}
						logger.debug("Done");
	    			}
    			};
    			this.remoteCallThread.start();
    		}
    		if (callback == null) {
    			// Synchronous, but include a 30 sec max anyway
    			int i=0;
				while (latestVersionInfo == null && this.remoteCallThread.isAlive() && i < 30) {
					try {
						Thread.sleep(1000);
						i++;
					} catch (InterruptedException e) {
						// Ignore
					}
				}
    		}
    	}
    	return latestVersionInfo;
    }
<commit_msg>Set installation status to marketplace add-ons<commit_after>/*
 * Zed Attack Proxy (ZAP) and its related class files.
 * 
 * ZAP is an HTTP/HTTPS proxy for assessing web application security.
 * 
 * Copyright 2011 The Zed Attack Proxy Project
 * 
 * Licensed under the Apache License, Version 2.0 (the "License"); 
 * you may not use this file except in compliance with the License. 
 * You may obtain a copy of the License at 
 * 
 *   http://www.apache.org/licenses/LICENSE-2.0 
 *   
 * Unless required by applicable law or agreed to in writing, software 
 * distributed under the License is distributed on an "AS IS" BASIS, 
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 
 * See the License for the specific language governing permissions and 
 * limitations under the License. 
 */
package org.zaproxy.zap.extension.autoupdate;
import java.awt.EventQueue;
import java.awt.Toolkit;
import java.awt.Window;
import java.awt.event.ActionEvent;
import java.awt.event.ActionListener;
import java.awt.event.KeyEvent;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.StringReader;
import java.lang.reflect.InvocationTargetException;
import java.net.MalformedURLException;
import java.net.URL;
import java.nio.file.FileAlreadyExistsException;
import java.nio.file.Path;
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import java.util.Vector;

import javax.net.ssl.SSLHandshakeException;
import javax.swing.ImageIcon;
import javax.swing.JButton;
import javax.swing.JCheckBox;
import javax.swing.JFileChooser;
import javax.swing.JOptionPane;
import javax.swing.KeyStroke;
import javax.swing.SwingWorker;
import javax.swing.filechooser.FileFilter;

import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.XMLPropertiesConfiguration;
import org.apache.commons.httpclient.URI;
import org.apache.commons.io.FileUtils;
import org.apache.log4j.Logger;
import org.parosproxy.paros.CommandLine;
import org.parosproxy.paros.Constant;
import org.parosproxy.paros.control.Control;
import org.parosproxy.paros.extension.CommandLineArgument;
import org.parosproxy.paros.extension.CommandLineListener;
import org.parosproxy.paros.extension.Extension;
import org.parosproxy.paros.extension.ExtensionAdaptor;
import org.parosproxy.paros.extension.ExtensionHook;
import org.parosproxy.paros.model.FileCopier;
import org.parosproxy.paros.model.Model;
import org.parosproxy.paros.network.HttpMessage;
import org.parosproxy.paros.network.HttpSender;
import org.parosproxy.paros.network.HttpStatusCode;
import org.parosproxy.paros.view.View;
import org.zaproxy.zap.ZAP;
import org.zaproxy.zap.control.AddOn;
import org.zaproxy.zap.control.AddOn.AddOnRunRequirements;
import org.zaproxy.zap.control.AddOnCollection;
import org.zaproxy.zap.control.AddOnCollection.Platform;
import org.zaproxy.zap.control.AddOnRunIssuesUtils;
import org.zaproxy.zap.control.AddOnUninstallationProgressCallback;
import org.zaproxy.zap.control.ExtensionFactory;
import org.zaproxy.zap.control.ZapRelease;
import org.zaproxy.zap.extension.autoupdate.AddOnDependencyChecker.AddOnChangesResult;
import org.zaproxy.zap.extension.autoupdate.AddOnDependencyChecker.UninstallationResult;
import org.zaproxy.zap.extension.autoupdate.UninstallationProgressDialogue.AddOnUninstallListener;
import org.zaproxy.zap.extension.autoupdate.UninstallationProgressDialogue.UninstallationProgressEvent;
import org.zaproxy.zap.extension.autoupdate.UninstallationProgressDialogue.UninstallationProgressHandler;
import org.zaproxy.zap.extension.log4j.ExtensionLog4j;
import org.zaproxy.zap.utils.ZapXmlConfiguration;
import org.zaproxy.zap.view.ScanStatus;
import org.zaproxy.zap.view.ZapMenuItem;

public class ExtensionAutoUpdate extends ExtensionAdaptor implements CheckForUpdateCallback, CommandLineListener {
	
	private static final String NAME = "ExtensionAutoUpdate";
	
	// The short URL means that the number of checkForUpdates can be tracked - see https://bitly.com/u/psiinon
	// Note that URLs must now use https (unless you change the code;)
    
    private static final String ZAP_VERSIONS_REL_XML_SHORT = "https://bit.ly/owaspzap-2-6-0";
    private static final String ZAP_VERSIONS_REL_XML_FULL = "https://raw.githubusercontent.com/zaproxy/zap-admin/master/ZapVersions-2.6.xml";
    
    private static final String ZAP_VERSIONS_DEV_XML_SHORT = "https://bit.ly/owaspzap-dev";
    private static final String ZAP_VERSIONS_DEV_XML_FULL = "https://raw.githubusercontent.com/zaproxy/zap-admin/master/ZapVersions-dev.xml";
    private static final String ZAP_VERSIONS_WEEKLY_XML_SHORT = "https://bit.ly/owaspzap-devw";

	// URLs for use when testing locally ;)
	//private static final String ZAP_VERSIONS_XML_SHORT = "https://localhost:8080/zapcfu/ZapVersions.xml";
	//private static final String ZAP_VERSIONS_XML_FULL = "https://localhost:8080/zapcfu/ZapVersions.xml";

	private static final String VERSION_FILE_NAME = "ZapVersions.xml";

	private ZapMenuItem menuItemCheckUpdate = null;
	private ZapMenuItem menuItemLoadAddOn = null;
    
    private static final Logger logger = Logger.getLogger(ExtensionAutoUpdate.class);
    
	private HttpSender httpSender = null;

    private DownloadManager downloadManager = null;
	private ManageAddOnsDialog addonsDialog = null;
	//private UpdateDialog updateDialog = null;
	private Thread downloadProgressThread = null;
	private Thread remoteCallThread = null; 
	private ScanStatus scanStatus = null;
	private JButton addonsButton = null;
    private JButton outOfDateButton = null;

	private AddOnCollection latestVersionInfo = null;
	private AddOnCollection localVersionInfo = null;
	private AddOnCollection previousVersionInfo = null;

    private AutoUpdateAPI api = null;

    private boolean oldZapAlertAdded = false;
    private boolean noCfuAlertAdded = false;

    // Files currently being downloaded
	private List<Downloader> downloadFiles = new ArrayList<>();

    private static final int ARG_CFU_INSTALL_IDX = 0;
    private static final int ARG_CFU_INSTALL_ALL_IDX = 1;
    private static final int ARG_CFU_UNINSTALL_IDX = 2;
    private static final int ARG_CFU_UPDATE_IDX = 3;
    private static final int ARG_CFU_LIST_IDX = 4;
    private static final int[] ARG_IDXS = {
    							ARG_CFU_INSTALL_IDX,
    							ARG_CFU_INSTALL_ALL_IDX,
    							ARG_CFU_UNINSTALL_IDX,
    							ARG_CFU_UPDATE_IDX,
    							ARG_CFU_LIST_IDX};
	private CommandLineArgument[] arguments = new CommandLineArgument[ARG_IDXS.length];

    public ExtensionAutoUpdate() {
        super();
 		initialize();
   }   

	/**
	 * This method initializes this
	 */
	private void initialize() {
        this.setName(NAME);
        this.setOrder(1);	// High order so that cmdline updates are installed asap
        this.downloadManager = new DownloadManager(Model.getSingleton().getOptionsParam().getConnectionParam());
        this.downloadManager.start();
        // Do this before it can get overwritten by the latest one
        this.getPreviousVersionInfo();
	}

	@Override
	public String getUIName() {
		return Constant.messages.getString("autoupdate.name");
	}
	
	@Override
	public void postInit() {
		switch (ZAP.getProcessType()) {
		case cmdline:
		case daemon:
		case zaas:
			this.warnIfOutOfDate();
			break;
		case desktop:
		default:
			break;
		}
	}

	/**
	 * This method initializes menuItemEncoder	
	 * 	
	 * @return javax.swing.JMenuItem	
	 */    
	private ZapMenuItem getMenuItemCheckUpdate() {
		if (menuItemCheckUpdate == null) {
			menuItemCheckUpdate = new ZapMenuItem("cfu.help.menu.check", 
					KeyStroke.getKeyStroke(KeyEvent.VK_U, Toolkit.getDefaultToolkit().getMenuShortcutKeyMask(), false));
			menuItemCheckUpdate.setText(Constant.messages.getString("cfu.help.menu.check"));
			menuItemCheckUpdate.addActionListener(new java.awt.event.ActionListener() { 
				@Override
				public void actionPerformed(java.awt.event.ActionEvent e) {    
					getAddOnsDialog().setVisible(true);
					getAddOnsDialog().checkForUpdates();
				}

			});
		}
		return menuItemCheckUpdate;
	}

	private ZapMenuItem getMenuItemLoadAddOn() {
		if (menuItemLoadAddOn == null) {
			menuItemLoadAddOn = new ZapMenuItem("cfu.file.menu.loadaddon", 
					KeyStroke.getKeyStroke(KeyEvent.VK_L, Toolkit.getDefaultToolkit().getMenuShortcutKeyMask(), false));
			menuItemLoadAddOn.addActionListener(new java.awt.event.ActionListener() { 
				@Override
				public void actionPerformed(java.awt.event.ActionEvent e) {
					try {
						JFileChooser chooser = new JFileChooser(Model.getSingleton().getOptionsParam().getUserDirectory());
						File file = null;
						chooser.setFileFilter(new FileFilter() {
						       @Override
						       public boolean accept(File file) {
						            if (file.isDirectory()) {
						                return true;
						            } else if (file.isFile() && AddOn.isAddOnFileName(file.getName())) {
						                return true;
						            }
						            return false;
						        }
						       @Override
						       public String getDescription() {
						           return Constant.messages.getString("file.format.zap.addon");
						       }
						});
						int rc = chooser.showOpenDialog(View.getSingleton().getMainFrame());
						if(rc == JFileChooser.APPROVE_OPTION) {
							file = chooser.getSelectedFile();
							if (file == null) {
								return;
							}
							installLocalAddOn(file.toPath());
						}
					} catch (Exception e1) {
						logger.error(e1.getMessage(), e1);
					}
				}
			});
		}
		return menuItemLoadAddOn;
	}
	
	private void installLocalAddOn(Path file) throws Exception {
		if (!AddOn.isAddOn(file)) {
			showWarningMessageInvalidAddOnFile();
			return;
		}

		AddOn ao;
		try {
			ao = new AddOn(file);
		} catch (Exception e) {
			showWarningMessageInvalidAddOnFile();
			return;
		}

		if (!ao.canLoadInCurrentVersion()) {
			showWarningMessageCantLoadAddOn(ao);
			return;
		}

		AddOnDependencyChecker dependencyChecker = new AddOnDependencyChecker(getLocalVersionInfo(), latestVersionInfo == null
				? getLocalVersionInfo()
				: latestVersionInfo);

		boolean update = false;
		AddOnChangesResult result;
		AddOn installedAddOn = getLocalVersionInfo().getAddOn(ao.getId());
		if (installedAddOn != null) {
			if (!ao.isUpdateTo(installedAddOn)) {
				View.getSingleton().showWarningDialog(
						MessageFormat.format(
								Constant.messages.getString("cfu.warn.addOnOlderVersion"),
								installedAddOn.getVersion(),
								View.getSingleton().getStatusUI(installedAddOn.getStatus()).toString(),
								ao.getVersion(),
								View.getSingleton().getStatusUI(ao.getStatus()).toString()));
				return;
			}

			result = dependencyChecker.calculateUpdateChanges(ao);
			update = true;
		} else {
			result = dependencyChecker.calculateInstallChanges(ao);
		}

		if (result.getOldVersions().isEmpty() && result.getUninstalls().isEmpty()) {
			AddOnRunRequirements reqs = ao.calculateRunRequirements(getLocalVersionInfo().getAddOns());
			if (!reqs.isRunnable()) {
				if (!AddOnRunIssuesUtils.askConfirmationAddOnNotRunnable(
						Constant.messages.getString("cfu.warn.addOnNotRunnable.message"),
						Constant.messages.getString("cfu.warn.addOnNotRunnable.question"),
						getLocalVersionInfo(),
						ao)) {
					return;
				}
			}
			
			installLocalAddOn(ao);
			return;
		}

		if (update) {
			if (!dependencyChecker.confirmUpdateChanges(getView().getMainFrame(), result)) {
				return;
			}
			// The new version of the add-on is installed manually
			result.getNewVersions().remove(ao);
		} else {
			if (!dependencyChecker.confirmInstallChanges(getView().getMainFrame(), result)) {
				return;
			}
			// The add-on is installed manually
			result.getInstalls().remove(ao);
		}
		
		processAddOnChanges(getView().getMainFrame(), result);
		installLocalAddOn(ao);
	}

	private void installLocalAddOn(AddOn ao) {
		File addOnFile;
		try {
			addOnFile = copyAddOnFileToLocalPluginFolder(ao);
		} catch (FileAlreadyExistsException e) {
			showWarningMessageAddOnFileAlreadyExists(e.getFile(), e.getOtherFile());
			logger.warn("Unable to copy add-on, a file with the same name already exists.", e);
			return;
		} catch (IOException e) {
			showWarningMessageUnableToCopyAddOnFile();
			logger.warn("Unable to copy add-on to local plugin folder.", e);
			return;
		}

		ao.setFile(addOnFile);

		install(ao);
	}

	private void showWarningMessageInvalidAddOnFile() {
		View.getSingleton().showWarningDialog(Constant.messages.getString("cfu.warn.invalidAddOn"));
	}

	private void showWarningMessageCantLoadAddOn(AddOn ao) {
		String message = MessageFormat.format(
				Constant.messages.getString("cfu.warn.cantload"),
				ao.getNotBeforeVersion(),
				ao.getNotFromVersion());
		View.getSingleton().showWarningDialog(message);
	}

	private static File copyAddOnFileToLocalPluginFolder(AddOn addOn) throws IOException {
		if (isFileInLocalPluginFolder(addOn.getFile())) {
			return addOn.getFile();
		}

		File targetFile = new File(Constant.FOLDER_LOCAL_PLUGIN, addOn.getNormalisedFileName());
		if (targetFile.exists()) {
			throw new FileAlreadyExistsException(addOn.getFile().getAbsolutePath(), targetFile.getAbsolutePath(), "");
		}

		FileCopier fileCopier = new FileCopier();
		fileCopier.copy(addOn.getFile(), targetFile);

		return targetFile;
	}

	private static boolean isFileInLocalPluginFolder(File file) {
		File fileLocalPluginFolder = new File(Constant.FOLDER_LOCAL_PLUGIN, file.getName());
		if (fileLocalPluginFolder.getAbsolutePath().equals(file.getAbsolutePath())) {
			return true;
		}
		return false;
	}

	private static void showWarningMessageAddOnFileAlreadyExists(String file, String targetFile) {
		String message = MessageFormat.format(Constant.messages.getString("cfu.warn.addOnAlreadExists"), file, targetFile);
		View.getSingleton().showWarningDialog(message);
	}

	private static void showWarningMessageUnableToCopyAddOnFile() {
		String pathPluginFolder = new File(Constant.FOLDER_LOCAL_PLUGIN).getAbsolutePath();
		String message = MessageFormat.format(Constant.messages.getString("cfu.warn.unableToCopyAddOn"), pathPluginFolder);
		View.getSingleton().showWarningDialog(message);
	}
	

	private synchronized ManageAddOnsDialog getAddOnsDialog() {
		if (addonsDialog == null) {
			addonsDialog = new ManageAddOnsDialog(this, this.getCurrentVersion(), getLocalVersionInfo());
			if (this.previousVersionInfo != null) {
				addonsDialog.setPreviousVersionInfo(this.previousVersionInfo);
			}
			if (this.latestVersionInfo != null) {
				addonsDialog.setLatestVersionInfo(this.latestVersionInfo);
			}
		}
		return addonsDialog;
	}
	
	private void downloadFile (URL url, File targetFile, long size, String hash) {
		if (View.isInitialised()) {
			// Report info to the Output tab
			View.getSingleton().getOutputPanel().append(
					MessageFormat.format(
							Constant.messages.getString("cfu.output.downloading") + "\n", 
							url.toString(),
							targetFile.getAbsolutePath()));
		}
		this.downloadFiles.add(this.downloadManager.downloadFile(url, targetFile, size, hash));
		if (View.isInitialised()) {
			// Means we do have a UI
			if (this.downloadProgressThread != null && ! this.downloadProgressThread.isAlive()) {
				this.downloadProgressThread = null;
			}
			if (this.downloadProgressThread == null) {
				this.downloadProgressThread = new Thread() {
					@Override
					public void run() {
						while (downloadManager.getCurrentDownloadCount() > 0) {
							getScanStatus().setScanCount(downloadManager.getCurrentDownloadCount());
							if (addonsDialog != null && addonsDialog.isVisible()) {
								addonsDialog.showProgress();
							}
							try {
								sleep(100);
							} catch (InterruptedException e) {
								// Ignore
							}
						}
						// Complete download progress
						if (addonsDialog != null) {
							addonsDialog.showProgress();
						}
						getScanStatus().setScanCount(0);
						installNewExtensions();
					}
				};
				this.downloadProgressThread.start();
			}
		}
	}
	
	public void installNewExtensions() {
    	final OptionsParamCheckForUpdates options = getModel().getOptionsParam().getCheckForUpdatesParam();
		List<Downloader> handledFiles = new ArrayList<>();
		
		for (Downloader dl : downloadFiles) {
			if (dl.getFinished() == null) {
				continue;
			}
			handledFiles.add(dl);
			try {
				if (!dl.isValidated()) {
					logger.debug("Ignoring unvalidated download: " + dl.getUrl());
					if (addonsDialog != null) {
						addonsDialog.notifyAddOnDownloadFailed(dl.getUrl().toString());
					} else {
						String url = dl.getUrl().toString();
						for (AddOn addOn : latestVersionInfo.getAddOns()) {
							if (url.equals(addOn.getUrl().toString())) {
								addOn.setInstallationStatus(AddOn.InstallationStatus.AVAILABLE);
								break;
							}
						}
					}
				} else if (AddOn.isAddOn(dl.getTargetFile().toPath())) {
					File f = dl.getTargetFile();
					if (! options.getDownloadDirectory().equals(dl.getTargetFile().getParentFile())) {
						// Move the file to the specified directory - we do this after its been downloaded
						// as these directories can be shared, and other ZAP instances could get incomplete
						// add-ons
						try {
							f = new File(options.getDownloadDirectory(), dl.getTargetFile().getName());
							logger.info("Moving downloaded add-on from " + dl.getTargetFile().getAbsolutePath() +
									" to " + f.getAbsolutePath());
							FileUtils.moveFile(dl.getTargetFile(), f);
						} catch (Exception e) {
							if (!f.exists() && dl.getTargetFile().exists()) {
								logger.error("Failed to move downloaded add-on from " + dl.getTargetFile().getAbsolutePath() +
										" to " + f.getAbsolutePath() + " - left at original location", e);
								f = dl.getTargetFile();
							} else {
								logger.error("Failed to move downloaded add-on from " + dl.getTargetFile().getAbsolutePath() +
										" to " + f.getAbsolutePath() + " - skipping", e);
								continue;
							}
						}
					}
					
					AddOn ao = new AddOn(f.toPath());
					if (ao.canLoadInCurrentVersion()) {
						install(ao);
					} else {
			    		logger.info("Cant load add-on " + ao.getName() + 
			    				" Not before=" + ao.getNotBeforeVersion() + " Not from=" + ao.getNotFromVersion() + 
			    				" Version=" + Constant.PROGRAM_VERSION);
					}
				}
			} catch (Exception e) {
				logger.error(e.getMessage(), e);
			}
		}
		
		for (Downloader dl : handledFiles) {
			// Cant remove in loop above as we're iterating through the list
			this.downloadFiles.remove(dl);
		}
	}
	
	public int getDownloadProgressPercent(URL url) throws Exception {
		return this.downloadManager.getProgressPercent(url);
	}
	
	public int getCurrentDownloadCount() {
		return this.downloadManager.getCurrentDownloadCount();
	}

	@Override
	public void hook(ExtensionHook extensionHook) {
	    super.hook(extensionHook);
	    if (getView() != null) {
	        extensionHook.getHookMenu().addHelpMenuItem(getMenuItemCheckUpdate());
	        extensionHook.getHookMenu().addFileMenuItem(getMenuItemLoadAddOn());
	        
			View.getSingleton().addMainToolbarButton(getAddonsButton());

			View.getSingleton().getMainFrame().getMainFooterPanel().addFooterToolbarRightLabel(getScanStatus().getCountLabel());
	    }
	    extensionHook.addCommandLine(getCommandLineArguments());
        this.api = new AutoUpdateAPI(this);
        this.api.addApiOptions(getModel().getOptionsParam().getCheckForUpdatesParam());
        extensionHook.addApiImplementor(this.api);
	}
	
	private ScanStatus getScanStatus() {
		if (scanStatus == null) {
	        scanStatus = new ScanStatus(
					new ImageIcon(
							ExtensionLog4j.class.getResource("/resource/icon/fugue/download.png")),
						Constant.messages.getString("cfu.downloads.icon.title"));
		}
		return scanStatus;
	}
    

	private JButton getAddonsButton() {
		if (addonsButton == null) {
			addonsButton = new JButton();
			addonsButton.setIcon(new ImageIcon(ExtensionAutoUpdate.class.getResource("/resource/icon/fugue/block.png")));
			addonsButton.setToolTipText(Constant.messages.getString("cfu.button.addons.browse"));
			addonsButton.setEnabled(true);
			addonsButton.addActionListener(new java.awt.event.ActionListener() { 
				@Override
				public void actionPerformed(java.awt.event.ActionEvent e) {
					getAddOnsDialog().setVisible(true);
				}
			});

		}
		return this.addonsButton;
	}
	
	@Override
	public String getAuthor() {
		return Constant.ZAP_TEAM;
	}

	@Override
	public String getDescription() {
		return Constant.messages.getString("autoupdate.desc");
	}

	@Override
	public URL getURL() {
		try {
			return new URL(Constant.ZAP_HOMEPAGE);
		} catch (MalformedURLException e) {
			return null;
		}
	}
	
	@Override
	public void destroy() {
		this.downloadManager.shutdown(true);
	}
	
    private HttpSender getHttpSender() {
        if (httpSender == null) {
            httpSender = new HttpSender(Model.getSingleton().getOptionsParam().getConnectionParam(), true, 
            		HttpSender.CHECK_FOR_UPDATES_INITIATOR);
        }
        return httpSender;
    }
    
    private int dayDiff(Date d1, Date d2) {
    	long diff = d1.getTime() - d2.getTime();
    	return (int) (diff / (1000 * 60 * 60 * 24));
    }
    
    public void alertIfNewVersions() {
    	// Kicks off a thread and pops up a window if there are new versions.
    	// Depending on the options the user has chosen.
    	// Only expect this to be called on startup and in desktop mode
    	
    	final OptionsParamCheckForUpdates options = getModel().getOptionsParam().getCheckForUpdatesParam();
    	
		if (View.isInitialised()) {
			if (options.isCheckOnStartUnset()) {
				// First time in
                int result = getView().showConfirmDialog(
                		Constant.messages.getString("cfu.confirm.startCheck"));
                if (result == JOptionPane.OK_OPTION) {
                	options.setCheckOnStart(true);
                	options.setCheckAddonUpdates(true);
                	options.setDownloadNewRelease(true);
                } else {
                	options.setCheckOnStart(false);
                }
                // Save
			    try {
			    	this.getModel().getOptionsParam().getConfig().save();
	            } catch (ConfigurationException ce) {
	            	logger.error(ce.getMessage(), ce);
	                getView().showWarningDialog(
	                		Constant.messages.getString("cfu.confirm.error"));
	                return;
	            }
			}
			if (! options.isCheckOnStart()) {
				alertIfOutOfDate(false);
				return;
			}
		}
    	
		if (! options.checkOnStart()) {
			// Top level option not set, dont do anything, unless already downloaded last release
			if (View.isInitialised() && this.getPreviousVersionInfo() != null) {
				ZapRelease rel = this.getPreviousVersionInfo().getZapRelease();
				if (rel != null && rel.isNewerThan(this.getCurrentVersion())) {
					File f = new File(Constant.FOLDER_LOCAL_PLUGIN, rel.getFileName());
					if (f.exists() && f.length() >= rel.getSize()) {
						// Already downloaded, prompt to install and exit
						this.promptToLaunchReleaseAndClose(rel.getVersion(), f);
					}
				}
			}
			return;
		}
		// Handle the response in a callback
		this.getLatestVersionInfo(this);
    }
    
    private void warnIfOutOfDate() {
    	final OptionsParamCheckForUpdates options = getModel().getOptionsParam().getCheckForUpdatesParam();
		Date today = new Date();
		Date releaseCreated = Constant.getReleaseCreateDate();
		if (releaseCreated != null) {
			// Should only be null for dev builds
			if (dayDiff(today, releaseCreated) > 365) {
				// Oh no, its more than a year old!
				if (ZAP.getProcessType().equals(ZAP.ProcessType.cmdline)) {
					CommandLine.error("This ZAP installation is over a year old - its probably very out of date");
				} else {
					logger.warn("This ZAP installation is over a year old - its probably very out of date");
				}
				return;
			}
		}
		
		Date lastChecked = options.getDayLastChecked();
		Date installDate = Constant.getInstallDate();
		if (installDate == null || dayDiff(today, installDate) < 90) {
			// Dont warn if installed in the last 3 months
		} else if (lastChecked == null || dayDiff(today, lastChecked) > 90) {
			// Not checked for update in 3 months :(
			if (ZAP.getProcessType().equals(ZAP.ProcessType.cmdline)) {
				CommandLine.error("No check for updates for over 3 month - add-ons may well be out of date");
			} else {
				logger.warn("No check for updates for over 3 month - add-ons may well be out of date");
			}
		}
    }
    

    
    private void alertIfOutOfDate(boolean alwaysPrompt) {
    	final OptionsParamCheckForUpdates options = getModel().getOptionsParam().getCheckForUpdatesParam();
		Date today = new Date();
		Date releaseCreated = Constant.getReleaseCreateDate();
		Date lastInstallWarning = options.getDayLastInstallWarned();
        int result = -1;
        logger.debug("Install created " + releaseCreated);
		if (releaseCreated != null) {
			// Should only be null for dev builds
			int daysOld = dayDiff(today, releaseCreated);
	        logger.debug("Install is " + daysOld + " days old");
			if (daysOld > 365) {
				// Oh no, its more than a year old!
				boolean setCfuOnStart = false;
				
				if (alwaysPrompt || lastInstallWarning == null || dayDiff(today, lastInstallWarning) > 30) {
					JCheckBox cfuOnStart = new JCheckBox(Constant.messages.getString("cfu.label.cfuonstart"));
					cfuOnStart.setSelected(true);
					String msg = Constant.messages.getString("cfu.label.oldzap"); 
					
					result = View.getSingleton().showYesNoDialog(
							View.getSingleton().getMainFrame(), new Object[]{msg, cfuOnStart}						);
					setCfuOnStart = cfuOnStart.isSelected();
					
				}
				options.setDayLastInstallWarned();
				
                if (result == JOptionPane.OK_OPTION) {
    				if (setCfuOnStart) {
    					options.setCheckOnStart(true);
    				}
					getAddOnsDialog().setVisible(true);
					getAddOnsDialog().checkForUpdates();
					
                } else if (!oldZapAlertAdded){
                	JButton button = new JButton(Constant.messages.getString("cfu.label.outofdatezap"));
                	button.setIcon(new ImageIcon(
                			ExtensionAutoUpdate.class.getResource("/resource/icon/16/050.png"))); // Alert triangle
                	button.addActionListener(new ActionListener() {
						@Override
						public void actionPerformed(ActionEvent e) {
							alertIfOutOfDate(true);
						}
                	});
                	
        			View.getSingleton().getMainFrame().getMainFooterPanel().addFooterToolbarLeftComponent(button );
        			oldZapAlertAdded = true;
                }
            	return;
			}
		}
		
		Date lastChecked = options.getDayLastChecked();
		Date lastUpdateWarning = options.getDayLastUpdateWarned();
		Date installDate = Constant.getInstallDate();
		if (installDate == null || dayDiff(today, installDate) < 90) {
			// Dont warn if installed in the last 3 months
		} else if (lastChecked == null || dayDiff(today, lastChecked) > 90) {
			// Not checked for updates in 3 months :(
			boolean setCfuOnStart = false;
			
			if (alwaysPrompt || lastUpdateWarning == null || dayDiff(today, lastUpdateWarning) > 30) {
				JCheckBox cfuOnStart = new JCheckBox(Constant.messages.getString("cfu.label.cfuonstart"));
				cfuOnStart.setSelected(true);
				String msg = Constant.messages.getString("cfu.label.norecentcfu");
				
				result = View.getSingleton().showYesNoDialog(
						View.getSingleton().getMainFrame(), new Object[]{msg, cfuOnStart});
				setCfuOnStart = cfuOnStart.isSelected();
				
			}
			options.setDayLastUpdateWarned();
			
            if (result == JOptionPane.OK_OPTION) {
				if (setCfuOnStart) {
					options.setCheckOnStart(true);
				}
				getAddOnsDialog().setVisible(true);
				getAddOnsDialog().checkForUpdates();
				if (noCfuAlertAdded) {
	    			View.getSingleton().getMainFrame().getMainFooterPanel().
    					removeFooterToolbarLeftComponent(getOutOfDateButton());
				}
				
            } else if (!noCfuAlertAdded){
    			View.getSingleton().getMainFrame().getMainFooterPanel().
    				addFooterToolbarLeftComponent(getOutOfDateButton());
    			noCfuAlertAdded = true;
            }
		}
    }
    
    private JButton getOutOfDateButton() {
    	if (outOfDateButton == null) {
    		outOfDateButton = new JButton(Constant.messages.getString("cfu.label.outofdateaddons"));
    		outOfDateButton.setIcon(new ImageIcon(
	    			ExtensionAutoUpdate.class.getResource("/resource/icon/16/050.png"))); // Alert triangle
    		outOfDateButton.addActionListener(new ActionListener() {
				@Override
				public void actionPerformed(ActionEvent e) {
					alertIfOutOfDate(true);
				}
	    	});
		}
    	return outOfDateButton;	
    }
    
    protected AddOnCollection getLocalVersionInfo () {
    	if (localVersionInfo == null) {
    		localVersionInfo = ExtensionFactory.getAddOnLoader().getAddOnCollection(); 
    	}
    	return localVersionInfo;
    }

    private ZapXmlConfiguration getRemoteConfigurationUrl(String url) throws 
    		IOException, ConfigurationException, InvalidCfuUrlException {
        HttpMessage msg = new HttpMessage(new URI(url, true), 
        		Model.getSingleton().getOptionsParam().getConnectionParam());
        getHttpSender().sendAndReceive(msg,true);
        if (msg.getResponseHeader().getStatusCode() != HttpStatusCode.OK) {
            throw new IOException();
        }
        if (! msg.getRequestHeader().isSecure()) {
        	// Only access the cfu page over https
            throw new InvalidCfuUrlException(msg.getRequestHeader().getURI().toString());
        }
        
    	ZapXmlConfiguration config = new ZapXmlConfiguration();
    	config.setDelimiterParsingDisabled(true);
    	config.load(new StringReader(msg.getResponseBody().toString()));

        // Save version file so we can report new addons next time
		File f = new File(Constant.FOLDER_LOCAL_PLUGIN, VERSION_FILE_NAME);
    	FileWriter out = null;
	    try {
	    	out = new FileWriter(f);
	    	out.write(msg.getResponseBody().toString());
		} catch (Exception e) {
			logger.error(e.getMessage(), e);
	    } finally {
	    	try {
				if (out != null) {
					out.close();
				}
			} catch (IOException e) {
				// Ignore
			}
		}

    	
    	return config;
    }

    protected String getLatestVersionNumber() {
    	if (this.getLatestVersionInfo() == null ||
    			this.getLatestVersionInfo().getZapRelease() == null) {
    		return null;
    	}
    	return this.getLatestVersionInfo().getZapRelease().getVersion();
    }
    
    protected boolean isLatestVersion() {
    	if (this.getLatestVersionInfo() == null ||
    			this.getLatestVersionInfo().getZapRelease() == null) {
    		return true;
    	}
		return  ! this.getLatestVersionInfo().getZapRelease().isNewerThan(this.getCurrentVersion());
    }
    
    protected boolean downloadLatestRelease() {
    	if (Constant.isKali()) {
    		if (View.isInitialised()) {
	    		// Just tell the user to use one of the Kali options
	    		View.getSingleton().showMessageDialog(this.getAddOnsDialog(), Constant.messages.getString("cfu.kali.options"));
    		}
    		return false;
    	}
    	if (this.getLatestVersionInfo() == null ||
    			this.getLatestVersionInfo().getZapRelease() == null) {
    		return false;
    	}
    	ZapRelease latestRelease = this.getLatestVersionInfo().getZapRelease();
		if (latestRelease.isNewerThan(this.getCurrentVersion())) {
			File f = new File(Constant.FOLDER_LOCAL_PLUGIN, latestRelease.getFileName());
			downloadFile(latestRelease.getUrl(), f, latestRelease.getSize(), latestRelease.getHash());
			return true;
		}
		return false;
    }
    
	private AddOnCollection getPreviousVersionInfo() {
		if (this.previousVersionInfo == null) {
			File f = new File(Constant.FOLDER_LOCAL_PLUGIN, VERSION_FILE_NAME);
			if (f.exists()) {
				try {
					this.previousVersionInfo = new AddOnCollection(new ZapXmlConfiguration(f), this.getPlatform());
				} catch (ConfigurationException e) {
					logger.error(e.getMessage(), e);
				} 
			}
		}
		return this.previousVersionInfo;
	}
    
	protected List<Downloader> getAllDownloadsProgress() {
		return this.downloadManager.getProgress();
	}

    
	protected List<AddOn> getUpdatedAddOns() {
    	return getLocalVersionInfo().getUpdatedAddOns(this.getLatestVersionInfo());
    }
    
	protected List<AddOn> getNewAddOns() {
    	if (this.getPreviousVersionInfo() != null) {
    		return this.getPreviousVersionInfo().getNewAddOns(this.getLatestVersionInfo());
    	}
    	return getLocalVersionInfo().getNewAddOns(this.getLatestVersionInfo());
    }

	protected AddOn getAddOn(String id) {
        AddOnCollection aoc = getLatestVersionInfo();
        if (aoc != null) {
            return aoc.getAddOn(id);
        }
        return null;
    }
    
	protected List<AddOn> getInstalledAddOns() {
        return getLocalVersionInfo().getInstalledAddOns();
    }

	protected List<AddOn> getMarketplaceAddOns() {
        AddOnCollection aoc = this.getLatestVersionInfo();
        if (aoc != null) {
            return aoc.getAddOns();
        }
        return Collections.<AddOn>emptyList();
    }

    protected AddOnCollection getLatestVersionInfo () {
    	return getLatestVersionInfo(null);
    }
    
    protected AddOnCollection getLatestVersionInfo (final CheckForUpdateCallback callback) {
    	if (latestVersionInfo == null) {
    		
    		if (this.remoteCallThread == null || !this.remoteCallThread.isAlive()) {
    			this.remoteCallThread = new Thread() {
    			
	    			@Override
	    			public void run() {
	    				// Using a thread as the first call could timeout
	    				// and we dont want the ui to hang in the meantime
	    				this.setName("ZAP-cfu");
						String shortUrl;
						String longUrl;
						if (Constant.isDevBuild()) {
							shortUrl = ZAP_VERSIONS_DEV_XML_SHORT;
							longUrl = ZAP_VERSIONS_DEV_XML_FULL;
						} else if (Constant.isDailyBuild()) {
							shortUrl = ZAP_VERSIONS_WEEKLY_XML_SHORT;
							longUrl = ZAP_VERSIONS_DEV_XML_FULL;
						} else {
							shortUrl = ZAP_VERSIONS_REL_XML_SHORT;
							longUrl = ZAP_VERSIONS_REL_XML_FULL;
						}
						logger.debug("Getting latest version info from " + shortUrl);
			    		try {
							latestVersionInfo = new AddOnCollection(getRemoteConfigurationUrl(shortUrl), getPlatform(), false);
						} catch (Exception e1) {
							logger.debug("Failed to access " + shortUrl, e1);
							logger.debug("Getting latest version info from " + longUrl);
				    		try {
				    			latestVersionInfo = new AddOnCollection(getRemoteConfigurationUrl(longUrl), getPlatform(), false);
				    		} catch (SSLHandshakeException e2) {
					    		if (callback != null) {
					    			callback.insecureUrl(longUrl, e2);
					    		}
							} catch (InvalidCfuUrlException e2) {
					    		if (callback != null) {
					    			callback.insecureUrl(longUrl, e2);
					    		}
							} catch (Exception e2) {
								logger.debug("Failed to access " + longUrl, e2);
							}
						}
						if (latestVersionInfo != null) {
							for (AddOn addOn : latestVersionInfo.getAddOns()) {
								AddOn localAddOn = localVersionInfo.getAddOn(addOn.getId());
								if (localAddOn != null && !addOn.isUpdateTo(localAddOn)) {
									addOn.setInstallationStatus(localAddOn.getInstallationStatus());
								}
							}
							if (callback != null) {
								logger.debug("Calling callback with  " + latestVersionInfo);
								callback.gotLatestData(latestVersionInfo);
							}
			    		}
						logger.debug("Done");
	    			}
    			};
    			this.remoteCallThread.start();
    		}
    		if (callback == null) {
    			// Synchronous, but include a 30 sec max anyway
    			int i=0;
				while (latestVersionInfo == null && this.remoteCallThread.isAlive() && i < 30) {
					try {
						Thread.sleep(1000);
						i++;
					} catch (InterruptedException e) {
						// Ignore
					}
				}
    		}
    	}
    	return latestVersionInfo;
    }

{'fraction_non_alphanumeric': 0.11066033425181042, 'fraction_numerical': 0.002907741515625649, 'mean_word_length': 3.3194976076555025, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 39, 'https://': 16, 'lorem ipsum': 0, 'www.': 2, 'xml': 10}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 108, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5217292', 'n_tokens_mistral': 25448, 'n_tokens_neox': 21616, 'n_words': 4344}