<commit_before>/*******************************************************************************
 * Copyright (c) 2004, 2007 Mylyn project committers and others.
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the Eclipse Public License v1.0
 * which accompanies this distribution, and is available at
 * http://www.eclipse.org/legal/epl-v10.html
 *******************************************************************************/

package org.eclipse.mylyn.internal.jira.core;

import java.util.ArrayList;
import java.util.Collection;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.eclipse.core.runtime.CoreException;
import org.eclipse.core.runtime.IProgressMonitor;
import org.eclipse.core.runtime.IStatus;
import org.eclipse.core.runtime.NullProgressMonitor;
import org.eclipse.core.runtime.Platform;
import org.eclipse.core.runtime.Status;
import org.eclipse.mylyn.commons.net.Policy;
import org.eclipse.mylyn.internal.jira.core.model.JiraFilter;
import org.eclipse.mylyn.internal.jira.core.model.JiraIssue;
import org.eclipse.mylyn.internal.jira.core.model.Priority;
import org.eclipse.mylyn.internal.jira.core.model.Project;
import org.eclipse.mylyn.internal.jira.core.model.filter.DateRangeFilter;
import org.eclipse.mylyn.internal.jira.core.model.filter.FilterDefinition;
import org.eclipse.mylyn.internal.jira.core.model.filter.IssueCollector;
import org.eclipse.mylyn.internal.jira.core.model.filter.Order;
import org.eclipse.mylyn.internal.jira.core.model.filter.RelativeDateRangeFilter;
import org.eclipse.mylyn.internal.jira.core.model.filter.RelativeDateRangeFilter.RangeType;
import org.eclipse.mylyn.internal.jira.core.service.JiraClient;
import org.eclipse.mylyn.internal.jira.core.service.JiraException;
import org.eclipse.mylyn.internal.jira.core.util.JiraUtil;
import org.eclipse.mylyn.tasks.core.AbstractRepositoryConnector;
import org.eclipse.mylyn.tasks.core.IRepositoryQuery;
import org.eclipse.mylyn.tasks.core.ITask;
import org.eclipse.mylyn.tasks.core.RepositoryStatus;
import org.eclipse.mylyn.tasks.core.TaskRepository;
import org.eclipse.mylyn.tasks.core.ITask.PriorityLevel;
import org.eclipse.mylyn.tasks.core.data.TaskAttribute;
import org.eclipse.mylyn.tasks.core.data.TaskData;
import org.eclipse.mylyn.tasks.core.data.TaskDataCollector;
import org.eclipse.mylyn.tasks.core.data.TaskMapper;
import org.eclipse.mylyn.tasks.core.data.TaskRelation;
import org.eclipse.mylyn.tasks.core.sync.ISynchronizationSession;

/**
 * @author <NAME>
 * @author <NAME>
 * @author <NAME>
 * @since 3.0
 */
public class JiraRepositoryConnector extends AbstractRepositoryConnector {

	private static final String ID_STATUS_RESOLVED = "5";

	private static final String ID_STATUS_CLOSED = "6";

	private static final String ERROR_REPOSITORY_CONFIGURATION = "The repository returned an unknown project. Please update the repository attributes.";

	private static final int MAX_MARK_STALE_QUERY_HITS = 500;

	private static final boolean TRACE_ENABLED = Boolean.valueOf(Platform.getDebugOption("org.eclipse.mylyn.internal.jira.ui/connector"));

	/** Repository address + Issue Prefix + Issue key = the issue's web address */
	public final static String ISSUE_URL_PREFIX = "/browse/";

	/** Repository address + Filter Prefix + Issue key = the filter's web address */
	public final static String FILTER_URL_PREFIX = "/secure/IssueNavigator.jspa?mode=hide";

	private final JiraTaskDataHandler taskDataHandler;

	private final JiraTaskAttachmentHandler attachmentHandler;

	public static final String UNASSIGNED_USER = "-1";

	/** Name initially given to new tasks. Public for testing */
	public static final String NEW_TASK_DESC = "New Task";

	public static final int RETURN_ALL_HITS = -1;

	public JiraRepositoryConnector() {
		taskDataHandler = new JiraTaskDataHandler(JiraClientFactory.getDefault());
		attachmentHandler = new JiraTaskAttachmentHandler();
	}

	@Override
	public String getLabel() {
		return JiraCorePlugin.LABEL;
	}

	@Override
	public String getConnectorKind() {
		return JiraCorePlugin.CONNECTOR_KIND;
	}

	@Override
	public boolean canCreateTaskFromKey(TaskRepository repository) {
		return true;
	}

	@Override
	public IStatus performQuery(TaskRepository repository, IRepositoryQuery repositoryQuery,
			TaskDataCollector resultCollector, ISynchronizationSession session, IProgressMonitor monitor) {
		monitor = Policy.monitorFor(monitor);
		try {
			monitor.beginTask("Query Repository", IProgressMonitor.UNKNOWN);
			JiraClient client = JiraClientFactory.getDefault().getJiraClient(repository);
			try {
				if (!client.getCache().hasDetails()) {
					client.getCache().refreshDetails(monitor);
				}
			} catch (JiraException e) {
				return JiraCorePlugin.toStatus(repository, e);
			}
			JiraFilter filter = JiraUtil.getQuery(repository, client, repositoryQuery, true);
			if (filter == null) {
				return RepositoryStatus.createStatus(repository, IStatus.ERROR, JiraCorePlugin.ID_PLUGIN,
						"The JIRA query is invalid");
			}
			try {
				QueryHitCollector collector = new QueryHitCollector(repository, client, resultCollector, session,
						monitor);
				client.search(filter, collector, monitor);
				return collector.getStatus();
			} catch (JiraException e) {
				IStatus status = JiraCorePlugin.toStatus(repository, e);
				trace(status);
				return status;
			}
		} finally {
			monitor.done();
		}
	}

	@Override
	public void preSynchronization(ISynchronizationSession session, IProgressMonitor monitor) throws CoreException {
		monitor = Policy.monitorFor(monitor);
		try {
			monitor.beginTask("Getting changed tasks", IProgressMonitor.UNKNOWN);

			session.setNeedsPerformQueries(true);

			if (!session.isFullSynchronization()) {
				return;
			}

			Date now = new Date();
			TaskRepository repository = session.getTaskRepository();
			FilterDefinition changedFilter = getSynchronizationFilter(session, now);
			if (changedFilter == null) {
				// could not determine last time, rerun queries
				repository.setSynchronizationTimeStamp(JiraUtil.dateToString(now));
				return;
			}

			List<JiraIssue> issues = new ArrayList<JiraIssue>();
			JiraClient client = JiraClientFactory.getDefault().getJiraClient(repository);
			// unlimited maxHits can create crazy amounts of traffic
			JiraIssueCollector issueCollector = new JiraIssueCollector(new NullProgressMonitor(), issues,
					MAX_MARK_STALE_QUERY_HITS);
			try {
				client.search(changedFilter, issueCollector, monitor);

				if (issues.isEmpty()) {
					// repository is unchanged
					repository.setSynchronizationTimeStamp(JiraUtil.dateToString(now));
					session.setNeedsPerformQueries(false);
					return;
				}

				HashMap<String, ITask> taskById = new HashMap<String, ITask>();
				for (ITask task : session.getTasks()) {
					taskById.put(task.getTaskId(), task);
				}
				for (JiraIssue issue : issues) {
					ITask task = taskById.get(issue.getId());
					if (task != null) {
						if (issue.getProject() == null) {
							throw new CoreException(new Status(IStatus.ERROR, JiraCorePlugin.ID_PLUGIN, 0,
									ERROR_REPOSITORY_CONFIGURATION, null));
						}

						// for JIRA sufficient information to create task data is returned by the query so no need to mark tasks as stale
						monitor.subTask(issue.getKey() + " " + issue.getSummary());
						// only load old task data from if necessary <commit_msg>NEW - bug 245012: query that checks for changed issues does not respect max search result setting https://bugs.eclipse.org/bugs/show_bug.cgi?id=245012<commit_after>/*******************************************************************************
 * Copyright (c) 2004, 2007 Mylyn project committers and others.
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the Eclipse Public License v1.0
 * which accompanies this distribution, and is available at
 * http://www.eclipse.org/legal/epl-v10.html
 *******************************************************************************/

package org.eclipse.mylyn.internal.jira.core;

import java.util.ArrayList;
import java.util.Collection;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.eclipse.core.runtime.CoreException;
import org.eclipse.core.runtime.IProgressMonitor;
import org.eclipse.core.runtime.IStatus;
import org.eclipse.core.runtime.NullProgressMonitor;
import org.eclipse.core.runtime.Platform;
import org.eclipse.core.runtime.Status;
import org.eclipse.mylyn.commons.net.Policy;
import org.eclipse.mylyn.internal.jira.core.model.JiraFilter;
import org.eclipse.mylyn.internal.jira.core.model.JiraIssue;
import org.eclipse.mylyn.internal.jira.core.model.Priority;
import org.eclipse.mylyn.internal.jira.core.model.Project;
import org.eclipse.mylyn.internal.jira.core.model.filter.DateRangeFilter;
import org.eclipse.mylyn.internal.jira.core.model.filter.FilterDefinition;
import org.eclipse.mylyn.internal.jira.core.model.filter.IssueCollector;
import org.eclipse.mylyn.internal.jira.core.model.filter.Order;
import org.eclipse.mylyn.internal.jira.core.model.filter.RelativeDateRangeFilter;
import org.eclipse.mylyn.internal.jira.core.model.filter.RelativeDateRangeFilter.RangeType;
import org.eclipse.mylyn.internal.jira.core.service.JiraClient;
import org.eclipse.mylyn.internal.jira.core.service.JiraException;
import org.eclipse.mylyn.internal.jira.core.util.JiraUtil;
import org.eclipse.mylyn.tasks.core.AbstractRepositoryConnector;
import org.eclipse.mylyn.tasks.core.IRepositoryQuery;
import org.eclipse.mylyn.tasks.core.ITask;
import org.eclipse.mylyn.tasks.core.RepositoryStatus;
import org.eclipse.mylyn.tasks.core.TaskRepository;
import org.eclipse.mylyn.tasks.core.ITask.PriorityLevel;
import org.eclipse.mylyn.tasks.core.data.TaskAttribute;
import org.eclipse.mylyn.tasks.core.data.TaskData;
import org.eclipse.mylyn.tasks.core.data.TaskDataCollector;
import org.eclipse.mylyn.tasks.core.data.TaskMapper;
import org.eclipse.mylyn.tasks.core.data.TaskRelation;
import org.eclipse.mylyn.tasks.core.sync.ISynchronizationSession;

/**
 * @author <NAME>
 * @author <NAME>
 * @author <NAME>
 * @since 3.0
 */
public class JiraRepositoryConnector extends AbstractRepositoryConnector {

	private static final String ID_STATUS_RESOLVED = "5";

	private static final String ID_STATUS_CLOSED = "6";

	private static final String ERROR_REPOSITORY_CONFIGURATION = "The repository returned an unknown project. Please update the repository attributes.";

	private static final int MAX_MARK_STALE_QUERY_HITS = 500;

	private static final boolean TRACE_ENABLED = Boolean.valueOf(Platform.getDebugOption("org.eclipse.mylyn.internal.jira.ui/connector"));

	/** Repository address + Issue Prefix + Issue key = the issue's web address */
	public final static String ISSUE_URL_PREFIX = "/browse/";

	/** Repository address + Filter Prefix + Issue key = the filter's web address */
	public final static String FILTER_URL_PREFIX = "/secure/IssueNavigator.jspa?mode=hide";

	private final JiraTaskDataHandler taskDataHandler;

	private final JiraTaskAttachmentHandler attachmentHandler;

	public static final String UNASSIGNED_USER = "-1";

	/** Name initially given to new tasks. Public for testing */
	public static final String NEW_TASK_DESC = "New Task";

	public static final int RETURN_ALL_HITS = -1;

	public JiraRepositoryConnector() {
		taskDataHandler = new JiraTaskDataHandler(JiraClientFactory.getDefault());
		attachmentHandler = new JiraTaskAttachmentHandler();
	}

	@Override
	public String getLabel() {
		return JiraCorePlugin.LABEL;
	}

	@Override
	public String getConnectorKind() {
		return JiraCorePlugin.CONNECTOR_KIND;
	}

	@Override
	public boolean canCreateTaskFromKey(TaskRepository repository) {
		return true;
	}

	@Override
	public IStatus performQuery(TaskRepository repository, IRepositoryQuery repositoryQuery,
			TaskDataCollector resultCollector, ISynchronizationSession session, IProgressMonitor monitor) {
		monitor = Policy.monitorFor(monitor);
		try {
			monitor.beginTask("Query Repository", IProgressMonitor.UNKNOWN);
			JiraClient client = JiraClientFactory.getDefault().getJiraClient(repository);
			try {
				if (!client.getCache().hasDetails()) {
					client.getCache().refreshDetails(monitor);
				}
			} catch (JiraException e) {
				return JiraCorePlugin.toStatus(repository, e);
			}
			JiraFilter filter = JiraUtil.getQuery(repository, client, repositoryQuery, true);
			if (filter == null) {
				return RepositoryStatus.createStatus(repository, IStatus.ERROR, JiraCorePlugin.ID_PLUGIN,
						"The JIRA query is invalid");
			}
			try {
				QueryHitCollector collector = new QueryHitCollector(repository, client, resultCollector, session,
						monitor);
				client.search(filter, collector, monitor);
				return collector.getStatus();
			} catch (JiraException e) {
				IStatus status = JiraCorePlugin.toStatus(repository, e);
				trace(status);
				return status;
			}
		} finally {
			monitor.done();
		}
	}

	@Override
	public void preSynchronization(ISynchronizationSession session, IProgressMonitor monitor) throws CoreException {
		monitor = Policy.monitorFor(monitor);
		try {
			monitor.beginTask("Getting changed tasks", IProgressMonitor.UNKNOWN);

			session.setNeedsPerformQueries(true);

			if (!session.isFullSynchronization()) {
				return;
			}

			Date now = new Date();
			TaskRepository repository = session.getTaskRepository();
			FilterDefinition changedFilter = getSynchronizationFilter(session, now);
			if (changedFilter == null) {
				// could not determine last time, rerun queries
				repository.setSynchronizationTimeStamp(JiraUtil.dateToString(now));
				return;
			}

			List<JiraIssue> issues = new ArrayList<JiraIssue>();
			JiraClient client = JiraClientFactory.getDefault().getJiraClient(repository);
			int maxResults = JiraUtil.getMaxSearchResults(repository);
			if (maxResults <= 0) {
				maxResults = MAX_MARK_STALE_QUERY_HITS;
			} else {
				maxResults = Math.min(MAX_MARK_STALE_QUERY_HITS, maxResults);
			}
			// unlimited maxHits can create crazy amounts of traffic
			JiraIssueCollector issueCollector = new JiraIssueCollector(new NullProgressMonitor(), issues, maxResults);
			try {
				client.search(changedFilter, issueCollector, monitor);

				if (issues.isEmpty()) {
					// repository is unchanged
					repository.setSynchronizationTimeStamp(JiraUtil.dateToString(now));
					session.setNeedsPerformQueries(false);
					return;
				}

				HashMap<String, ITask> taskById = new HashMap<String, ITask>();
				for (ITask task : session.getTasks()) {
					taskById.put(task.getTaskId(), task);
				}
				for (JiraIssue issue : issues) {
					ITask task = taskById.get(issue.getId());
					if (task != null) {
						if (issue.getProject() == null) {
							throw new CoreException(new Status(IStatus.ERROR, JiraCorePlugin.ID_PLUGIN, 0,
									ERROR_REPOSITORY_CONFIGURATION, null));
						}

						// for JIRA sufficient information to create task data is returned by the query so no need to mark tasks as stale
						monitor.subTask(issue.getKey() + " " + issue.getSummary());
						// only load old task data from if necessary 
{'fraction_non_alphanumeric': 0.12013744813278009, 'fraction_numerical': 0.003695539419087137, 'mean_word_length': 4.912226906860866, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 17, 'https://': 1, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5665154', 'n_tokens_mistral': 5003, 'n_tokens_neox': 4457, 'n_words': 1006}