<commit_before>package org.corpus_tools.search.service;

import java.io.File;
import java.util.ArrayList;

import org.corpus_tools.graphannis.API;
import org.corpus_tools.graphannis.API.CorpusStorageManager;
import org.corpus_tools.graphannis.API.CorpusStorageManager.CorpusInfo;
import org.corpus_tools.graphannis.API.GraphUpdate;
import org.corpus_tools.graphannis.API.StringVector;
import org.corpus_tools.graphannis.QueryToJSON;
import org.corpus_tools.graphannis.SaltImport;
import org.corpus_tools.salt.common.SDocumentGraph;
import org.corpus_tools.salt.util.SaltUtil;
import org.eclipse.core.resources.IContainer;
import org.eclipse.core.resources.IFile;
import org.eclipse.core.resources.IProject;
import org.eclipse.core.resources.IResource;
import org.eclipse.core.resources.IWorkspace;
import org.eclipse.core.resources.IWorkspaceRoot;
import org.eclipse.core.resources.ResourcesPlugin;
import org.eclipse.core.runtime.CoreException;
import org.eclipse.core.runtime.IProgressMonitor;
import org.eclipse.core.runtime.IStatus;
import org.eclipse.core.runtime.Status;
import org.eclipse.core.runtime.jobs.Job;
import org.eclipse.e4.core.di.annotations.Creatable;
import org.eclipse.emf.common.util.URI;

import annis.service.objects.Match;
import annis.service.objects.MatchGroup;

@Creatable
public class SearchService {
	
	public static final String IDX_FOLDER = "idx-graphannis";
	
	private final CorpusStorageManager corpusManager;
	
	public SearchService() {
		IWorkspace workspace = ResourcesPlugin.getWorkspace();
		File corpusIndexLocation  = new File(workspace.getRoot().getLocation().toOSString(), IDX_FOLDER);
		this.corpusManager  = new CorpusStorageManager(corpusIndexLocation.getAbsolutePath());
	}
	
	private void handleResource(IResource res, String projectName, IProgressMonitor monitor) throws CoreException {
		if(res instanceof IFile) {
			IFile file = (IFile) res;
			if("salt".equals(file.getFileExtension()) && !"saltProject.salt".equals(file.getName())) {
				URI location = URI.createURI(file.getLocationURI().toASCIIString());

				monitor.setTaskName("Importing document " + location.lastSegment());
				SDocumentGraph docGraph = SaltUtil.loadDocumentGraph(location);
				
				addDocument(projectName, docGraph);
			} 
		} else if (res instanceof IContainer ){
			for (IResource child : ((IContainer) res).members()) {
				handleResource(child, projectName, monitor);
			}
		}
	}
	
	public void reindexAllDocuments(boolean blockUI) {
		Job job = new Job("Re-indexing documents") {
			
			@Override
			protected IStatus run(IProgressMonitor monitor) {
				
				
				// get all documents of workspace
				IWorkspace workspace = ResourcesPlugin.getWorkspace();
				IWorkspaceRoot root = workspace.getRoot();
				for(IProject p : root.getProjects()) {<commit_msg>Check if index job was cancelled.<commit_after>package org.corpus_tools.search.service;

import java.io.File;
import java.util.ArrayList;

import org.corpus_tools.graphannis.API;
import org.corpus_tools.graphannis.API.CorpusStorageManager;
import org.corpus_tools.graphannis.API.CorpusStorageManager.CorpusInfo;
import org.corpus_tools.graphannis.API.GraphUpdate;
import org.corpus_tools.graphannis.API.StringVector;
import org.corpus_tools.graphannis.QueryToJSON;
import org.corpus_tools.graphannis.SaltImport;
import org.corpus_tools.salt.common.SDocumentGraph;
import org.corpus_tools.salt.util.SaltUtil;
import org.eclipse.core.resources.IContainer;
import org.eclipse.core.resources.IFile;
import org.eclipse.core.resources.IProject;
import org.eclipse.core.resources.IResource;
import org.eclipse.core.resources.IWorkspace;
import org.eclipse.core.resources.IWorkspaceRoot;
import org.eclipse.core.resources.ResourcesPlugin;
import org.eclipse.core.runtime.CoreException;
import org.eclipse.core.runtime.IProgressMonitor;
import org.eclipse.core.runtime.IStatus;
import org.eclipse.core.runtime.Status;
import org.eclipse.core.runtime.jobs.Job;
import org.eclipse.e4.core.di.annotations.Creatable;
import org.eclipse.emf.common.util.URI;

import annis.service.objects.Match;
import annis.service.objects.MatchGroup;

@Creatable
public class SearchService {
	
	public static final String IDX_FOLDER = "idx-graphannis";
	
	private final CorpusStorageManager corpusManager;
	
	public SearchService() {
		IWorkspace workspace = ResourcesPlugin.getWorkspace();
		File corpusIndexLocation  = new File(workspace.getRoot().getLocation().toOSString(), IDX_FOLDER);
		this.corpusManager  = new CorpusStorageManager(corpusIndexLocation.getAbsolutePath());
	}
	
	private void handleResource(IResource res, String projectName, IProgressMonitor monitor) throws CoreException {
		
		if(monitor.isCanceled()) {
			return;
		}
		
		if(res instanceof IFile) {
			IFile file = (IFile) res;
			if("salt".equals(file.getFileExtension()) && !"saltProject.salt".equals(file.getName())) {
				URI location = URI.createURI(file.getLocationURI().toASCIIString());

				monitor.setTaskName("Importing document " + location.lastSegment());
				SDocumentGraph docGraph = SaltUtil.loadDocumentGraph(location);
				
				addDocument(projectName, docGraph);
			} 
		} else if (res instanceof IContainer ){
			for (IResource child : ((IContainer) res).members()) {
				handleResource(child, projectName, monitor);
			}
		}
	}
	
	public void reindexAllDocuments(boolean blockUI) {
		Job job = new Job("Re-indexing documents") {
			
			@Override
			protected IStatus run(IProgressMonitor monitor) {
				
				
				// get all documents of workspace
				IWorkspace workspace = ResourcesPlugin.getWorkspace();
				IWorkspaceRoot root = workspace.getRoot();
				for(IProject p : root.getProjects()) {
{'fraction_non_alphanumeric': 0.10724381625441697, 'fraction_numerical': 0.00035335689045936394, 'mean_word_length': 5.353535353535354, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5112470', 'n_tokens_mistral': 1854, 'n_tokens_neox': 1688, 'n_words': 315}