<commit_before>package fr.univnantes.termsuite.ui.services.impl;

import java.io.File;
import java.io.FileFilter;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import javax.annotation.PostConstruct;
import javax.inject.Inject;
import javax.inject.Named;
import javax.inject.Singleton;

import org.eclipse.core.runtime.jobs.Job;
import org.eclipse.core.runtime.preferences.IEclipsePreferences;
import org.eclipse.core.runtime.preferences.InstanceScope;
import org.eclipse.e4.core.contexts.IEclipseContext;
import org.eclipse.e4.core.services.events.IEventBroker;
import org.eclipse.e4.core.services.log.ILoggerProvider;
import org.eclipse.e4.core.services.log.Logger;
import org.eclipse.e4.ui.di.UISynchronize;
import org.eclipse.e4.ui.services.IServiceConstants;
import org.eclipse.e4.ui.workbench.modeling.EPartService;
import org.eclipse.e4.ui.workbench.modeling.EPartService.PartState;
import org.eclipse.emf.ecore.resource.Resource;
import org.eclipse.emf.ecore.xmi.impl.XMIResourceFactoryImpl;
import org.eclipse.jface.dialogs.MessageDialog;
import org.eclipse.swt.widgets.Shell;

import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.primitives.Ints;

import eu.project.ttc.engines.cleaner.TermProperty;
import eu.project.ttc.engines.desc.Lang;
import eu.project.ttc.engines.desc.TermSuiteCollection;
import eu.project.ttc.models.Document;
import eu.project.ttc.models.OccurrenceType;
import eu.project.ttc.models.TermIndex;
import eu.project.ttc.models.TermOccurrence;
import eu.project.ttc.tools.TermSuitePipeline;
import fr.univnantes.termsuite.ui.TermSuiteEvents;
import fr.univnantes.termsuite.ui.TermSuiteUI;
import fr.univnantes.termsuite.ui.TermSuiteUIPreferences;
import fr.univnantes.termsuite.ui.dialogs.CorpusSelectionDialog;
import fr.univnantes.termsuite.ui.jobs.RunPipelineJob;
import fr.univnantes.termsuite.ui.model.termsuiteui.ECollectionType;
import fr.univnantes.termsuite.ui.model.termsuiteui.ECorporaList;
import fr.univnantes.termsuite.ui.model.termsuiteui.ECorpus;
import fr.univnantes.termsuite.ui.model.termsuiteui.EDocument;
import fr.univnantes.termsuite.ui.model.termsuiteui.ELang;
import fr.univnantes.termsuite.ui.model.termsuiteui.EPipeline;
import fr.univnantes.termsuite.ui.model.termsuiteui.ESingleLanguageCorpus;
import fr.univnantes.termsuite.ui.model.termsuiteui.ETagger;
import fr.univnantes.termsuite.ui.model.termsuiteui.ETaggerConfig;
import fr.univnantes.termsuite.ui.model.termsuiteui.ETerminology;
import fr.univnantes.termsuite.ui.model.termsuiteui.TermsuiteuiFactory;
import fr.univnantes.termsuite.ui.services.CorpusService;
import fr.univnantes.termsuite.ui.services.LinguisticResourcesService;
import fr.univnantes.termsuite.ui.services.TaggerService;
import fr.univnantes.termsuite.ui.services.TerminoService;
import fr.univnantes.termsuite.ui.util.FileUtil;
import fr.univnantes.termsuite.ui.util.LangUtil;
import fr.univnantes.termsuite.ui.util.WorkspaceUtil;


@Singleton
@SuppressWarnings("restriction")
public class CorpusServiceImpl implements CorpusService {

	@Inject
	@Named(IServiceConstants.ACTIVE_SHELL) Shell activeShell;

	@Inject UISynchronize sync;

	@Inject 
	IEventBroker eventBroker;
	
	@Inject 
	IEclipseContext context;
	
	@Inject EPartService partService;
	
	@Inject
	@Named(IServiceConstants.ACTIVE_SHELL) 
	private Shell parent;
	
	private Logger logger;
	
	private Map<String, EDocument> documentCache = null;
	
	
	private static final String CAS_JSON_DIR = "json";
	private static final String CAS_XMI_DIR = "xmi";
	private static final String CAS_TSV_DIR = "tsv";
	
	private static final String JSON_EXTENSION = "json";
	private static final String TBX_EXTENSION = "tbx";
	private static final String TSV_EXTENSION = "tsv";

	
	private ECorporaList corpora = TermsuiteuiFactory.eINSTANCE.createECorporaList();

	public CorpusServiceImpl() {
		Map<String, Object> m = Resource.Factory.Registry.INSTANCE.getExtensionToFactoryMap();
		// Register the XMI resource factory for the .pipeline extension
		m.put(CORPUS_EXTENSION, new XMIResourceFactoryImpl());
	
		loadCorpora();
	}
	
	@PostConstruct
	public void init(ILoggerProvider loggerProvider) {
		this.logger = loggerProvider.getClassLogger(this.getClass());
		this.logger.debug("Initalizing service " + this.getClass());
	}
	
	
	
	/* (non-Javadoc)
	 * @see fr.univnantes.termsuite.ui.services.CorpusService#getCorporaList()
	 */
	@Override
	public ECorporaList getCorporaList() {
		return corpora;
	}

	/* (non-Javadoc)
	 * @see fr.univnantes.termsuite.ui.services.CorpusService#getDocumentPath(fr.univnantes.termsuite.ui.model.termsuiteui.ESingleLanguageCorpus)
	 */
	@Override
	public Path getDocumentPath(ESingleLanguageCorpus slc) {
		return Paths.get(getPath(slc).toString(), slc.getCollectionType().toString().toLowerCase());
	}
	
	/* (non-Javadoc)
	 * @see fr.univnantes.termsuite.ui.services.CorpusService#getPath(fr.univnantes.termsuite.ui.model.termsuiteui.ESingleLanguageCorpus)
	 */
	@Override
	public Path getPath(ESingleLanguageCorpus slc) {
		Lang termsuiteLang = LangUtil.getTermsuiteLang(slc.getLanguage());
		return Paths.get(slc.getCorpus().getPath(), termsuiteLang.getNameUC());
	}
	
	/* (non-Javadoc)
	 * @see fr.univnantes.termsuite.ui.services.CorpusService#getPath(fr.univnantes.termsuite.ui.model.termsuiteui.EDocument)
	 */
	@Override
	public Path getPath(EDocument d) {
		return Paths.get(getDocumentPath(d.getSingleLanguageCorpus()).toString(), d.getFilename());
	}

	/* (non-Javadoc)
	 * @see fr.univnantes.termsuite.ui.services.CorpusService#asFile(fr.univnantes.termsuite.ui.model.termsuiteui.EDocument)
	 */
	@Override
	public File asFile(EDocument d) {
		return getPath(d).toFile();
	}

	/* (non-Javadoc)
	 * @see fr.univnantes.termsuite.ui.services.CorpusService#createCorpus(java.lang.String)
	 */
	@Override
	public ECorpus createCorpus(String name, String corpusPath) {
		ECorpus corpus = TermsuiteuiFactory.eINSTANCE.createECorpus();
		corpus.setName(name);
		corpus.setPath(Paths.get(corpusPath).toString());
		for(File sl:candidateSLCChildrenForPath(corpus.getPath())) {
			ECollectionType type = Files.exists(Paths.get(sl.getAbsolutePath(), "tei")) ?
					ECollectionType.TEI:
							ECollectionType.TXT;
								
			ESingleLanguageCorpus slc = TermsuiteuiFactory.eINSTANCE.createESingleLanguageCorpus();
			slc.setCorpus(corpus);
			slc.setLanguage(LangUtil.getLangByNameUC(sl.getName()));
			slc.setCollectionType(type);
			
			for(File f:getDocumentPath(slc).toFile().listFiles()) {
				EDocument doc = TermsuiteuiFactory.eINSTANCE.createEDocument();
				doc.setFilename(f.getName());
				slc.getDocuments().add(doc);
			}
			corpus.getSingleLanguageCorpora().add(slc);
		}
		corpora.getCorpora().add(corpus);
		resetCache();
		try {
			saveCorpus(corpus);
		} catch (IOException e) {
			e.printStackTrace();
		}
		return corpus;
	}

	@Override
	public File[] candidateSLCChildrenForPath(String path) {
		return new File(path).listFiles(new FileFilter() {
				
				@Override
				public boolean accept(File child) {
					if(!child.isDirectory())
						return false;
					for(ELang l:SUPPORTED_LANGUAGES) {
						if(LangUtil.getTermsuiteLang(l).getNameUC().equals(child.getName())) 
							return true;
					}
					return false;
				}
			});
	}

	
	/* (non-Javadoc)
	 * @see fr.univnantes.termsuite.ui.services.CorpusService#getCollectionType(fr.univnantes.termsuite.ui.model.termsuiteui.ESingleLanguageCorpus)
	 */
	@Override
	public TermSuiteCollection getCollectionType(ESingleLanguageCorpus corpus) {
		switch (corpus.getCollectionType()) {
		case TEI:
			return TermSuiteCollection.TEI;
		default:
			return TermSuiteCollection.TXT;
		}
	}
	
	/* (non-Javadoc)
	 * @see fr.univnantes.termsuite.ui.services.CorpusService#getCollectionPath(fr.univnantes.termsuite.ui.model.termsuiteui.ESingleLanguageCorpus)
	 */
	@Override
	public String getCollectionPath(ESingleLanguageCorpus corpus) {
		return Paths.get(
				corpus.getCorpus().getPath(), 
				LangUtil.getTermsuiteLang(corpus.getLanguage()).getNameUC(),
				getCollectionType(corpus).toString().toLowerCase()
				).toString();
	}


	@Override
	public void runPipelineOnCorpus(EPipeline pipeline, ESingleLanguageCorpus corpus) {
		List<ESingleLanguageCorpus> l = Lists.newArrayList();
		l.add(corpus);
		runPipelineOnCorpus(pipeline, l);
	}



	@Override
	public void runPipelineOnSeveralCorpus(EPipeline pipeline, Iterable<ESingleLanguageCorpus> corpusList) {
		runPipelineOnCorpus(pipeline, corpusList);		
	}
	
	
	/* (non-Javadoc)
	 * @see fr.univnantes.termsuite.ui.services.CorpusService#runPipelineOnCorpus(fr.univnantes.termsuite.ui.model.termsuiteui.EPipeline, fr.univnantes.termsuite.ui.model.termsuiteui.ESingleLanguageCorpus)
	 */
	private void runPipelineOnCorpus(final EPipeline pipeline, final Iterable<ESingleLanguageCorpus> corpusList) {
		LinguisticResourcesService linguisticResourcesService = context.get(LinguisticResourcesService.class);
		String resourcePrefix = String.format(
				"platform:/plugin/%s/", 
				linguisticResourcesService.getLinguisticResourceBundle().getSymbolicName());

		for(final ESingleLanguageCorpus corpus:corpusList) {
			final TermSuitePipeline tsp = toTermSuitePipeline(pipeline, corpus, resourcePrefix);
	
			String jobName = "Running pipeline " + pipeline.getFilename() + " on corpus " + corpus.getCorpus().getPath() + "/" + corpus.getLanguage().getName();
			
			Job job = new RunPipelineJob(jobName, tsp, sync) {
				@Override
				public void terminologyCreated(final TermIndex termIndex) {
					sync.asyncExec(new Runnable() {
						@Override
						public void run() {
							ETerminology terminology = context.get(TerminoService.class).createTerminology(
									corpus, 
									pipeline.getTargetTerminologyName(), 
									getTerminoJsonFile(corpus, pipeline),
									pipeline.isSpotWithOccurrences(),
									pipeline.isContextualizerEnabled());
							try {
								saveCorpus(corpus.getCorpus());
								eventBroker.post(TermSuiteEvents.NEW_TERMINOLOGY, terminology);
							} catch(IOException e) {
								MessageDialog.openError(activeShell, "Error saving terminology", e.getMessage());
							}
						}
					});
				}
			};
			job.schedule();
			sync.asyncExec(new Runnable() {
				@Override
				public void run() {
					partService.showPart(TermSuiteUI.PROGRESS_VIEW_ID, PartState.VISIBLE);
				}
			});
		}

	}



	private TermSuitePipeline toTermSuitePipeline(EPipeline pipeline, ESingleLanguageCorpus corpus, String resourcePrefix) {
		TaggerService taggerService = context.get(TaggerService.class);
		
		Lang tsLang = LangUtil.getTermsuiteLang(corpus.getLanguage());
		
		TermSuitePipeline tsp = TermSuitePipeline.create(tsLang.getCode(), resourcePrefix);
		
		tsp
			.setCollection(
					this.getCollectionType(corpus), 
					this.getCollectionPath(corpus), 
					"UTF-8");
		
		tsp.aeWordTokenizer();
		
		ETaggerConfig taggerConfig = taggerService.getTaggerConfig(pipeline);
		if(taggerConfig.getTaggerType() == ETagger.TREE_TAGGER) {
			tsp.setTreeTaggerHome(taggerConfig.getPath())
				.aeTreeTagger();
		} else if(taggerConfig.getTaggerType() == ETagger.MATE) {
			tsp.setMateModelPath(taggerConfig.getPath())
				.aeMateTaggerLemmatizer();
		}
		
		tsp.aeUrlFilter()
			.aeStemmer()
			.setSpotWithOccurrences(pipeline.isSpotWithOccurrences())
			.aeRegexSpotter()
			.aeSpecificityComputer();
			
		if(pipeline.isMorphosyntacticAnalysisEnabled()) {
			tsp.aeCompostSplitter();
		}

		if(pipeline.isSyntacticVariationEnabled()) {
			tsp.aeSyntacticVariantGatherer();
		}

		if(pipeline.isGraphicalVariationAnalysisEnabled()) {
			tsp.setGraphicalVariantSimilarityThreshold((float)pipeline.getGraphicalSimilarityThreshhold())
				.aeGraphicalVariantGatherer();
		}
		
		if(pipeline.isContextualizerEnabled()) {
			tsp.setContextualizeCoTermsType(pipeline.isContextAllowMWTAsCooc() ? OccurrenceType.ALL : OccurrenceType.SINGLE_WORD)
				/*
				 * TODO Set as GUI param
				 */
			   .setContextualizeWithCoOccurrenceFrequencyThreshhold(2)
			   .aeContextualizer(pipeline.getContextScope(), !pipeline.isContextualizeOnSWTOnly());
		}

		tsp.aeExtensionDetector()
			.aeScorer()
			.aeRanker(TermProperty.SPECIFICITY, true);

		
		
		if(pipeline.isFilteringEnabled()) {
			tsp.haeCasStatCounter("Before Filtering");
			
			TermProperty termProperty = TermProperty.forName(pipeline.getFilteringProperty());
			/*
			 * TODO Set as GUI param
			 */
			tsp.setKeepVariantsWhileCleaning(true); 
			switch(pipeline.getFilteringMode()) {
			case THRESHOLD: 
				tsp.aeThresholdCleaner(termProperty, (float)pipeline.getFilteringThreshold());
				break;
			case TOP_N: 
				tsp.aeTopNCleaner(termProperty, pipeline.getFilteringTopN());
			}
			
			tsp.haeCasStatCounter("After Filtering");

		}
		
		if(pipeline.isBigCorporaHandlingEnabled()) {
			TermProperty termProperty = TermProperty.forName(pipeline.getBigCorporaFilteringProperty());
			switch(pipeline.getBigCorporaCleaningMode()) {
			case DOCUMENT_PERIOD: 
				tsp.aeThresholdCleanerPeriodic(
						termProperty, 
						(float)pipeline.getBigCorporaFilteringPropertyThreshold(), 
						pipeline.getBigCorporaDocumentPeriod());
				break;
			case MAX_NUMBER_OF_TERMS: 
				tsp.aeMaxSizeThresholdCleaner(termProperty, pipeline.getBigCorporaMaxNumberOfTerms());
			}
		}
		
		/* CAS export */
		if(pipeline.isExportCasToJsonEnabled())
			tsp.haeXmiCasExporter(getCasOutputDirectory(corpus, pipeline, CAS_JSON_DIR));
		if(pipeline.isExportCasToTsvEnabled())
			tsp.haeSpotterTSVWriter(getCasOutputDirectory(corpus, pipeline, CAS_TSV_DIR));
		if(pipeline.isExportCasToXmiEnabled())
			tsp.haeJsonExporter(getCasOutputDirectory(corpus, pipeline, CAS_XMI_DIR));

		/* termino export */
		if(pipeline.isExportTerminoToJsonEnabled()) {
			tsp.setExportJsonWithContext(true);
			tsp.setExportJsonWithOccurrences(true);
			tsp.haeJsonExporter(getTerminoOutputDirectory(corpus, pipeline, JSON_EXTENSION));
		}
		if(pipeline.isExportTerminoToTbxEnabled())
			tsp.haeTbxExporter(getTerminoOutputDirectory(corpus, pipeline, TBX_EXTENSION));
		if(pipeline.isExportTerminoToTsvEnabled())
			/*
			 * TODO Set as GUI params
			 */
			tsp.setTsvShowHeaders(true)
			   .setTsvShowScores(false)
			   .setTsvExportProperties(TermProperty.PATTERN, TermProperty.PILOT, TermProperty.FREQUENCY, TermProperty.WR_LOG)
			   .haeTsvExporter(getTerminoOutputDirectory(corpus, pipeline, TSV_EXTENSION));
		return tsp;
	}

	private void loadCorpora() {
		List<ECorpus> list = WorkspaceUtil.loadResources(CORPUS_DIR, CORPUS_EXTENSION, ECorpus.class);
		corpora.getCorpora().addAll(list);
	}

	
	@Override
	public void saveCorpus(ECorpus corpus) throws IOException {
		resetCache();
		WorkspaceUtil.saveResource(corpus, CORPUS_DIR, corpus.getName(), CORPUS_EXTENSION);
	}


	@Override
	public String getOutputDirectory(ESingleLanguageCorpus corpus) {
		IEclipsePreferences preferences = InstanceScope.INSTANCE.getNode(TermSuiteUI.PLUGIN_ID);
		String rootOutputDir = preferences.get(
				TermSuiteUIPreferences.OUTPUT_DIRECTORY, 
				WorkspaceUtil.getWorkspacePath( "output").toString());
		Path path = Paths.get(
				rootOutputDir, 
				corpus.getCorpus().getName(), 
				corpus.getLanguage().getName());
		return path.toString();		
	}

	@Override
	public String getOutputDirectory(ESingleLanguageCorpus corpus, EPipeline pipeline) {
		return Paths.get(getOutputDirectory(corpus), 
				pipeline.getTargetTerminologyName()).toString();
	}

	@Override
	public String getTerminoOutputDirectory(ESingleLanguageCorpus corpus, EPipeline pipeline, String extension) {
		Path path = Paths.get(getOutputDirectory(corpus, pipeline), pipeline.getTargetTerminologyName() + "." + extension);
		FileUtil.mkdirs(path.toFile());
		return path.toString();
	}
	
	@Override
	public String getCasOutputDirectory(ESingleLanguageCorpus corpus, EPipeline pipeline, String casDirname) {
		Path path = Paths.get(getOutputDirectory(corpus, pipeline), casDirname);
		FileUtil.mkdirs(path.toFile());
		return path.toString();		
	}



	@Override
	public String getTerminoJsonFile(ESingleLanguageCorpus corpus, EPipeline pipeline) {
		return getTerminoOutputDirectory(corpus, pipeline, JSON_EXTENSION);
	}



	@Override
	public String getTerminoTsvFile(ESingleLanguageCorpus corpus, EPipeline pipeline) {
		return getTerminoOutputDirectory(corpus, pipeline, TSV_EXTENSION);
	}



	@Override
	public String getTerminoTbxFile(ESingleLanguageCorpus corpus, EPipeline pipeline) {
		return getTerminoOutputDirectory(corpus, pipeline, TBX_EXTENSION);
	}



	@Override
	public String getXmiCasOutputDirectory(ESingleLanguageCorpus corpus, EPipeline pipeline) {
		return getCasOutputDirectory(corpus, pipeline, CAS_XMI_DIR);
	}



	@Override
	public String getTsvCasOutputDirectory(ESingleLanguageCorpus corpus, EPipeline pipeline) {
		return getCasOutputDirectory(corpus, pipeline, CAS_TSV_DIR);
	}



	@Override
	public String getJsonCasOutputDirectory(ESingleLanguageCorpus corpus, EPipeline pipeline) {
		return getCasOutputDirectory(corpus, pipeline, CAS_JSON_DIR);
	}



	@Override
	public void removeCorpus(ECorpus s) {
		corpora.getCorpora().remove(s);
		WorkspaceUtil.removeResource(CORPUS_DIR, s.getName(), CORPUS_EXTENSION);
		resetCache();
		eventBroker.post(TermSuiteEvents.CORPUS_REMOVED, s);

	}



	private void resetCache() {
		documentCache = null;
	}

	private Map<String, EDocument> getDocumentCache() {
		if (documentCache == null) {
			final String FILE_PROTOCOL = "file:";
			documentCache = Maps.newHashMap();
			for (ECorpus corpus : getCorporaList().getCorpora()) {
				for (ESingleLanguageCorpus slc : corpus.getSingleLanguageCorpora()) {
					for (EDocument d : slc.getDocuments()) {
						File file = asFile(d);
						documentCache.put(FILE_PROTOCOL + file.getAbsolutePath(), d);
						documentCache.put(FILE_PROTOCOL + file.getPath(), d);
					}
				}
			}
		}
		return documentCache;
	}


	
	@Override
	public EDocument resolveEDocument(Document sourceDocument) {
		EDocument doc = getDocumentCache().get(sourceDocument.getUrl());
		
		if(doc == null)
			this.logger.warn("Could not resolve document with url {}", sourceDocument.getUrl());
		
		return doc;<commit_msg>DEBUG bad use of Eclipse logging message caused a NPE <commit_after>package fr.univnantes.termsuite.ui.services.impl;

import java.io.File;
import java.io.FileFilter;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import javax.annotation.PostConstruct;
import javax.inject.Inject;
import javax.inject.Named;
import javax.inject.Singleton;

import org.eclipse.core.runtime.jobs.Job;
import org.eclipse.core.runtime.preferences.IEclipsePreferences;
import org.eclipse.core.runtime.preferences.InstanceScope;
import org.eclipse.e4.core.contexts.IEclipseContext;
import org.eclipse.e4.core.services.events.IEventBroker;
import org.eclipse.e4.core.services.log.ILoggerProvider;
import org.eclipse.e4.core.services.log.Logger;
import org.eclipse.e4.ui.di.UISynchronize;
import org.eclipse.e4.ui.services.IServiceConstants;
import org.eclipse.e4.ui.workbench.modeling.EPartService;
import org.eclipse.e4.ui.workbench.modeling.EPartService.PartState;
import org.eclipse.emf.ecore.resource.Resource;
import org.eclipse.emf.ecore.xmi.impl.XMIResourceFactoryImpl;
import org.eclipse.jface.dialogs.MessageDialog;
import org.eclipse.swt.widgets.Shell;

import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.primitives.Ints;

import eu.project.ttc.engines.cleaner.TermProperty;
import eu.project.ttc.engines.desc.Lang;
import eu.project.ttc.engines.desc.TermSuiteCollection;
import eu.project.ttc.models.Document;
import eu.project.ttc.models.OccurrenceType;
import eu.project.ttc.models.TermIndex;
import eu.project.ttc.models.TermOccurrence;
import eu.project.ttc.tools.TermSuitePipeline;
import fr.univnantes.termsuite.ui.TermSuiteEvents;
import fr.univnantes.termsuite.ui.TermSuiteUI;
import fr.univnantes.termsuite.ui.TermSuiteUIPreferences;
import fr.univnantes.termsuite.ui.dialogs.CorpusSelectionDialog;
import fr.univnantes.termsuite.ui.jobs.RunPipelineJob;
import fr.univnantes.termsuite.ui.model.termsuiteui.ECollectionType;
import fr.univnantes.termsuite.ui.model.termsuiteui.ECorporaList;
import fr.univnantes.termsuite.ui.model.termsuiteui.ECorpus;
import fr.univnantes.termsuite.ui.model.termsuiteui.EDocument;
import fr.univnantes.termsuite.ui.model.termsuiteui.ELang;
import fr.univnantes.termsuite.ui.model.termsuiteui.EPipeline;
import fr.univnantes.termsuite.ui.model.termsuiteui.ESingleLanguageCorpus;
import fr.univnantes.termsuite.ui.model.termsuiteui.ETagger;
import fr.univnantes.termsuite.ui.model.termsuiteui.ETaggerConfig;
import fr.univnantes.termsuite.ui.model.termsuiteui.ETerminology;
import fr.univnantes.termsuite.ui.model.termsuiteui.TermsuiteuiFactory;
import fr.univnantes.termsuite.ui.services.CorpusService;
import fr.univnantes.termsuite.ui.services.LinguisticResourcesService;
import fr.univnantes.termsuite.ui.services.TaggerService;
import fr.univnantes.termsuite.ui.services.TerminoService;
import fr.univnantes.termsuite.ui.util.FileUtil;
import fr.univnantes.termsuite.ui.util.LangUtil;
import fr.univnantes.termsuite.ui.util.WorkspaceUtil;


@Singleton
@SuppressWarnings("restriction")
public class CorpusServiceImpl implements CorpusService {

	@Inject
	@Named(IServiceConstants.ACTIVE_SHELL) Shell activeShell;

	@Inject UISynchronize sync;

	@Inject 
	IEventBroker eventBroker;
	
	@Inject 
	IEclipseContext context;
	
	@Inject EPartService partService;
	
	@Inject
	@Named(IServiceConstants.ACTIVE_SHELL) 
	private Shell parent;
	
	private Logger logger;
	
	private Map<String, EDocument> documentCache = null;
	
	
	private static final String CAS_JSON_DIR = "json";
	private static final String CAS_XMI_DIR = "xmi";
	private static final String CAS_TSV_DIR = "tsv";
	
	private static final String JSON_EXTENSION = "json";
	private static final String TBX_EXTENSION = "tbx";
	private static final String TSV_EXTENSION = "tsv";

	
	private ECorporaList corpora = TermsuiteuiFactory.eINSTANCE.createECorporaList();

	public CorpusServiceImpl() {
		Map<String, Object> m = Resource.Factory.Registry.INSTANCE.getExtensionToFactoryMap();
		// Register the XMI resource factory for the .pipeline extension
		m.put(CORPUS_EXTENSION, new XMIResourceFactoryImpl());
	
		loadCorpora();
	}
	
	@PostConstruct
	public void init(ILoggerProvider loggerProvider) {
		this.logger = loggerProvider.getClassLogger(this.getClass());
		this.logger.debug("Initalizing service " + this.getClass());
	}
	
	
	
	/* (non-Javadoc)
	 * @see fr.univnantes.termsuite.ui.services.CorpusService#getCorporaList()
	 */
	@Override
	public ECorporaList getCorporaList() {
		return corpora;
	}

	/* (non-Javadoc)
	 * @see fr.univnantes.termsuite.ui.services.CorpusService#getDocumentPath(fr.univnantes.termsuite.ui.model.termsuiteui.ESingleLanguageCorpus)
	 */
	@Override
	public Path getDocumentPath(ESingleLanguageCorpus slc) {
		return Paths.get(getPath(slc).toString(), slc.getCollectionType().toString().toLowerCase());
	}
	
	/* (non-Javadoc)
	 * @see fr.univnantes.termsuite.ui.services.CorpusService#getPath(fr.univnantes.termsuite.ui.model.termsuiteui.ESingleLanguageCorpus)
	 */
	@Override
	public Path getPath(ESingleLanguageCorpus slc) {
		Lang termsuiteLang = LangUtil.getTermsuiteLang(slc.getLanguage());
		return Paths.get(slc.getCorpus().getPath(), termsuiteLang.getNameUC());
	}
	
	/* (non-Javadoc)
	 * @see fr.univnantes.termsuite.ui.services.CorpusService#getPath(fr.univnantes.termsuite.ui.model.termsuiteui.EDocument)
	 */
	@Override
	public Path getPath(EDocument d) {
		return Paths.get(getDocumentPath(d.getSingleLanguageCorpus()).toString(), d.getFilename());
	}

	/* (non-Javadoc)
	 * @see fr.univnantes.termsuite.ui.services.CorpusService#asFile(fr.univnantes.termsuite.ui.model.termsuiteui.EDocument)
	 */
	@Override
	public File asFile(EDocument d) {
		return getPath(d).toFile();
	}

	/* (non-Javadoc)
	 * @see fr.univnantes.termsuite.ui.services.CorpusService#createCorpus(java.lang.String)
	 */
	@Override
	public ECorpus createCorpus(String name, String corpusPath) {
		ECorpus corpus = TermsuiteuiFactory.eINSTANCE.createECorpus();
		corpus.setName(name);
		corpus.setPath(Paths.get(corpusPath).toString());
		for(File sl:candidateSLCChildrenForPath(corpus.getPath())) {
			ECollectionType type = Files.exists(Paths.get(sl.getAbsolutePath(), "tei")) ?
					ECollectionType.TEI:
							ECollectionType.TXT;
								
			ESingleLanguageCorpus slc = TermsuiteuiFactory.eINSTANCE.createESingleLanguageCorpus();
			slc.setCorpus(corpus);
			slc.setLanguage(LangUtil.getLangByNameUC(sl.getName()));
			slc.setCollectionType(type);
			
			for(File f:getDocumentPath(slc).toFile().listFiles()) {
				EDocument doc = TermsuiteuiFactory.eINSTANCE.createEDocument();
				doc.setFilename(f.getName());
				slc.getDocuments().add(doc);
			}
			corpus.getSingleLanguageCorpora().add(slc);
		}
		corpora.getCorpora().add(corpus);
		resetCache();
		try {
			saveCorpus(corpus);
		} catch (IOException e) {
			e.printStackTrace();
		}
		return corpus;
	}

	@Override
	public File[] candidateSLCChildrenForPath(String path) {
		return new File(path).listFiles(new FileFilter() {
				
				@Override
				public boolean accept(File child) {
					if(!child.isDirectory())
						return false;
					for(ELang l:SUPPORTED_LANGUAGES) {
						if(LangUtil.getTermsuiteLang(l).getNameUC().equals(child.getName())) 
							return true;
					}
					return false;
				}
			});
	}

	
	/* (non-Javadoc)
	 * @see fr.univnantes.termsuite.ui.services.CorpusService#getCollectionType(fr.univnantes.termsuite.ui.model.termsuiteui.ESingleLanguageCorpus)
	 */
	@Override
	public TermSuiteCollection getCollectionType(ESingleLanguageCorpus corpus) {
		switch (corpus.getCollectionType()) {
		case TEI:
			return TermSuiteCollection.TEI;
		default:
			return TermSuiteCollection.TXT;
		}
	}
	
	/* (non-Javadoc)
	 * @see fr.univnantes.termsuite.ui.services.CorpusService#getCollectionPath(fr.univnantes.termsuite.ui.model.termsuiteui.ESingleLanguageCorpus)
	 */
	@Override
	public String getCollectionPath(ESingleLanguageCorpus corpus) {
		return Paths.get(
				corpus.getCorpus().getPath(), 
				LangUtil.getTermsuiteLang(corpus.getLanguage()).getNameUC(),
				getCollectionType(corpus).toString().toLowerCase()
				).toString();
	}


	@Override
	public void runPipelineOnCorpus(EPipeline pipeline, ESingleLanguageCorpus corpus) {
		List<ESingleLanguageCorpus> l = Lists.newArrayList();
		l.add(corpus);
		runPipelineOnCorpus(pipeline, l);
	}



	@Override
	public void runPipelineOnSeveralCorpus(EPipeline pipeline, Iterable<ESingleLanguageCorpus> corpusList) {
		runPipelineOnCorpus(pipeline, corpusList);		
	}
	
	
	/* (non-Javadoc)
	 * @see fr.univnantes.termsuite.ui.services.CorpusService#runPipelineOnCorpus(fr.univnantes.termsuite.ui.model.termsuiteui.EPipeline, fr.univnantes.termsuite.ui.model.termsuiteui.ESingleLanguageCorpus)
	 */
	private void runPipelineOnCorpus(final EPipeline pipeline, final Iterable<ESingleLanguageCorpus> corpusList) {
		LinguisticResourcesService linguisticResourcesService = context.get(LinguisticResourcesService.class);
		String resourcePrefix = String.format(
				"platform:/plugin/%s/", 
				linguisticResourcesService.getLinguisticResourceBundle().getSymbolicName());

		for(final ESingleLanguageCorpus corpus:corpusList) {
			final TermSuitePipeline tsp = toTermSuitePipeline(pipeline, corpus, resourcePrefix);
	
			String jobName = "Running pipeline " + pipeline.getFilename() + " on corpus " + corpus.getCorpus().getPath() + "/" + corpus.getLanguage().getName();
			
			Job job = new RunPipelineJob(jobName, tsp, sync) {
				@Override
				public void terminologyCreated(final TermIndex termIndex) {
					sync.asyncExec(new Runnable() {
						@Override
						public void run() {
							ETerminology terminology = context.get(TerminoService.class).createTerminology(
									corpus, 
									pipeline.getTargetTerminologyName(), 
									getTerminoJsonFile(corpus, pipeline),
									pipeline.isSpotWithOccurrences(),
									pipeline.isContextualizerEnabled());
							try {
								saveCorpus(corpus.getCorpus());
								eventBroker.post(TermSuiteEvents.NEW_TERMINOLOGY, terminology);
							} catch(IOException e) {
								MessageDialog.openError(activeShell, "Error saving terminology", e.getMessage());
							}
						}
					});
				}
			};
			job.schedule();
			sync.asyncExec(new Runnable() {
				@Override
				public void run() {
					partService.showPart(TermSuiteUI.PROGRESS_VIEW_ID, PartState.VISIBLE);
				}
			});
		}

	}



	private TermSuitePipeline toTermSuitePipeline(EPipeline pipeline, ESingleLanguageCorpus corpus, String resourcePrefix) {
		TaggerService taggerService = context.get(TaggerService.class);
		
		Lang tsLang = LangUtil.getTermsuiteLang(corpus.getLanguage());
		
		TermSuitePipeline tsp = TermSuitePipeline.create(tsLang.getCode(), resourcePrefix);
		
		tsp
			.setCollection(
					this.getCollectionType(corpus), 
					this.getCollectionPath(corpus), 
					"UTF-8");
		
		tsp.aeWordTokenizer();
		
		ETaggerConfig taggerConfig = taggerService.getTaggerConfig(pipeline);
		if(taggerConfig.getTaggerType() == ETagger.TREE_TAGGER) {
			tsp.setTreeTaggerHome(taggerConfig.getPath())
				.aeTreeTagger();
		} else if(taggerConfig.getTaggerType() == ETagger.MATE) {
			tsp.setMateModelPath(taggerConfig.getPath())
				.aeMateTaggerLemmatizer();
		}
		
		tsp.aeUrlFilter()
			.aeStemmer()
			.setSpotWithOccurrences(pipeline.isSpotWithOccurrences())
			.aeRegexSpotter()
			.aeSpecificityComputer();
			
		if(pipeline.isMorphosyntacticAnalysisEnabled()) {
			tsp.aeCompostSplitter();
		}

		if(pipeline.isSyntacticVariationEnabled()) {
			tsp.aeSyntacticVariantGatherer();
		}

		if(pipeline.isGraphicalVariationAnalysisEnabled()) {
			tsp.setGraphicalVariantSimilarityThreshold((float)pipeline.getGraphicalSimilarityThreshhold())
				.aeGraphicalVariantGatherer();
		}
		
		if(pipeline.isContextualizerEnabled()) {
			tsp.setContextualizeCoTermsType(pipeline.isContextAllowMWTAsCooc() ? OccurrenceType.ALL : OccurrenceType.SINGLE_WORD)
				/*
				 * TODO Set as GUI param
				 */
			   .setContextualizeWithCoOccurrenceFrequencyThreshhold(2)
			   .aeContextualizer(pipeline.getContextScope(), !pipeline.isContextualizeOnSWTOnly());
		}

		tsp.aeExtensionDetector()
			.aeScorer()
			.aeRanker(TermProperty.SPECIFICITY, true);

		
		
		if(pipeline.isFilteringEnabled()) {
			tsp.haeCasStatCounter("Before Filtering");
			
			TermProperty termProperty = TermProperty.forName(pipeline.getFilteringProperty());
			/*
			 * TODO Set as GUI param
			 */
			tsp.setKeepVariantsWhileCleaning(true); 
			switch(pipeline.getFilteringMode()) {
			case THRESHOLD: 
				tsp.aeThresholdCleaner(termProperty, (float)pipeline.getFilteringThreshold());
				break;
			case TOP_N: 
				tsp.aeTopNCleaner(termProperty, pipeline.getFilteringTopN());
			}
			
			tsp.haeCasStatCounter("After Filtering");

		}
		
		if(pipeline.isBigCorporaHandlingEnabled()) {
			TermProperty termProperty = TermProperty.forName(pipeline.getBigCorporaFilteringProperty());
			switch(pipeline.getBigCorporaCleaningMode()) {
			case DOCUMENT_PERIOD: 
				tsp.aeThresholdCleanerPeriodic(
						termProperty, 
						(float)pipeline.getBigCorporaFilteringPropertyThreshold(), 
						pipeline.getBigCorporaDocumentPeriod());
				break;
			case MAX_NUMBER_OF_TERMS: 
				tsp.aeMaxSizeThresholdCleaner(termProperty, pipeline.getBigCorporaMaxNumberOfTerms());
			}
		}
		
		/* CAS export */
		if(pipeline.isExportCasToJsonEnabled())
			tsp.haeXmiCasExporter(getCasOutputDirectory(corpus, pipeline, CAS_JSON_DIR));
		if(pipeline.isExportCasToTsvEnabled())
			tsp.haeSpotterTSVWriter(getCasOutputDirectory(corpus, pipeline, CAS_TSV_DIR));
		if(pipeline.isExportCasToXmiEnabled())
			tsp.haeJsonExporter(getCasOutputDirectory(corpus, pipeline, CAS_XMI_DIR));

		/* termino export */
		if(pipeline.isExportTerminoToJsonEnabled()) {
			tsp.setExportJsonWithContext(true);
			tsp.setExportJsonWithOccurrences(true);
			tsp.haeJsonExporter(getTerminoOutputDirectory(corpus, pipeline, JSON_EXTENSION));
		}
		if(pipeline.isExportTerminoToTbxEnabled())
			tsp.haeTbxExporter(getTerminoOutputDirectory(corpus, pipeline, TBX_EXTENSION));
		if(pipeline.isExportTerminoToTsvEnabled())
			/*
			 * TODO Set as GUI params
			 */
			tsp.setTsvShowHeaders(true)
			   .setTsvShowScores(false)
			   .setTsvExportProperties(TermProperty.PATTERN, TermProperty.PILOT, TermProperty.FREQUENCY, TermProperty.WR_LOG)
			   .haeTsvExporter(getTerminoOutputDirectory(corpus, pipeline, TSV_EXTENSION));
		return tsp;
	}

	private void loadCorpora() {
		List<ECorpus> list = WorkspaceUtil.loadResources(CORPUS_DIR, CORPUS_EXTENSION, ECorpus.class);
		corpora.getCorpora().addAll(list);
	}

	
	@Override
	public void saveCorpus(ECorpus corpus) throws IOException {
		resetCache();
		WorkspaceUtil.saveResource(corpus, CORPUS_DIR, corpus.getName(), CORPUS_EXTENSION);
	}


	@Override
	public String getOutputDirectory(ESingleLanguageCorpus corpus) {
		IEclipsePreferences preferences = InstanceScope.INSTANCE.getNode(TermSuiteUI.PLUGIN_ID);
		String rootOutputDir = preferences.get(
				TermSuiteUIPreferences.OUTPUT_DIRECTORY, 
				WorkspaceUtil.getWorkspacePath( "output").toString());
		Path path = Paths.get(
				rootOutputDir, 
				corpus.getCorpus().getName(), 
				corpus.getLanguage().getName());
		return path.toString();		
	}

	@Override
	public String getOutputDirectory(ESingleLanguageCorpus corpus, EPipeline pipeline) {
		return Paths.get(getOutputDirectory(corpus), 
				pipeline.getTargetTerminologyName()).toString();
	}

	@Override
	public String getTerminoOutputDirectory(ESingleLanguageCorpus corpus, EPipeline pipeline, String extension) {
		Path path = Paths.get(getOutputDirectory(corpus, pipeline), pipeline.getTargetTerminologyName() + "." + extension);
		FileUtil.mkdirs(path.toFile());
		return path.toString();
	}
	
	@Override
	public String getCasOutputDirectory(ESingleLanguageCorpus corpus, EPipeline pipeline, String casDirname) {
		Path path = Paths.get(getOutputDirectory(corpus, pipeline), casDirname);
		FileUtil.mkdirs(path.toFile());
		return path.toString();		
	}



	@Override
	public String getTerminoJsonFile(ESingleLanguageCorpus corpus, EPipeline pipeline) {
		return getTerminoOutputDirectory(corpus, pipeline, JSON_EXTENSION);
	}



	@Override
	public String getTerminoTsvFile(ESingleLanguageCorpus corpus, EPipeline pipeline) {
		return getTerminoOutputDirectory(corpus, pipeline, TSV_EXTENSION);
	}



	@Override
	public String getTerminoTbxFile(ESingleLanguageCorpus corpus, EPipeline pipeline) {
		return getTerminoOutputDirectory(corpus, pipeline, TBX_EXTENSION);
	}



	@Override
	public String getXmiCasOutputDirectory(ESingleLanguageCorpus corpus, EPipeline pipeline) {
		return getCasOutputDirectory(corpus, pipeline, CAS_XMI_DIR);
	}



	@Override
	public String getTsvCasOutputDirectory(ESingleLanguageCorpus corpus, EPipeline pipeline) {
		return getCasOutputDirectory(corpus, pipeline, CAS_TSV_DIR);
	}



	@Override
	public String getJsonCasOutputDirectory(ESingleLanguageCorpus corpus, EPipeline pipeline) {
		return getCasOutputDirectory(corpus, pipeline, CAS_JSON_DIR);
	}



	@Override
	public void removeCorpus(ECorpus s) {
		corpora.getCorpora().remove(s);
		WorkspaceUtil.removeResource(CORPUS_DIR, s.getName(), CORPUS_EXTENSION);
		resetCache();
		eventBroker.post(TermSuiteEvents.CORPUS_REMOVED, s);

	}



	private void resetCache() {
		documentCache = null;
	}

	private Map<String, EDocument> getDocumentCache() {
		if (documentCache == null) {
			final String FILE_PROTOCOL = "file:";
			documentCache = Maps.newHashMap();
			for (ECorpus corpus : getCorporaList().getCorpora()) {
				for (ESingleLanguageCorpus slc : corpus.getSingleLanguageCorpora()) {
					for (EDocument d : slc.getDocuments()) {
						File file = asFile(d);
						documentCache.put(FILE_PROTOCOL + file.getAbsolutePath(), d);
						documentCache.put(FILE_PROTOCOL + file.getPath(), d);
					}
				}
			}
		}
		return documentCache;
	}


	
	@Override
	public EDocument resolveEDocument(Document sourceDocument) {
		EDocument doc = getDocumentCache().get(sourceDocument.getUrl());
		
		if(doc == null)
			this.logger.warn("Could not resolve document with url " + sourceDocument.getUrl());
		
		return doc;
{'fraction_non_alphanumeric': 0.11039241052177663, 'fraction_numerical': 0.0005390254420008624, 'mean_word_length': 5.167719414893617, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6208179', 'n_tokens_mistral': 13937, 'n_tokens_neox': 12149, 'n_words': 1769}