<commit_before>package fr.inra.maiage.bibliome.alvisnlp.bibliomefactory.modules.yatea;

import java.io.BufferedOutputStream;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.PrintStream;
import java.io.StringReader;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import fr.inra.maiage.bibliome.alvisnlp.bibliomefactory.modules.SectionModule.SectionResolvedObjects;
import fr.inra.maiage.bibliome.alvisnlp.core.corpus.Annotation;
import fr.inra.maiage.bibliome.alvisnlp.core.corpus.Corpus;
import fr.inra.maiage.bibliome.alvisnlp.core.corpus.Layer;
import fr.inra.maiage.bibliome.alvisnlp.core.corpus.Section;
import fr.inra.maiage.bibliome.alvisnlp.core.corpus.expressions.EvaluationContext;
import fr.inra.maiage.bibliome.alvisnlp.core.module.ModuleException;
import fr.inra.maiage.bibliome.alvisnlp.core.module.ProcessingContext;
import fr.inra.maiage.bibliome.alvisnlp.core.module.ProcessingException;
import fr.inra.maiage.bibliome.alvisnlp.core.module.lib.ExternalHandler;
import fr.inra.maiage.bibliome.util.Files;
import fr.inra.maiage.bibliome.util.Iterators;
import fr.inra.maiage.bibliome.util.Strings;
import fr.inra.maiage.bibliome.util.files.InputFile;
import fr.inra.maiage.bibliome.util.files.OutputFile;
import fr.inra.maiage.bibliome.util.streams.SourceStream;

public class YateaExtractorExternalHandler<S extends SectionResolvedObjects> extends ExternalHandler<Corpus,AbstractYateaExtractor<S>> {
	private final EvaluationContext evalCtx;
	private final Properties defaultConfig = new Properties();
	private final Properties options = new Properties();
	
	public YateaExtractorExternalHandler(ProcessingContext<Corpus> processingContext, AbstractYateaExtractor<S> module, Corpus annotable) {
		super(processingContext, module, annotable);
		this.evalCtx = new EvaluationContext(getLogger());
	}

	@Override
	protected void prepare() throws IOException, ModuleException {
		ensureDirs();
		TestifiedTerminology testifiedTerminology = getModule().getTestifiedTerminology();
		InputFile testifiedTerminologyFile = testifiedTerminology == null ? null : testifiedTerminology.ensureFile(getModule(), getProcessingContext(), getAnnotable());
		createRCFile(testifiedTerminologyFile);
		createInput();
	}
	
	private void ensureDirs() throws ProcessingException {
		AbstractYateaExtractor<S> owner = getModule();
		File outputDir = owner.getOutputDir();
		if (outputDir != null && !outputDir.exists() && !outputDir.mkdirs()) {
			throw new ProcessingException("could not create " + outputDir.getAbsolutePath());
		}
		File workingDir = getWorkingDirectory();
		if (!workingDir.exists() && !workingDir.mkdirs()) {
			throw new ProcessingException("could not create " + outputDir.getAbsolutePath());
		}
	}

	private void createRCFile(InputFile testifiedTerminologyFile) throws IOException {
    	createConfig(testifiedTerminologyFile);
    	AbstractYateaExtractor.writeYateaConfig(getRCFile(), defaultConfig, options);
    }
   
    private void createConfig(InputFile testifiedTerminology) throws IOException {
		AbstractYateaExtractor<S> owner = getModule();
    	readYateaConfig(owner.getRcFile(), defaultConfig, options);
    	updateProperties(defaultConfig, owner.getYateaDefaultConfig());
    	updateProperties(options, owner.getYateaOptions());
    	updateProperty(defaultConfig, "CONFIG_DIR", owner.getConfigDir().getAbsolutePath());
    	updateProperty(defaultConfig, "LOCALE_DIR", owner.getLocaleDir().getAbsolutePath());
    	updateProperty(options, "output-path", owner.getOutputDir());
    	updateProperty(options, "language", owner.getLanguage());
    	updateProperty(options, "termino", testifiedTerminology);
    	updateProperty(options, "suffix", owner.getSuffix());
		getLogger().fine("owner.getOutputDir() = " + owner.getOutputDir());
		getLogger().fine("options = " + options);
		getLogger().fine("defaultConfig = " + defaultConfig);
    }

    public Properties getDefaultConfig() {
		return defaultConfig;
	}

	public Properties getOptions() {
		return options;
	}

	private static void readYateaConfig(SourceStream source, Properties defaultConfig, Properties options) throws IOException {
		BufferedReader r = source.getBufferedReader();
		Properties current = null;
		LOOP: while (true) {
			String line = r.readLine();
			if (line == null) {
				break;
			}<commit_msg>fixed NPE when optional parameters were not set (#114)<commit_after>package fr.inra.maiage.bibliome.alvisnlp.bibliomefactory.modules.yatea;

import java.io.BufferedOutputStream;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.PrintStream;
import java.io.StringReader;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import fr.inra.maiage.bibliome.alvisnlp.bibliomefactory.modules.SectionModule.SectionResolvedObjects;
import fr.inra.maiage.bibliome.alvisnlp.core.corpus.Annotation;
import fr.inra.maiage.bibliome.alvisnlp.core.corpus.Corpus;
import fr.inra.maiage.bibliome.alvisnlp.core.corpus.Layer;
import fr.inra.maiage.bibliome.alvisnlp.core.corpus.Section;
import fr.inra.maiage.bibliome.alvisnlp.core.corpus.expressions.EvaluationContext;
import fr.inra.maiage.bibliome.alvisnlp.core.module.ModuleException;
import fr.inra.maiage.bibliome.alvisnlp.core.module.ProcessingContext;
import fr.inra.maiage.bibliome.alvisnlp.core.module.ProcessingException;
import fr.inra.maiage.bibliome.alvisnlp.core.module.lib.ExternalHandler;
import fr.inra.maiage.bibliome.util.Files;
import fr.inra.maiage.bibliome.util.Iterators;
import fr.inra.maiage.bibliome.util.Strings;
import fr.inra.maiage.bibliome.util.files.InputFile;
import fr.inra.maiage.bibliome.util.files.OutputFile;
import fr.inra.maiage.bibliome.util.streams.SourceStream;

public class YateaExtractorExternalHandler<S extends SectionResolvedObjects> extends ExternalHandler<Corpus,AbstractYateaExtractor<S>> {
	private final EvaluationContext evalCtx;
	private final Properties defaultConfig = new Properties();
	private final Properties options = new Properties();
	
	public YateaExtractorExternalHandler(ProcessingContext<Corpus> processingContext, AbstractYateaExtractor<S> module, Corpus annotable) {
		super(processingContext, module, annotable);
		this.evalCtx = new EvaluationContext(getLogger());
	}

	@Override
	protected void prepare() throws IOException, ModuleException {
		ensureDirs();
		TestifiedTerminology testifiedTerminology = getModule().getTestifiedTerminology();
		InputFile testifiedTerminologyFile = testifiedTerminology == null ? null : testifiedTerminology.ensureFile(getModule(), getProcessingContext(), getAnnotable());
		createRCFile(testifiedTerminologyFile);
		createInput();
	}
	
	private void ensureDirs() throws ProcessingException {
		AbstractYateaExtractor<S> owner = getModule();
		File outputDir = owner.getOutputDir();
		if (outputDir != null && !outputDir.exists() && !outputDir.mkdirs()) {
			throw new ProcessingException("could not create " + outputDir.getAbsolutePath());
		}
		File workingDir = getWorkingDirectory();
		if (!workingDir.exists() && !workingDir.mkdirs()) {
			throw new ProcessingException("could not create " + outputDir.getAbsolutePath());
		}
	}

	private void createRCFile(InputFile testifiedTerminologyFile) throws IOException {
    	createConfig(testifiedTerminologyFile);
    	AbstractYateaExtractor.writeYateaConfig(getRCFile(), defaultConfig, options);
    }
   
    private void createConfig(InputFile testifiedTerminology) throws IOException {
		AbstractYateaExtractor<S> owner = getModule();
    	readYateaConfig(owner.getRcFile(), defaultConfig, options);
    	updateProperties(defaultConfig, owner.getYateaDefaultConfig());
    	updateProperties(options, owner.getYateaOptions());
    	if (owner.getConfigDir() != null) {
    		updateProperty(defaultConfig, "CONFIG_DIR", owner.getConfigDir().getAbsolutePath());
    	}
    	if (owner.getLocaleDir() != null) {
    		updateProperty(defaultConfig, "LOCALE_DIR", owner.getLocaleDir().getAbsolutePath());
    	}
    	updateProperty(options, "output-path", owner.getOutputDir());
    	updateProperty(options, "language", owner.getLanguage());
    	updateProperty(options, "termino", testifiedTerminology);
    	updateProperty(options, "suffix", owner.getSuffix());
		getLogger().fine("owner.getOutputDir() = " + owner.getOutputDir());
		getLogger().fine("options = " + options);
		getLogger().fine("defaultConfig = " + defaultConfig);
    }

    public Properties getDefaultConfig() {
		return defaultConfig;
	}

	public Properties getOptions() {
		return options;
	}

	private static void readYateaConfig(SourceStream source, Properties defaultConfig, Properties options) throws IOException {
		BufferedReader r = source.getBufferedReader();
		Properties current = null;
		LOOP: while (true) {
			String line = r.readLine();
			if (line == null) {
				break;
			}
{'fraction_non_alphanumeric': 0.11223284287401998, 'fraction_numerical': 0.0003221995489206315, 'mean_word_length': 6.049205147615443, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6278716', 'n_tokens_mistral': 2987, 'n_tokens_neox': 2851, 'n_words': 498}