<commit_before>package fr.inra.maiage.bibliome.alvisnlp.bibliomefactory.modules.yatea;

import java.io.IOException;
import java.io.InputStream;
import java.util.logging.Logger;

import javax.xml.parsers.ParserConfigurationException;
import javax.xml.parsers.SAXParser;
import javax.xml.parsers.SAXParserFactory;

import fr.inra.maiage.bibliome.alvisnlp.bibliomefactory.modules.SectionModule.SectionResolvedObjects;
import fr.inra.maiage.bibliome.alvisnlp.bibliomefactory.modules.trie.TrieProjector;
import fr.inra.maiage.bibliome.alvisnlp.bibliomefactory.modules.yatea.YateaTermsProjector.Term;
import org.xml.sax.Attributes;
import org.xml.sax.SAXException;
import org.xml.sax.helpers.DefaultHandler;

import fr.inra.maiage.bibliome.alvisnlp.core.corpus.Annotation;
import fr.inra.maiage.bibliome.alvisnlp.core.corpus.Corpus;
import fr.inra.maiage.bibliome.alvisnlp.core.corpus.DefaultNames;
import fr.inra.maiage.bibliome.alvisnlp.core.corpus.NameType;
import fr.inra.maiage.bibliome.alvisnlp.core.corpus.expressions.ResolverException;
import fr.inra.maiage.bibliome.alvisnlp.core.module.ModuleException;
import fr.inra.maiage.bibliome.alvisnlp.core.module.ProcessingContext;
import fr.inra.maiage.bibliome.alvisnlp.core.module.TimerCategory;
import fr.inra.maiage.bibliome.alvisnlp.core.module.lib.AlvisNLPModule;
import fr.inra.maiage.bibliome.alvisnlp.core.module.lib.Param;
import fr.inra.maiage.bibliome.alvisnlp.core.module.lib.TimeThis;
import fr.inra.maiage.bibliome.util.Iterators;
import fr.inra.maiage.bibliome.util.marshall.Decoder;
import fr.inra.maiage.bibliome.util.marshall.Encoder;
import fr.inra.maiage.bibliome.util.streams.SourceStream;
import fr.inra.maiage.bibliome.util.trie.Trie;

@AlvisNLPModule
public abstract class YateaTermsProjector extends TrieProjector<SectionResolvedObjects,Term> {
	private Boolean mnpOnly = false;
	private Boolean projectLemmas = false;
	private SourceStream yateaFile;
	private String termId = "term-id";
	private String monoHeadId = "mono-head";
	private String termLemma;
	private String termPOS = DefaultNames.getPosTagFeature();
	private String head = "head";
	private String modifier = "modifier";
	
	public static final class Term {
		private String id;
		private String form;
		private String lemma;
		private String pos;
		private String monoHead;
		private String head;
		private String modifier;
		
		@Override
		public String toString() {
			return "Term [id=" + id + ", form=" + form + "]";
		}
	}

	private final class YateaHandler extends DefaultHandler {
		private Term currentTerm;
		private final Trie<Term> dict;
		private final StringBuilder chars = new StringBuilder();
		private boolean syntacticAnalysis;
		private boolean dismissed;
		private boolean mnp;
		
		private YateaHandler(Trie<Term> dict) {
			super();
			this.dict = dict;<commit_msg>default value for parameter termLemma<commit_after>package fr.inra.maiage.bibliome.alvisnlp.bibliomefactory.modules.yatea;

import java.io.IOException;
import java.io.InputStream;
import java.util.logging.Logger;

import javax.xml.parsers.ParserConfigurationException;
import javax.xml.parsers.SAXParser;
import javax.xml.parsers.SAXParserFactory;

import fr.inra.maiage.bibliome.alvisnlp.bibliomefactory.modules.SectionModule.SectionResolvedObjects;
import fr.inra.maiage.bibliome.alvisnlp.bibliomefactory.modules.trie.TrieProjector;
import fr.inra.maiage.bibliome.alvisnlp.bibliomefactory.modules.yatea.YateaTermsProjector.Term;
import org.xml.sax.Attributes;
import org.xml.sax.SAXException;
import org.xml.sax.helpers.DefaultHandler;

import fr.inra.maiage.bibliome.alvisnlp.core.corpus.Annotation;
import fr.inra.maiage.bibliome.alvisnlp.core.corpus.Corpus;
import fr.inra.maiage.bibliome.alvisnlp.core.corpus.DefaultNames;
import fr.inra.maiage.bibliome.alvisnlp.core.corpus.NameType;
import fr.inra.maiage.bibliome.alvisnlp.core.corpus.expressions.ResolverException;
import fr.inra.maiage.bibliome.alvisnlp.core.module.ModuleException;
import fr.inra.maiage.bibliome.alvisnlp.core.module.ProcessingContext;
import fr.inra.maiage.bibliome.alvisnlp.core.module.TimerCategory;
import fr.inra.maiage.bibliome.alvisnlp.core.module.lib.AlvisNLPModule;
import fr.inra.maiage.bibliome.alvisnlp.core.module.lib.Param;
import fr.inra.maiage.bibliome.alvisnlp.core.module.lib.TimeThis;
import fr.inra.maiage.bibliome.util.Iterators;
import fr.inra.maiage.bibliome.util.marshall.Decoder;
import fr.inra.maiage.bibliome.util.marshall.Encoder;
import fr.inra.maiage.bibliome.util.streams.SourceStream;
import fr.inra.maiage.bibliome.util.trie.Trie;

@AlvisNLPModule
public abstract class YateaTermsProjector extends TrieProjector<SectionResolvedObjects,Term> {
	private Boolean mnpOnly = false;
	private Boolean projectLemmas = false;
	private SourceStream yateaFile;
	private String termId = "term-id";
	private String monoHeadId = "mono-head";
	private String termLemma = DefaultNames.getCanonicalFormFeature();
	private String termPOS = DefaultNames.getPosTagFeature();
	private String head = "head";
	private String modifier = "modifier";
	
	public static final class Term {
		private String id;
		private String form;
		private String lemma;
		private String pos;
		private String monoHead;
		private String head;
		private String modifier;
		
		@Override
		public String toString() {
			return "Term [id=" + id + ", form=" + form + "]";
		}
	}

	private final class YateaHandler extends DefaultHandler {
		private Term currentTerm;
		private final Trie<Term> dict;
		private final StringBuilder chars = new StringBuilder();
		private boolean syntacticAnalysis;
		private boolean dismissed;
		private boolean mnp;
		
		private YateaHandler(Trie<Term> dict) {
			super();
			this.dict = dict;
{'fraction_non_alphanumeric': 0.10117276387187117, 'fraction_numerical': 0.0, 'mean_word_length': 5.567816091954023, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 12}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4447706', 'n_tokens_mistral': 2073, 'n_tokens_neox': 1991, 'n_words': 342}