<commit_before>package fr.inria.aviz.tikaextensions.tika.xml;

import java.util.Collections;
import java.util.Set;

import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.Property;
import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.mime.MediaType;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.xml.AttributeMetadataHandler;
import org.apache.tika.sax.TeeContentHandler;
import org.apache.tika.sax.TextContentHandler;
import org.xml.sax.ContentHandler;
import fr.inria.aviz.tikaextensions.tika.CendariProperties;

/**
 * Class EADParser
 * 
 * @author <NAME>
 * @version $Revision$
 */
public class EADParser extends AbstractXMLParser {
    private static final String NAMESPACE_URI_EAD = "";
    
    private static final Set<MediaType> SUPPORTED_TYPES =
            Collections.singleton(MediaType.application("ead+xml"));
    
    /**
     * {@inheritDoc}
     */
    @Override
    public Set<MediaType> getSupportedTypes(ParseContext context) {
        return SUPPORTED_TYPES;
    }
    
    private static ContentHandler getEADHandler(
            Metadata metadata, Property property, String element, 
            String ...context) {
        return new ContextualElementMetadataHandler(
                NAMESPACE_URI_EAD, element,
                metadata, property, context);
    }

    protected ContentHandler getContentHandler(
            ContentHandler handler, Metadata metadata, ParseContext context) {
        ContentHandler defaultContentHandler = new TextContentHandler(handler, true);
        return new TeeContentHandler(
                defaultContentHandler,
                getEADHandler(metadata, TikaCoreProperties.TITLE, 
                        "titleproper", "titlestmt"),
                //TODO check the specific EAD elements
                getEADHandler(metadata, TikaCoreProperties.KEYWORDS, 
                        "term", "keywords"),
                //getTEIHandler(metadata, TikaCoreProperties.CREATOR, "creator"),
                getEADHandler(metadata, TikaCoreProperties.DESCRIPTION, 
                        "description"),
                //getEADHandler(metadata, TikaCoreProperties.PUBLISHER, "publisher"),
                getEADHandler(metadata, TikaCoreProperties.CONTRIBUTOR, 
                        "name", "titleStmt", "respStmt"),
                //getEADHandler(metadata, CendariProperties.DATE, "date"),
                new ElementAttributeMetadataHandler(
                        NAMESPACE_URI_EAD, "normal",
                            metadata, CendariProperties.DATE, "unitdate") {
                    protected void addMetadata(String date) {
                        String[] dates = date.split("/");
                        for (String d : dates) 
                            super.addMetadata(d);
                    };
                },
                //getEADHandler(metadata, TikaCoreProperties.CREATED, "date"),
                //getEADHandler(metadata, TikaCoreProperties.TYPE, "type"),
                //getEADHandler(metadata, TikaCoreProperties.FORMAT, "format"),
                //getEADHandler(metadata, TikaCoreProperties.IDENTIFIER, "identifier"),
                getEADHandler(metadata, TikaCoreProperties.RIGHTS, "licence"),
                getEADHandler(metadata, CendariProperties.NERD, "did"),
                getEADHandler(metadata, CendariProperties.NERD, "archref"),
                getEADHandler(metadata, CendariProperties.NERD, "eventgrp"),
                getEADHandler(metadata, CendariProperties.NERD, "chronitem"),
                getEADHandler(metadata, CendariProperties.NERD, "acqinfo"),
                getEADHandler(metadata, CendariProperties.NERD, "appraisal"),
                getEADHandler(metadata, CendariProperties.NERD, "bibliography"),
                getEADHandler(metadata, CendariProperties.NERD, "bioghist"),
                getEADHandler(metadata, CendariProperties.NERD, "c"),
                getEADHandler(metadata, CendariProperties.NERD, "c01"),
                getEADHandler(metadata, CendariProperties.NERD, "c02"),
                getEADHandler(metadata, CendariProperties.NERD, "c03"),
                getEADHandler(metadata, CendariProperties.NERD, "c04"),
                getEADHandler(metadata, CendariProperties.NERD, "c05"),
                getEADHandler(metadata, CendariProperties.NERD, "c06"),
                getEADHandler(metadata, CendariProperties.NERD, "c07"),
                getEADHandler(metadata, CendariProperties.NERD, "c08"),
                getEADHandler(metadata, CendariProperties.NERD, "c09"),
                getEADHandler(metadata, CendariProperties.NERD, "c10"),
                getEADHandler(metadata, CendariProperties.NERD, "c11"),
                getEADHandler(metadata, CendariProperties.NERD, "c12"),
                getEADHandler(metadata, CendariProperties.NERD, "custodhist"),
                getEADHandler(metadata, CendariProperties.NERD, "daodesc"),
                getEADHandler(metadata, CendariProperties.NERD, "descgrp"),
                getEADHandler(metadata, CendariProperties.NERD, "div"),
                getEADHandler(metadata, CendariProperties.NERD, "dsc"),
                getEADHandler(metadata, CendariProperties.NERD, "dscgrp"),
                getEADHandler(metadata, CendariProperties.NERD, "odd"),
                getEADHandler(metadata, CendariProperties.NERD, "relatedmaterial"),
                getEADHandler(metadata, CendariProperties.NERD, "scopecontent"),
                getEADHandler(metadata, CendariProperties.NERD, "separatedmaterial"),
                getEADHandler(metadata, CendariProperties.NERD, "table"),
                getEADHandler(metadata, CendariProperties.NERD, "relatedmaterial"),
                getEADHandler(metadata, CendariProperties.NERD, "blockquote"),
                getEADHandler(metadata, CendariProperties.NERD, "note"),
                getEADHandler(metadata, CendariProperties.NERD, "publicationstmt"),
                getEADHandler(metadata, CendariProperties.NERD, "seriesstmt"),
                getEADHandler(metadata, CendariProperties.NERD, "titlepage"),
                getEADHandler(metadata, CendariProperties.NERD, "titlestmt"),
                getEADHandler(metadata, CendariProperties.NERD, "abstract"),
                getEADHandler(metadata, CendariProperties.NERD, "date", "creation"),
                getEADHandler(metadata, CendariProperties.NERD, "event"),
                getEADHandler(metadata, CendariProperties.NERD, "item"),
                getEADHandler(metadata, CendariProperties.NERD, "label"),
                getEADHandler(metadata, CendariProperties.NERD, "origination"),
                getEADHandler(metadata, CendariProperties.NERD, "titlestmt"),
                getEADHandler(metadata, CendariProperties.NERD, "name", "controlaccess"),
                getEADHandler(metadata, CendariProperties.NERD, "geogname", "controlaccess"),
                getEADHandler(metadata, CendariProperties.NERD, "subject", "controlaccess"),
                
                new AttributeMetadataHandler(NAMESPACE_URI_XML, "lang", metadata, 
                        CendariProperties.LANG),
                new AttributeMetadataHandler(NAMESPACE_URI_EAD, "langcode", metadata, 
                        CendariProperties.LANG)
        );
    }

}<commit_msg>Added cendari:reference for EAD files<commit_after>package fr.inria.aviz.tikaextensions.tika.xml;

import java.util.Collections;
import java.util.Set;

import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.Property;
import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.mime.MediaType;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.xml.AttributeMetadataHandler;
import org.apache.tika.sax.TeeContentHandler;
import org.apache.tika.sax.TextContentHandler;
import org.xml.sax.ContentHandler;
import fr.inria.aviz.tikaextensions.tika.CendariProperties;

/**
 * Class EADParser
 * 
 * @author <NAME>
 * @version $Revision$
 */
public class EADParser extends AbstractXMLParser {
    private static final String NAMESPACE_URI_EAD = "";
    
    private static final Set<MediaType> SUPPORTED_TYPES =
            Collections.singleton(MediaType.application("ead+xml"));
    
    /**
     * {@inheritDoc}
     */
    @Override
    public Set<MediaType> getSupportedTypes(ParseContext context) {
        return SUPPORTED_TYPES;
    }
    
    private static ContentHandler getEADHandler(
            Metadata metadata, Property property, String element, 
            String ...context) {
        return new ContextualElementMetadataHandler(
                NAMESPACE_URI_EAD, element,
                metadata, property, context);
    }

    protected ContentHandler getContentHandler(
            ContentHandler handler, Metadata metadata, ParseContext context) {
        ContentHandler defaultContentHandler = new TextContentHandler(handler, true);
        return new TeeContentHandler(
                defaultContentHandler,
                getEADHandler(metadata, TikaCoreProperties.TITLE, 
                        "titleproper", "titlestmt"),
                //TODO check the specific EAD elements
                getEADHandler(metadata, TikaCoreProperties.KEYWORDS, 
                        "term", "keywords"),
                //getTEIHandler(metadata, TikaCoreProperties.CREATOR, "creator"),
                getEADHandler(metadata, TikaCoreProperties.DESCRIPTION, 
                        "description"),
                //getEADHandler(metadata, TikaCoreProperties.PUBLISHER, "publisher"),
                getEADHandler(metadata, TikaCoreProperties.CONTRIBUTOR, 
                        "name", "titleStmt", "respStmt"),
                //getEADHandler(metadata, CendariProperties.DATE, "date"),
                new ElementAttributeMetadataHandler(
                        NAMESPACE_URI_EAD, "normal",
                            metadata, CendariProperties.DATE, "unitdate") {
                    protected void addMetadata(String date) {
                        String[] dates = date.split("/");
                        for (String d : dates) 
                            super.addMetadata(d);
                    };
                },
                //getEADHandler(metadata, TikaCoreProperties.CREATED, "date"),
                //getEADHandler(metadata, TikaCoreProperties.TYPE, "type"),
                //getEADHandler(metadata, TikaCoreProperties.FORMAT, "format"),
                //getEADHandler(metadata, TikaCoreProperties.IDENTIFIER, "identifier"),
                getEADHandler(metadata, TikaCoreProperties.RIGHTS, "licence"),
                getEADHandler(metadata, CendariProperties.NERD, "did"),
                getEADHandler(metadata, CendariProperties.NERD, "archref"),
                getEADHandler(metadata, CendariProperties.NERD, "eventgrp"),
                getEADHandler(metadata, CendariProperties.NERD, "chronitem"),
                getEADHandler(metadata, CendariProperties.NERD, "acqinfo"),
                getEADHandler(metadata, CendariProperties.NERD, "appraisal"),
                getEADHandler(metadata, CendariProperties.NERD, "bibliography"),
                getEADHandler(metadata, CendariProperties.NERD, "bioghist"),
                getEADHandler(metadata, CendariProperties.NERD, "c"),
                getEADHandler(metadata, CendariProperties.NERD, "c01"),
                getEADHandler(metadata, CendariProperties.NERD, "c02"),
                getEADHandler(metadata, CendariProperties.NERD, "c03"),
                getEADHandler(metadata, CendariProperties.NERD, "c04"),
                getEADHandler(metadata, CendariProperties.NERD, "c05"),
                getEADHandler(metadata, CendariProperties.NERD, "c06"),
                getEADHandler(metadata, CendariProperties.NERD, "c07"),
                getEADHandler(metadata, CendariProperties.NERD, "c08"),
                getEADHandler(metadata, CendariProperties.NERD, "c09"),
                getEADHandler(metadata, CendariProperties.NERD, "c10"),
                getEADHandler(metadata, CendariProperties.NERD, "c11"),
                getEADHandler(metadata, CendariProperties.NERD, "c12"),
                getEADHandler(metadata, CendariProperties.NERD, "custodhist"),
                getEADHandler(metadata, CendariProperties.NERD, "daodesc"),
                getEADHandler(metadata, CendariProperties.NERD, "descgrp"),
                getEADHandler(metadata, CendariProperties.NERD, "div"),
                getEADHandler(metadata, CendariProperties.NERD, "dsc"),
                getEADHandler(metadata, CendariProperties.NERD, "dscgrp"),
                getEADHandler(metadata, CendariProperties.NERD, "odd"),
                getEADHandler(metadata, CendariProperties.NERD, "relatedmaterial"),
                getEADHandler(metadata, CendariProperties.NERD, "scopecontent"),
                getEADHandler(metadata, CendariProperties.NERD, "separatedmaterial"),
                getEADHandler(metadata, CendariProperties.NERD, "table"),
                getEADHandler(metadata, CendariProperties.NERD, "relatedmaterial"),
                getEADHandler(metadata, CendariProperties.NERD, "blockquote"),
                getEADHandler(metadata, CendariProperties.NERD, "note"),
                getEADHandler(metadata, CendariProperties.NERD, "publicationstmt"),
                getEADHandler(metadata, CendariProperties.NERD, "seriesstmt"),
                getEADHandler(metadata, CendariProperties.NERD, "titlepage"),
                getEADHandler(metadata, CendariProperties.NERD, "titlestmt"),
                getEADHandler(metadata, CendariProperties.NERD, "abstract"),
                getEADHandler(metadata, CendariProperties.NERD, "date", "creation"),
                getEADHandler(metadata, CendariProperties.NERD, "event"),
                getEADHandler(metadata, CendariProperties.NERD, "item"),
                getEADHandler(metadata, CendariProperties.NERD, "label"),
                getEADHandler(metadata, CendariProperties.NERD, "origination"),
                getEADHandler(metadata, CendariProperties.NERD, "titlestmt"),
                getEADHandler(metadata, CendariProperties.NERD, "name", "controlaccess"),
                getEADHandler(metadata, CendariProperties.NERD, "geogname", "controlaccess"),
                getEADHandler(metadata, CendariProperties.NERD, "subject", "controlaccess"),
                
                new AttributeMetadataHandler(NAMESPACE_URI_XML, "lang", metadata, 
                        CendariProperties.LANG),
                new AttributeMetadataHandler(NAMESPACE_URI_EAD, "langcode", metadata, 
                        CendariProperties.LANG),
                new AttributeMetadataHandler("", "url", metadata, 
                            CendariProperties.REFERENCE)
        );
    }

}
{'fraction_non_alphanumeric': 0.09905120785949802, 'fraction_numerical': 0.0032299306910705873, 'mean_word_length': 2.501885014137606, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4230385', 'n_tokens_mistral': 4157, 'n_tokens_neox': 3931, 'n_words': 649}