<commit_before>package io.bdrc.xmltoldmigration.xml2files;

import static io.bdrc.libraries.LangStrings.EWTS_TAG;
import static io.bdrc.libraries.LangStrings.getBCP47;
import static io.bdrc.libraries.LangStrings.isDeva;
import static io.bdrc.libraries.LangStrings.isLikelyEnglish;
import static io.bdrc.libraries.LangStrings.normalizeTibetan;
import static io.bdrc.libraries.Models.ADM;
import static io.bdrc.libraries.Models.BDA;
import static io.bdrc.libraries.Models.BDO;
import static io.bdrc.libraries.Models.BDR;
import static io.bdrc.libraries.Models.addReleased;
import static io.bdrc.libraries.Models.createAdminRoot;
import static io.bdrc.libraries.Models.getAdminData;
import static io.bdrc.libraries.Models.getEvent;
import static io.bdrc.libraries.Models.getFacetNode;
import static io.bdrc.libraries.Models.setPrefixes;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;

import javax.xml.transform.Source;
import javax.xml.transform.dom.DOMSource;
import javax.xml.validation.Validator;

import org.apache.jena.datatypes.DatatypeFormatException;
import org.apache.jena.datatypes.RDFDatatype;
import org.apache.jena.datatypes.xsd.XSDDatatype;
import org.apache.jena.datatypes.xsd.XSDDateTime;
import org.apache.jena.ontology.OntModel;
import org.apache.jena.rdf.model.Literal;
import org.apache.jena.rdf.model.Model;
import org.apache.jena.rdf.model.ModelFactory;
import org.apache.jena.rdf.model.Property;
import org.apache.jena.rdf.model.Resource;
import org.apache.jena.rdf.model.ResourceFactory;
import org.apache.jena.rdf.model.Statement;
import org.apache.jena.rdf.model.StmtIterator;
import org.apache.jena.reasoner.ValidityReport;
import org.apache.jena.vocabulary.RDF;
import org.apache.jena.vocabulary.RDFS;
import org.apache.jena.vocabulary.SKOS;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.SAXException;

import com.atlascopco.hunspell.Hunspell;

import io.bdrc.ewtsconverter.EwtsConverter;
import io.bdrc.libraries.Models.FacetType;
//import io.bdrc.libraries.BdrcDateType;
import io.bdrc.xmltoldmigration.MigrationHelpers;
import io.bdrc.xmltoldmigration.helpers.EwtsFixer;
import io.bdrc.xmltoldmigration.helpers.ExceptionHelper;
import openllet.core.exceptions.InternalReasonerException;

public class CommonMigration  {

    public static final String FPL_LIBRARY_ID = "G1TLMFPL000001";

    public static final String IMAGE_ITEM_SUFFIX = "";

    public static final int ET_LANG = ExceptionHelper.ET_LANG;

    public static final EwtsConverter converter = new EwtsConverter();
    public static final EwtsConverter converterAlalc = new EwtsConverter(true, true, false, false, EwtsConverter.Mode.ALALC);
    public static final String hunspellBoPath = "src/main/resources/hunspell-bo/";
    public static final Hunspell speller = new Hunspell(hunspellBoPath+"bo.dic", hunspellBoPath+"bo.aff");

    public static final Map<String, String> logWhoToUri = new HashMap<>();
    public static final Map<String, Boolean> genreTopics = new HashMap<>();
    public static final Map<Integer, Boolean> isTraditional = new HashMap<>();
    public static final Map<String, String> creatorMigrations = new HashMap<>();

    static {
        fillLogWhoToUri();
        fillGenreTopics();
        getTcList();
        initCreatorMigrations();
    }

    private static void initCreatorMigrations() {
        final ClassLoader classLoader = MigrationHelpers.class.getClassLoader();
        final InputStream inputStream = classLoader.getResourceAsStream("creator-migrations.txt");
        final BufferedReader in = new BufferedReader(new InputStreamReader(inputStream));
        String line = null;
        try {
            while((line = in.readLine()) != null) {
                String[] key_val = line.split(",");
                creatorMigrations.put(key_val[0], key_val[1]);
            }
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    private static void getTcList() {
        final ClassLoader classLoader = MigrationHelpers.class.getClassLoader();
        final InputStream inputStream = classLoader.getResourceAsStream("tclist.txt");
        final BufferedReader in = new BufferedReader(new InputStreamReader(inputStream));
        String line = null;
        try {
            while((line = in.readLine()) != null) {
                isTraditional.put(line.codePointAt(0), true);
            }
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    private static final String userNumFormat = "%05d";

    private static void fillGenreTopics() {
        final ClassLoader classLoader = CommonMigration.class.getClassLoader();
        final InputStream inputStream = classLoader.getResourceAsStream("topics-genres.txt");
        final BufferedReader in = new BufferedReader(new InputStreamReader(inputStream));
        String line = null;
        try {
            while((line = in.readLine()) != null) {
                genreTopics.put(line, true);
            }
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    private static String getCreatorRoleUri(String type) {
        if (type.startsWith("has"))
            type = type.substring(3);
        return BDR+creatorMigrations.get(type);
    }

    /**
     * Creates a new named AgentAsCreator node and adds bdo:creator node to the supplied work.
     * 
     * @param work that the AgentAsCreator is a creator for
     * @param person that is the creating agent
     * @param roleKey the name of the type pof role of the creator
     * @param rootAdmWork the root AdminData that contains the adm:facetIndex
     */
    public static void addAgentAsCreator(Resource work, Resource person, String roleKey, Resource workA) {
        Model m = work.getModel();
        Resource agentAsCreator = getFacetNode(FacetType.CREATOR, work);
        work.addProperty(m.createProperty(BDO+"creator"), agentAsCreator);
        agentAsCreator.addProperty(m.createProperty(BDO+"agent"), person);
        Resource role = m.createResource(getCreatorRoleUri(roleKey));
        agentAsCreator.addProperty(m.createProperty(BDO+"role"), role);
    }

    public static void addDatesToEvent(String dateStr, Resource r, String eventProp, String eventType) {
        Resource event = getEvent(r, eventType, eventProp);
        addDates(dateStr, event, r);<commit_msg>creator on the right level<commit_after>package io.bdrc.xmltoldmigration.xml2files;

import static io.bdrc.libraries.LangStrings.EWTS_TAG;
import static io.bdrc.libraries.LangStrings.getBCP47;
import static io.bdrc.libraries.LangStrings.isDeva;
import static io.bdrc.libraries.LangStrings.isLikelyEnglish;
import static io.bdrc.libraries.LangStrings.normalizeTibetan;
import static io.bdrc.libraries.Models.ADM;
import static io.bdrc.libraries.Models.BDA;
import static io.bdrc.libraries.Models.BDO;
import static io.bdrc.libraries.Models.BDR;
import static io.bdrc.libraries.Models.addReleased;
import static io.bdrc.libraries.Models.createAdminRoot;
import static io.bdrc.libraries.Models.getAdminData;
import static io.bdrc.libraries.Models.getEvent;
import static io.bdrc.libraries.Models.getFacetNode;
import static io.bdrc.libraries.Models.setPrefixes;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;

import javax.xml.transform.Source;
import javax.xml.transform.dom.DOMSource;
import javax.xml.validation.Validator;

import org.apache.jena.datatypes.DatatypeFormatException;
import org.apache.jena.datatypes.RDFDatatype;
import org.apache.jena.datatypes.xsd.XSDDatatype;
import org.apache.jena.datatypes.xsd.XSDDateTime;
import org.apache.jena.ontology.OntModel;
import org.apache.jena.rdf.model.Literal;
import org.apache.jena.rdf.model.Model;
import org.apache.jena.rdf.model.ModelFactory;
import org.apache.jena.rdf.model.Property;
import org.apache.jena.rdf.model.Resource;
import org.apache.jena.rdf.model.ResourceFactory;
import org.apache.jena.rdf.model.Statement;
import org.apache.jena.rdf.model.StmtIterator;
import org.apache.jena.reasoner.ValidityReport;
import org.apache.jena.vocabulary.RDF;
import org.apache.jena.vocabulary.RDFS;
import org.apache.jena.vocabulary.SKOS;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.SAXException;

import com.atlascopco.hunspell.Hunspell;

import io.bdrc.ewtsconverter.EwtsConverter;
import io.bdrc.libraries.Models.FacetType;
//import io.bdrc.libraries.BdrcDateType;
import io.bdrc.xmltoldmigration.MigrationHelpers;
import io.bdrc.xmltoldmigration.helpers.EwtsFixer;
import io.bdrc.xmltoldmigration.helpers.ExceptionHelper;
import openllet.core.exceptions.InternalReasonerException;

public class CommonMigration  {

    public static final String FPL_LIBRARY_ID = "G1TLMFPL000001";

    public static final String IMAGE_ITEM_SUFFIX = "";

    public static final int ET_LANG = ExceptionHelper.ET_LANG;

    public static final EwtsConverter converter = new EwtsConverter();
    public static final EwtsConverter converterAlalc = new EwtsConverter(true, true, false, false, EwtsConverter.Mode.ALALC);
    public static final String hunspellBoPath = "src/main/resources/hunspell-bo/";
    public static final Hunspell speller = new Hunspell(hunspellBoPath+"bo.dic", hunspellBoPath+"bo.aff");

    public static final Map<String, String> logWhoToUri = new HashMap<>();
    public static final Map<String, Boolean> genreTopics = new HashMap<>();
    public static final Map<Integer, Boolean> isTraditional = new HashMap<>();
    public static final Map<String, String> creatorMigrations = new HashMap<>();

    static {
        fillLogWhoToUri();
        fillGenreTopics();
        getTcList();
        initCreatorMigrations();
    }

    private static void initCreatorMigrations() {
        final ClassLoader classLoader = MigrationHelpers.class.getClassLoader();
        final InputStream inputStream = classLoader.getResourceAsStream("creator-migrations.txt");
        final BufferedReader in = new BufferedReader(new InputStreamReader(inputStream));
        String line = null;
        try {
            while((line = in.readLine()) != null) {
                String[] key_val = line.split(",");
                creatorMigrations.put(key_val[0], key_val[1]);
            }
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    private static void getTcList() {
        final ClassLoader classLoader = MigrationHelpers.class.getClassLoader();
        final InputStream inputStream = classLoader.getResourceAsStream("tclist.txt");
        final BufferedReader in = new BufferedReader(new InputStreamReader(inputStream));
        String line = null;
        try {
            while((line = in.readLine()) != null) {
                isTraditional.put(line.codePointAt(0), true);
            }
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    private static final String userNumFormat = "%05d";

    private static void fillGenreTopics() {
        final ClassLoader classLoader = CommonMigration.class.getClassLoader();
        final InputStream inputStream = classLoader.getResourceAsStream("topics-genres.txt");
        final BufferedReader in = new BufferedReader(new InputStreamReader(inputStream));
        String line = null;
        try {
            while((line = in.readLine()) != null) {
                genreTopics.put(line, true);
            }
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    private static String getCreatorRoleUri(String type) {
        if (type.startsWith("has"))
            type = type.substring(3);
        return BDR+creatorMigrations.get(type);
    }

    
    public static final List<String> creatorForInstance = Arrays.asList(new String[]{"hasCalligrapher", "hasScribe", "hasEditor"});
    
    /**
     * Creates a new named AgentAsCreator node and adds bdo:creator node to the supplied work.
     * 
     * @param work that the AgentAsCreator is a creator for
     * @param person that is the creating agent
     * @param roleKey the name of the type pof role of the creator
     * @param rootAdmWork the root AdminData that contains the adm:facetIndex
     */
    public static void addAgentAsCreator(Resource work, Resource person, String roleKey, Resource workA) {
        Model m = work.getModel();
        Resource agentAsCreator;
        if (workA != null && !creatorForInstance.contains(roleKey)) {
            agentAsCreator = getFacetNode(FacetType.CREATOR, workA);
        } else {
            agentAsCreator = getFacetNode(FacetType.CREATOR, work);            
        }
        work.addProperty(m.createProperty(BDO+"creator"), agentAsCreator);
        agentAsCreator.addProperty(m.createProperty(BDO+"agent"), person);
        Resource role = m.createResource(getCreatorRoleUri(roleKey));
        agentAsCreator.addProperty(m.createProperty(BDO+"role"), role);
    }

    public static void addDatesToEvent(String dateStr, Resource r, String eventProp, String eventType) {
        Resource event = getEvent(r, eventType, eventProp);
        addDates(dateStr, event, r);
{'fraction_non_alphanumeric': 0.10152024557813186, 'fraction_numerical': 0.0029235491887151, 'mean_word_length': 3.7020618556701033, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 18}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '830279', 'n_tokens_mistral': 4174, 'n_tokens_neox': 4143, 'n_words': 887}