<commit_before>package io.lumify.mimeTypeOntologyMapper;

import io.lumify.core.ingest.graphProperty.GraphPropertyWorkData;
import io.lumify.core.ingest.graphProperty.GraphPropertyWorker;
import io.lumify.core.ingest.graphProperty.GraphPropertyWorkerPrepareData;
import io.lumify.core.model.ontology.Concept;
import io.lumify.core.model.properties.LumifyProperties;
import io.lumify.core.util.LumifyLogger;
import io.lumify.core.util.LumifyLoggerFactory;
import org.securegraph.Element;
import org.securegraph.Property;

import java.io.InputStream;

import static org.securegraph.util.Preconditions.checkNotNull;

public class MimeTypeOntologyMapperGraphPropertyWorker extends GraphPropertyWorker {
    private static final LumifyLogger LOGGER = LumifyLoggerFactory.getLogger(MimeTypeOntologyMapperGraphPropertyWorker.class);
    public static final String CONFIG_ONTOLOGY_IRI_IMAGE = "ontology.iri.image";
    public static final String CONFIG_ONTOLOGY_IRI_AUDIO = "ontology.iri.audio";
    public static final String CONFIG_ONTOLOGY_IRI_VIDEO = "ontology.iri.video";
    public static final String CONFIG_ONTOLOGY_IRI_DOCUMENT = "ontology.iri.document";
    private Concept imageConcept;
    private Concept audioConcept;
    private Concept videoConcept;
    private Concept documentConcept;

    @Override
    public void prepare(GraphPropertyWorkerPrepareData workerPrepareData) throws Exception {
        super.prepare(workerPrepareData);

        String imageIri = getConfiguration().get(CONFIG_ONTOLOGY_IRI_IMAGE);
        if (imageIri != null) {
            imageConcept = getOntologyRepository().getConceptByIRI(imageIri);
            checkNotNull(imageConcept, "Could not find concept (" + CONFIG_ONTOLOGY_IRI_IMAGE + ")" + imageIri);
        }

        String audioIri = getConfiguration().get(CONFIG_ONTOLOGY_IRI_AUDIO);
        if (audioIri != null) {
            audioConcept = getOntologyRepository().getConceptByIRI(audioIri);
            checkNotNull(audioConcept, "Could not find concept (" + CONFIG_ONTOLOGY_IRI_AUDIO + ")" + audioIri);
        }

        String videoIri = getConfiguration().get(CONFIG_ONTOLOGY_IRI_VIDEO);
        if (videoIri != null) {
            videoConcept = getOntologyRepository().getConceptByIRI(videoIri);
            checkNotNull(videoConcept, "Could not find concept (" + CONFIG_ONTOLOGY_IRI_VIDEO + ")" + videoIri);
        }

        String documentIri = getConfiguration().get(CONFIG_ONTOLOGY_IRI_DOCUMENT);
        if (documentIri != null) {
            documentConcept = getOntologyRepository().getConceptByIRI(documentIri);
            checkNotNull(documentConcept, "Could not find concept (" + CONFIG_ONTOLOGY_IRI_DOCUMENT + ")" + documentIri);
        }
    }

    @Override
    public void execute(InputStream in, GraphPropertyWorkData data) throws Exception {
        String mimeType = LumifyProperties.MIME_TYPE.getPropertyValue(data.getElement());
        Concept concept = null;

        if (imageConcept != null && mimeType.startsWith("image")) {
            concept = imageConcept;
        } else if (audioConcept != null && mimeType.startsWith("audio")) {
            concept = audioConcept;
        } else if (videoConcept != null && mimeType.startsWith("video")) {
            concept = videoConcept;
        } else if (documentConcept != null) {
            concept = documentConcept;
        }

        if (concept == null) {
            LOGGER.debug("skipping, no concept mapped for vertex " + data.getElement().getId());
            return;
        }

        LOGGER.debug("assigning concept type %s to vertex %s", concept.getTitle(), data.getElement().getId());
        LumifyProperties.CONCEPT_TYPE.setProperty(data.getElement(), concept.getTitle(), data.createPropertyMetadata(), data.getVisibility(), getAuthorizations());
        getGraph().flush();
        getWorkQueueRepository().pushGraphPropertyQueue(data.getElement(), null, LumifyProperties.CONCEPT_TYPE.getPropertyName());
    }

    @Override
    public boolean isHandled(Element element, Property property) {
        if (property == null) {
            return false;
        }

        if (!property.getName().equals(LumifyProperties.RAW.getPropertyName())) {
            return false;
        }

        String mimeType = LumifyProperties.MIME_TYPE.getPropertyValue(element);
        return mimeType != null;
    }
}<commit_msg>don't assign mime types to vertices with an existing mime type<commit_after>package io.lumify.mimeTypeOntologyMapper;

import io.lumify.core.ingest.graphProperty.GraphPropertyWorkData;
import io.lumify.core.ingest.graphProperty.GraphPropertyWorker;
import io.lumify.core.ingest.graphProperty.GraphPropertyWorkerPrepareData;
import io.lumify.core.model.ontology.Concept;
import io.lumify.core.model.properties.LumifyProperties;
import io.lumify.core.util.LumifyLogger;
import io.lumify.core.util.LumifyLoggerFactory;
import org.securegraph.Element;
import org.securegraph.Property;

import java.io.InputStream;

import static org.securegraph.util.Preconditions.checkNotNull;

public class MimeTypeOntologyMapperGraphPropertyWorker extends GraphPropertyWorker {
    private static final LumifyLogger LOGGER = LumifyLoggerFactory.getLogger(MimeTypeOntologyMapperGraphPropertyWorker.class);
    public static final String CONFIG_ONTOLOGY_IRI_IMAGE = "ontology.iri.image";
    public static final String CONFIG_ONTOLOGY_IRI_AUDIO = "ontology.iri.audio";
    public static final String CONFIG_ONTOLOGY_IRI_VIDEO = "ontology.iri.video";
    public static final String CONFIG_ONTOLOGY_IRI_DOCUMENT = "ontology.iri.document";
    private Concept imageConcept;
    private Concept audioConcept;
    private Concept videoConcept;
    private Concept documentConcept;

    @Override
    public void prepare(GraphPropertyWorkerPrepareData workerPrepareData) throws Exception {
        super.prepare(workerPrepareData);

        String imageIri = getConfiguration().get(CONFIG_ONTOLOGY_IRI_IMAGE);
        if (imageIri != null) {
            imageConcept = getOntologyRepository().getConceptByIRI(imageIri);
            checkNotNull(imageConcept, "Could not find concept (" + CONFIG_ONTOLOGY_IRI_IMAGE + ")" + imageIri);
        }

        String audioIri = getConfiguration().get(CONFIG_ONTOLOGY_IRI_AUDIO);
        if (audioIri != null) {
            audioConcept = getOntologyRepository().getConceptByIRI(audioIri);
            checkNotNull(audioConcept, "Could not find concept (" + CONFIG_ONTOLOGY_IRI_AUDIO + ")" + audioIri);
        }

        String videoIri = getConfiguration().get(CONFIG_ONTOLOGY_IRI_VIDEO);
        if (videoIri != null) {
            videoConcept = getOntologyRepository().getConceptByIRI(videoIri);
            checkNotNull(videoConcept, "Could not find concept (" + CONFIG_ONTOLOGY_IRI_VIDEO + ")" + videoIri);
        }

        String documentIri = getConfiguration().get(CONFIG_ONTOLOGY_IRI_DOCUMENT);
        if (documentIri != null) {
            documentConcept = getOntologyRepository().getConceptByIRI(documentIri);
            checkNotNull(documentConcept, "Could not find concept (" + CONFIG_ONTOLOGY_IRI_DOCUMENT + ")" + documentIri);
        }
    }

    @Override
    public void execute(InputStream in, GraphPropertyWorkData data) throws Exception {
        String mimeType = LumifyProperties.MIME_TYPE.getPropertyValue(data.getElement());
        Concept concept = null;

        if (imageConcept != null && mimeType.startsWith("image")) {
            concept = imageConcept;
        } else if (audioConcept != null && mimeType.startsWith("audio")) {
            concept = audioConcept;
        } else if (videoConcept != null && mimeType.startsWith("video")) {
            concept = videoConcept;
        } else if (documentConcept != null) {
            concept = documentConcept;
        }

        if (concept == null) {
            LOGGER.debug("skipping, no concept mapped for vertex " + data.getElement().getId());
            return;
        }

        LOGGER.debug("assigning concept type %s to vertex %s", concept.getTitle(), data.getElement().getId());
        LumifyProperties.CONCEPT_TYPE.setProperty(data.getElement(), concept.getTitle(), data.createPropertyMetadata(), data.getVisibility(), getAuthorizations());
        getGraph().flush();
        getWorkQueueRepository().pushGraphPropertyQueue(data.getElement(), null, LumifyProperties.CONCEPT_TYPE.getPropertyName());
    }

    @Override
    public boolean isHandled(Element element, Property property) {
        if (property == null) {
            return false;
        }

        if (!property.getName().equals(LumifyProperties.RAW.getPropertyName())) {
            return false;
        }

        String mimeType = LumifyProperties.MIME_TYPE.getPropertyValue(element);
        if (mimeType == null) {
            return false;
        }

        String existingConceptType = LumifyProperties.CONCEPT_TYPE.getPropertyValue(element);
        if (existingConceptType != null) {
            return false;
        }

        return true;
    }
}
{'fraction_non_alphanumeric': 0.09729309962280897, 'fraction_numerical': 0.0, 'mean_word_length': 3.5278754394776493, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7398002', 'n_tokens_mistral': 2589, 'n_tokens_neox': 2535, 'n_words': 486}