<commit_before>package com.epimorphics.registry.store;

import com.epimorphics.appbase.webapi.WebApiException;
import com.epimorphics.registry.core.Description;
import com.epimorphics.registry.core.RegisterItem;
import org.apache.commons.lang.text.StrBuilder;
import org.apache.jena.query.ParameterizedSparqlString;
import org.apache.jena.query.Query;
import org.apache.jena.query.QueryFactory;
import org.apache.jena.rdf.model.*;
import org.apache.jena.riot.system.IRIResolver;
import org.apache.jena.sparql.path.Path;
import org.apache.jena.sparql.path.PathFactory;

import javax.ws.rs.core.Response;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.Objects;
import java.util.stream.Stream;

import static java.util.stream.Collectors.joining;

/**
 * Instantiates SPARQL queries that locate and classify registry entries which are
 * similar to a given set of resources, based on their indexed text properties.
 */
class FindSimilar {

    /**
     * Instantiate this class.
     * @param store A store which the queries generated by the returned instance should be run against.
     * @return A new instance.
     */
    static FindSimilar newInstance(Store store) {
        return new FindSimilar(store.getIndexedProperties());
    }

    private final List<Property> indexProps;
    private final Path textPath;

    private FindSimilar(List<Property> indexProps) {
        this.indexProps = indexProps;
        this.textPath = getTextPath(indexProps);
    }

    private Path getTextPath(List<Property> indexProps) {
        return indexProps.stream()
                .map(Property::asNode)
                .map(PathFactory::pathLink)
                .reduce(PathFactory::pathAlt).orElseThrow(() ->
                        new WebApiException(Response.Status.BAD_REQUEST, "Unable to find similar entries: a text index is required!")
                );
    }

    private String getValuesRow(RegisterItem item, String text, Double similarity) {
        String uri = item.getRoot().getURI();
        if (IRIResolver.checkIRI(uri)) {
            throw new WebApiException(Response.Status.BAD_REQUEST, "Unable to find similar entries: Resource URI " + uri + " is not valid.");
        }

        String query = Arrays.stream(text.split(" "))
                .map(word -> word + "~" + String.format("%.1f", similarity))
                .collect(joining(" AND ", "(", ")"));

        ParameterizedSparqlString pss = new ParameterizedSparqlString("( ?resource ?query ?text ?isRegister )");
        pss.setIri("resource", uri);
        pss.setLiteral("query", query);
        pss.setLiteral("text", text);
        pss.setLiteral("isRegister", item.isRegister());

        return pss.toString();
    }

    private Resource getResource(Description desc) {
        if (desc.isRegisterItem()) {
            return desc.asRegisterItem().getEntity();
        } else {
            return desc.asRegister().getRegister();
        }
    }

    private Stream<String> getValuesRows(Collection<RegisterItem> items, Double similarity) {
        return items.stream().flatMap(item -> {
            Resource resource = getResource(item);
            Stream<Statement> statements = indexProps.stream().map(resource::getProperty).filter(Objects::nonNull);
            return statements
                    .map(Statement::getObject)<commit_msg>ADD Support for multi valued text properties<commit_after>package com.epimorphics.registry.store;

import com.epimorphics.appbase.webapi.WebApiException;
import com.epimorphics.registry.core.Description;
import com.epimorphics.registry.core.RegisterItem;
import org.apache.commons.lang.text.StrBuilder;
import org.apache.jena.query.ParameterizedSparqlString;
import org.apache.jena.query.Query;
import org.apache.jena.query.QueryFactory;
import org.apache.jena.rdf.model.*;
import org.apache.jena.riot.system.IRIResolver;
import org.apache.jena.sparql.path.Path;
import org.apache.jena.sparql.path.PathFactory;

import javax.ws.rs.core.Response;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.Objects;
import java.util.stream.Stream;

import static java.util.stream.Collectors.joining;

/**
 * Instantiates SPARQL queries that locate and classify registry entries which are
 * similar to a given set of resources, based on their indexed text properties.
 */
class FindSimilar {

    /**
     * Instantiate this class.
     * @param store A store which the queries generated by the returned instance should be run against.
     * @return A new instance.
     */
    static FindSimilar newInstance(Store store) {
        return new FindSimilar(store.getIndexedProperties());
    }

    private final List<Property> indexProps;
    private final Path textPath;

    private FindSimilar(List<Property> indexProps) {
        this.indexProps = indexProps;
        this.textPath = getTextPath(indexProps);
    }

    private Path getTextPath(List<Property> indexProps) {
        return indexProps.stream()
                .map(Property::asNode)
                .map(PathFactory::pathLink)
                .reduce(PathFactory::pathAlt).orElseThrow(() ->
                        new WebApiException(Response.Status.BAD_REQUEST, "Unable to find similar entries: a text index is required!")
                );
    }

    private String getValuesRow(RegisterItem item, String text, Double similarity) {
        String uri = item.getRoot().getURI();
        if (IRIResolver.checkIRI(uri)) {
            throw new WebApiException(Response.Status.BAD_REQUEST, "Unable to find similar entries: Resource URI " + uri + " is not valid.");
        }

        String query = Arrays.stream(text.split(" "))
                .map(word -> word + "~" + String.format("%.1f", similarity))
                .collect(joining(" AND ", "(", ")"));

        ParameterizedSparqlString pss = new ParameterizedSparqlString("( ?resource ?query ?text ?isRegister )");
        pss.setIri("resource", uri);
        pss.setLiteral("query", query);
        pss.setLiteral("text", text);
        pss.setLiteral("isRegister", item.isRegister());

        return pss.toString();
    }

    private Resource getResource(Description desc) {
        if (desc.isRegisterItem()) {
            return desc.asRegisterItem().getEntity();
        } else {
            return desc.asRegister().getRegister();
        }
    }

    private Stream<String> getValuesRows(Collection<RegisterItem> items, Double similarity) {
        return items.stream().flatMap(item -> {
            Resource resource = getResource(item);
            Stream<Statement> statements = indexProps.stream()
                    .map(resource::listProperties)
                    .map(StmtIterator::toList)
                    .flatMap(List::stream)
                    .filter(Objects::nonNull);

            return statements
                    .map(Statement::getObject)
{'fraction_non_alphanumeric': 0.11141658226016496, 'fraction_numerical': 0.0002893937201562726, 'mean_word_length': 2.9272727272727272, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2890786', 'n_tokens_mistral': 1928, 'n_tokens_neox': 1861, 'n_words': 435}