<commit_before>package nl.mpi.kinnate.kintypestrings;

import nl.mpi.kinnate.kindata.GraphSorter;
import nl.mpi.kinnate.kindata.EntityData;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import nl.mpi.kinnate.kindata.DataTypes;
import nl.mpi.kinnate.kindata.EntityData.SymbolType;
import nl.mpi.kinnate.kindata.EntityRelation;
import nl.mpi.kinnate.kintypestrings.ParserHighlight.ParserHighlightType;
import nl.mpi.kinnate.svg.DataStoreSvg;
import nl.mpi.kinnate.uniqueidentifiers.UniqueIdentifier;

/**
 *  Document   : KinTypeStringConverter
 *  Created on : Sep 29, 2010, 12:52:33 PM
 *  Author     : <NAME>
 */
public class KinTypeStringConverter extends GraphSorter {

    DataStoreSvg dataStoreSvg;

    public KinTypeStringConverter(DataStoreSvg dataStoreSvg) {
        this.dataStoreSvg = dataStoreSvg;
    }

    public enum QueryType {

        Contains, Greater, Less, Equals
    }

    public class QueryTerm {

        public QueryTerm(String fieldXPath, QueryType comparatorType, String searchValue) {
            this.fieldXPath = fieldXPath;
            this.comparatorType = comparatorType;
            this.searchValue = searchValue;
        }
        public String fieldXPath;
        public QueryType comparatorType;
        public String searchValue;
    }

    public class KinTypeElement {

        public KinTypeElement() {
            entityData = new ArrayList<EntityData>();
        }
        public KinTypeElement prevType;
        public KinTypeElement nextType;
        public KinType kinType;
        public ArrayList<QueryTerm> queryTerms;
        public ArrayList<EntityData> entityData; // there may be multiple entities for each kin term
        ParserHighlight[] highlightLocs;
    }

    public void setEgoKinTypeString(EntityData entityData) {
        for (KinType kinType : dataStoreSvg.getKinTypeDefinitions()) {
            if (kinType.isEgoType() && kinType.matchesEgonessAndSymbol(entityData, null)) { // todo: should this be passing a kin type modifier?
                entityData.addKinTypeString(kinType.codeString);
            }
        }
    }
//    public void highlightComments(String[] kinTypeStrings, ParserHighlight[][] parserHighlight) {
//        int lineCounter = 0;
//        for (String currentString : kinTypeStrings) {
//            parserHighlight[lineCounter] = new ParserHighlight[currentString.length()];
//            ParserHighlight currentHighlight = ParserHighlight.Unknown;
//            for (int charCounter = 0; charCounter < currentString.length(); charCounter++) {
//                if (currentHighlight != ParserHighlight.Comment && currentString.charAt(charCounter) == '#') {
//                    currentHighlight = ParserHighlight.Comment;
//                }
//                parserHighlight[lineCounter][charCounter] = currentHighlight;
//            }
//
//            lineCounter++;
//        }
//    }

//    @Deprecated
//    public boolean compareRequiresNextRelation(EntityData adjacentEntity, KinType requiredKinType, EntityRelation entityRelation) {
////        adjacentEntity.appendTempLabel("compareRequiresNextRelation" + "F: " + QueryParser.foundOrder++); // temp for testing // todo: remove testing labels
//        if (adjacentEntity.getSymbolType().equals(EntityData.SymbolType.union.name())) {
//            return true;
//        }
//        // todo: continue here....
//        if (requiredKinType.relationType.equals(DataTypes.RelationType.sibling) && entityRelation.relationType.equals(DataTypes.RelationType.ancestor)) {
//            return true;
//        }
//        return false;
//    }
//    @Deprecated
//    public boolean compareRelationsToKinType(EntityData egoEntity, EntityData alterEntity, KinType requiredKinType, EntityRelation entityRelation, int generationalDistance) {
////        egoEntity.appendTempLabel("compareRelationsToKinType-egoEntity" + "F: " + QueryParser.foundOrder++);
////        alterEntity.appendTempLabel("compareRelationsToKinType-alterEntity" + "F: " + QueryParser.foundOrder++);
//        // temp for testing // todo: remove testing labels
////        System.out.println("egoEntity.isEgo: " + egoEntity.isEgo);
////        System.out.println("alterEntity.isEgo: " + alterEntity.isEgo);
////        System.out.println("egoEntity.symbol: " + egoEntity.getSymbolType());
////        System.out.println("alterEntity.symbol: " + alterEntity.getSymbolType());
////        System.out.println("entityRelation.relationType: " + entityRelation.relationType);
////        System.out.println("entityRelation.symgenerationalDistancebol: " + entityRelation.generationalDistance);
//        // note that this will get the kin type reversed for one of the adjacent entities and this must be accounted for in the kin type comparison
//        // todo: note that the ego and alter are not correct labels
//        // this array will get the kin type reversed for one of the adjacent entities
//        if (egoEntity.isEgo && requiredKinType.isEgoType()) {// && alter.getSymbolType().equals(EntityData.SymbolType.triangle.name())) {
//            return true;
//        }
//        if (requiredKinType.relationType.equals(entityRelation.relationType)
//                && requiredKinType.symbolType.name().equals(alterEntity.getSymbolType())) {
//            return true;
//        }
//        return false;
//    }
    public ArrayList<KinTypeElement> getKinTypeElements(String consumableString, ParserHighlight parserHighlight) {
        int initialLength = consumableString.length();
        if (consumableString.startsWith("[")) {
            // todo: this is added so that a query can start with a [ since the initial kin type is redundant, however the addition of x= causes syntax highlighing issues partly because the ParserHighlight always creates an empty highlight ahead of the current one, but also it would be better to not be modifying this string
            consumableString = "x=" + consumableString;
        }
        ArrayList<KinTypeElement> kinTypeElementList = new ArrayList<KinTypeElement>();
        KinTypeElement previousElement = null;
        boolean foundKinType = true;
        String errorMessage = null;
        while (foundKinType && consumableString.length() > 0) {
            for (KinType currentReferenceKinType : dataStoreSvg.getKinTypeDefinitions()) {
                foundKinType = false;
                if (consumableString.startsWith(currentReferenceKinType.codeString)) {
                    parserHighlight = parserHighlight.addHighlight(ParserHighlightType.KinType, initialLength - consumableString.length(), currentReferenceKinType.displayString);
                    KinTypeElement currentElement = new KinTypeElement();
                    if (previousElement != null) {
                        previousElement.nextType = currentElement;
                        currentElement.prevType = previousElement;
                    }
                    previousElement = currentElement;
                    currentElement.kinType = currentReferenceKinType;
                    consumableString = consumableString.substring(currentReferenceKinType.codeString.length());

                    if (consumableString.startsWith("[")) {
                        // todo: Ticket #1087 Multiple query terms should be possible in the kin type string queries. Eg: Ef=[Kundarr]PM=[Louise] (Based on Joe's data).
                        int highlightPosition = initialLength - consumableString.length();
                        String highlightMessage = "Query: ";
//                        consumableString = consumableString.substring("=".length());
                        while (consumableString.startsWith("[")) {
                            // todo: allow multiple terms such as "=[foo][bar]" or "=[foo][bar][NAME=Bob]"
                            int queryStart = "[".length();
                            int queryEnd = consumableString.indexOf("]");
                            if (queryEnd == -1) {
                                // if the terms are incomplete then ignore the rest of the line
                                highlightMessage += "No closing bracket ']' found";
                                errorMessage = highlightMessage;
                                foundKinType = false;
                                break;
                            }
                            if (queryEnd - queryStart < 3) {
                                // the query string must be more than 2 chars
                                highlightMessage += "Query must be over 2 chars long";
                                errorMessage = highlightMessage;
                                foundKinType = false;
                                break;
                            }
                            if (currentElement.queryTerms == null) {
                                currentElement.queryTerms = new ArrayList<QueryTerm>();
                            }
                            String queryText = consumableString.substring(queryStart, queryEnd);
                            consumableString = consumableString.substring(queryEnd + 1);
                            String[] queryTerm;
                            QueryType currentQueryType = null;
                            queryTerm = queryText.split("=="); // detect which comparitor is used
                            if (queryTerm.length > 1) {
                                currentQueryType = QueryType.Equals;
                            } else {
                                queryTerm = queryText.split("=");
                                if (queryTerm.length > 1) {
                                    currentQueryType = QueryType.Contains;
                                } else {
                                    queryTerm = queryText.split("\\>");
                                    if (queryTerm.length > 1) {
                                        currentQueryType = QueryType.Greater;
                                    } else {
                                        queryTerm = queryText.split("\\<");
                                        if (queryTerm.length > 1) {
                                            currentQueryType = QueryType.Less;
                                        }
                                    }
                                }
                            }

                            if (currentQueryType == null) {
                                currentElement.queryTerms.add(new QueryTerm("*", QueryType.Contains, queryText));
                                highlightMessage += "Any field containing '" + queryText + "'";
                            } else {
                                if (queryTerm[0].length() > 2 && queryTerm[1].length() > 2) {
                                    // namespace wild cards *:* are inserted here so that the user does not need to specify the namespace
                                    currentElement.queryTerms.add(new QueryTerm("*:" + queryTerm[0].replaceAll("\\.", "/*:"), currentQueryType, queryTerm[1]));
                                    highlightMessage += "Only the field '" + queryTerm[0] + "' containing '" + queryTerm[1] + "'";
                                }
                            }
                        }
                        parserHighlight = parserHighlight.addHighlight(ParserHighlightType.Query, highlightPosition, highlightMessage);
                    }
                    kinTypeElementList.add(currentElement);
                    foundKinType = true;
                    break;
                }
            }
        }
        if (!foundKinType && !consumableString.startsWith("#")) {
            parserHighlight = parserHighlight.addHighlight(ParserHighlightType.Error, initialLength - consumableString.length(), errorMessage);
        }
        if (consumableString.contains("#")) {
            // check for any comments
            parserHighlight = parserHighlight.addHighlight(ParserHighlightType.Comment, initialLength - consumableString.length() + consumableString.indexOf("#"), null);
        }
        return kinTypeElementList;
    }

    public ArrayList<KinType> getKinTypes(String consumableString) {
        ArrayList<KinType> kinTypeList = new ArrayList<KinType>();
        boolean foundKinType = true;
        while (foundKinType && consumableString.length() > 0) {
            for (KinType currentReferenceKinType : dataStoreSvg.getKinTypeDefinitions()) {<commit_msg>Reduced the limit on the minimum characters passed to a query when the field name is specified.<commit_after>package nl.mpi.kinnate.kintypestrings;

import nl.mpi.kinnate.kindata.GraphSorter;
import nl.mpi.kinnate.kindata.EntityData;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import nl.mpi.kinnate.kindata.DataTypes;
import nl.mpi.kinnate.kindata.EntityData.SymbolType;
import nl.mpi.kinnate.kindata.EntityRelation;
import nl.mpi.kinnate.kintypestrings.ParserHighlight.ParserHighlightType;
import nl.mpi.kinnate.svg.DataStoreSvg;
import nl.mpi.kinnate.uniqueidentifiers.UniqueIdentifier;

/**
 *  Document   : KinTypeStringConverter
 *  Created on : Sep 29, 2010, 12:52:33 PM
 *  Author     : <NAME>
 */
public class KinTypeStringConverter extends GraphSorter {

    DataStoreSvg dataStoreSvg;

    public KinTypeStringConverter(DataStoreSvg dataStoreSvg) {
        this.dataStoreSvg = dataStoreSvg;
    }

    public enum QueryType {

        Contains, Greater, Less, Equals
    }

    public class QueryTerm {

        public QueryTerm(String fieldXPath, QueryType comparatorType, String searchValue) {
            this.fieldXPath = fieldXPath;
            this.comparatorType = comparatorType;
            this.searchValue = searchValue;
        }
        public String fieldXPath;
        public QueryType comparatorType;
        public String searchValue;
    }

    public class KinTypeElement {

        public KinTypeElement() {
            entityData = new ArrayList<EntityData>();
        }
        public KinTypeElement prevType;
        public KinTypeElement nextType;
        public KinType kinType;
        public ArrayList<QueryTerm> queryTerms;
        public ArrayList<EntityData> entityData; // there may be multiple entities for each kin term
        ParserHighlight[] highlightLocs;
    }

    public void setEgoKinTypeString(EntityData entityData) {
        for (KinType kinType : dataStoreSvg.getKinTypeDefinitions()) {
            if (kinType.isEgoType() && kinType.matchesEgonessAndSymbol(entityData, null)) { // todo: should this be passing a kin type modifier?
                entityData.addKinTypeString(kinType.codeString);
            }
        }
    }
//    public void highlightComments(String[] kinTypeStrings, ParserHighlight[][] parserHighlight) {
//        int lineCounter = 0;
//        for (String currentString : kinTypeStrings) {
//            parserHighlight[lineCounter] = new ParserHighlight[currentString.length()];
//            ParserHighlight currentHighlight = ParserHighlight.Unknown;
//            for (int charCounter = 0; charCounter < currentString.length(); charCounter++) {
//                if (currentHighlight != ParserHighlight.Comment && currentString.charAt(charCounter) == '#') {
//                    currentHighlight = ParserHighlight.Comment;
//                }
//                parserHighlight[lineCounter][charCounter] = currentHighlight;
//            }
//
//            lineCounter++;
//        }
//    }

//    @Deprecated
//    public boolean compareRequiresNextRelation(EntityData adjacentEntity, KinType requiredKinType, EntityRelation entityRelation) {
////        adjacentEntity.appendTempLabel("compareRequiresNextRelation" + "F: " + QueryParser.foundOrder++); // temp for testing // todo: remove testing labels
//        if (adjacentEntity.getSymbolType().equals(EntityData.SymbolType.union.name())) {
//            return true;
//        }
//        // todo: continue here....
//        if (requiredKinType.relationType.equals(DataTypes.RelationType.sibling) && entityRelation.relationType.equals(DataTypes.RelationType.ancestor)) {
//            return true;
//        }
//        return false;
//    }
//    @Deprecated
//    public boolean compareRelationsToKinType(EntityData egoEntity, EntityData alterEntity, KinType requiredKinType, EntityRelation entityRelation, int generationalDistance) {
////        egoEntity.appendTempLabel("compareRelationsToKinType-egoEntity" + "F: " + QueryParser.foundOrder++);
////        alterEntity.appendTempLabel("compareRelationsToKinType-alterEntity" + "F: " + QueryParser.foundOrder++);
//        // temp for testing // todo: remove testing labels
////        System.out.println("egoEntity.isEgo: " + egoEntity.isEgo);
////        System.out.println("alterEntity.isEgo: " + alterEntity.isEgo);
////        System.out.println("egoEntity.symbol: " + egoEntity.getSymbolType());
////        System.out.println("alterEntity.symbol: " + alterEntity.getSymbolType());
////        System.out.println("entityRelation.relationType: " + entityRelation.relationType);
////        System.out.println("entityRelation.symgenerationalDistancebol: " + entityRelation.generationalDistance);
//        // note that this will get the kin type reversed for one of the adjacent entities and this must be accounted for in the kin type comparison
//        // todo: note that the ego and alter are not correct labels
//        // this array will get the kin type reversed for one of the adjacent entities
//        if (egoEntity.isEgo && requiredKinType.isEgoType()) {// && alter.getSymbolType().equals(EntityData.SymbolType.triangle.name())) {
//            return true;
//        }
//        if (requiredKinType.relationType.equals(entityRelation.relationType)
//                && requiredKinType.symbolType.name().equals(alterEntity.getSymbolType())) {
//            return true;
//        }
//        return false;
//    }
    public ArrayList<KinTypeElement> getKinTypeElements(String consumableString, ParserHighlight parserHighlight) {
        int initialLength = consumableString.length();
        if (consumableString.startsWith("[")) {
            // todo: this is added so that a query can start with a [ since the initial kin type is redundant, however the addition of x= causes syntax highlighing issues partly because the ParserHighlight always creates an empty highlight ahead of the current one, but also it would be better to not be modifying this string
            consumableString = "x=" + consumableString;
        }
        ArrayList<KinTypeElement> kinTypeElementList = new ArrayList<KinTypeElement>();
        KinTypeElement previousElement = null;
        boolean foundKinType = true;
        String errorMessage = null;
        while (foundKinType && consumableString.length() > 0) {
            for (KinType currentReferenceKinType : dataStoreSvg.getKinTypeDefinitions()) {
                foundKinType = false;
                if (consumableString.startsWith(currentReferenceKinType.codeString)) {
                    parserHighlight = parserHighlight.addHighlight(ParserHighlightType.KinType, initialLength - consumableString.length(), currentReferenceKinType.displayString);
                    KinTypeElement currentElement = new KinTypeElement();
                    if (previousElement != null) {
                        previousElement.nextType = currentElement;
                        currentElement.prevType = previousElement;
                    }
                    previousElement = currentElement;
                    currentElement.kinType = currentReferenceKinType;
                    consumableString = consumableString.substring(currentReferenceKinType.codeString.length());

                    if (consumableString.startsWith("[")) {
                        // todo: Ticket #1087 Multiple query terms should be possible in the kin type string queries. Eg: Ef=[Kundarr]PM=[Louise] (Based on Joe's data).
                        int highlightPosition = initialLength - consumableString.length();
                        String highlightMessage = "Query: ";
//                        consumableString = consumableString.substring("=".length());
                        while (consumableString.startsWith("[")) {
                            // todo: allow multiple terms such as "=[foo][bar]" or "=[foo][bar][NAME=Bob]"
                            int queryStart = "[".length();
                            int queryEnd = consumableString.indexOf("]");
                            if (queryEnd == -1) {
                                // if the terms are incomplete then ignore the rest of the line
                                highlightMessage += "No closing bracket ']' found";
                                errorMessage = highlightMessage;
                                foundKinType = false;
                                break;
                            }
                            if (queryEnd - queryStart < 3) {
                                // the query string must be more than 2 chars
                                highlightMessage += "Query must be over 2 chars long";
                                errorMessage = highlightMessage;
                                foundKinType = false;
                                break;
                            }
                            if (currentElement.queryTerms == null) {
                                currentElement.queryTerms = new ArrayList<QueryTerm>();
                            }
                            String queryText = consumableString.substring(queryStart, queryEnd);
                            consumableString = consumableString.substring(queryEnd + 1);
                            String[] queryTerm;
                            QueryType currentQueryType = null;
                            queryTerm = queryText.split("=="); // detect which comparitor is used
                            if (queryTerm.length > 1) {
                                currentQueryType = QueryType.Equals;
                            } else {
                                queryTerm = queryText.split("=");
                                if (queryTerm.length > 1) {
                                    currentQueryType = QueryType.Contains;
                                } else {
                                    queryTerm = queryText.split("\\>");
                                    if (queryTerm.length > 1) {
                                        currentQueryType = QueryType.Greater;
                                    } else {
                                        queryTerm = queryText.split("\\<");
                                        if (queryTerm.length > 1) {
                                            currentQueryType = QueryType.Less;
                                        }
                                    }
                                }
                            }

                            if (currentQueryType == null) {
                                currentElement.queryTerms.add(new QueryTerm("*", QueryType.Contains, queryText));
                                highlightMessage += "Any field containing '" + queryText + "'";
                            } else {
                                if (queryTerm[0].length() > 0 && queryTerm[1].length() > 0) {
                                    // namespace wild cards *:* are inserted here so that the user does not need to specify the namespace
                                    currentElement.queryTerms.add(new QueryTerm("*:" + queryTerm[0].replaceAll("\\.", "/*:"), currentQueryType, queryTerm[1]));
                                    highlightMessage += "Only the field '" + queryTerm[0] + "' containing '" + queryTerm[1] + "'";
                                }
                            }
                        }
                        parserHighlight = parserHighlight.addHighlight(ParserHighlightType.Query, highlightPosition, highlightMessage);
                    }
                    kinTypeElementList.add(currentElement);
                    foundKinType = true;
                    break;
                }
            }
        }
        if (!foundKinType && !consumableString.startsWith("#")) {
            parserHighlight = parserHighlight.addHighlight(ParserHighlightType.Error, initialLength - consumableString.length(), errorMessage);
        }
        if (consumableString.contains("#")) {
            // check for any comments
            parserHighlight = parserHighlight.addHighlight(ParserHighlightType.Comment, initialLength - consumableString.length() + consumableString.indexOf("#"), null);
        }
        return kinTypeElementList;
    }

    public ArrayList<KinType> getKinTypes(String consumableString) {
        ArrayList<KinType> kinTypeList = new ArrayList<KinType>();
        boolean foundKinType = true;
        while (foundKinType && consumableString.length() > 0) {
            for (KinType currentReferenceKinType : dataStoreSvg.getKinTypeDefinitions()) {
{'fraction_non_alphanumeric': 0.09419021090330283, 'fraction_numerical': 0.0029446876243533625, 'mean_word_length': 1.8135915808329601, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 40, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5110175', 'n_tokens_mistral': 6152, 'n_tokens_neox': 5751, 'n_words': 1360}