<commit_before>package org.phema.executer.translator;

import com.typesafe.config.Config;
import com.typesafe.config.ConfigObject;
import org.phema.executer.exception.PhemaUserException;
import org.phema.executer.hqmf.v2.DataCriteria;
import org.phema.executer.hqmf.v2.Range;
import org.phema.executer.hqmf.v2.TemporalReference;
import org.phema.executer.hqmf.v2.Value;
import org.phema.executer.i2b2.OntologyService;
import org.phema.executer.i2b2.ProjectManagementService;
import org.phema.executer.models.i2b2.Concept;
import org.phema.executer.models.i2b2.I2b2OverrideRule;
import org.phema.executer.models.i2b2.I2b2TerminologyRule;
import org.phema.executer.models.i2b2.I2b2ValueSetRule;
import org.phema.executer.util.ConfigHelper;
import org.phema.executer.valueSets.models.Member;
import org.phema.executer.valueSets.models.ValueSet;

import java.util.*;
import java.util.stream.Collectors;

/**
 * Created by <NAME> on 12/22/17.
 */
public class ValueSetToI2b2Ontology {
    private static final String DEFAULT_DELIMITER = ":";

    private ArrayList<I2b2TerminologyRule> terminologyRules;
    private ArrayList<I2b2ValueSetRule> valueSetRules;
    private ArrayList<I2b2OverrideRule> overrideRules;
    private OntologyService ontService;

    public class TranslationResult {
        public ArrayList<Member> UnmappedMembers;
        public HashMap<Member, List<Concept>> MappedMembers;
        public HashMap<Member, List<Concept>> FilteredOutMembers;
        public ArrayList<Concept> DistinctMappedConcepts;

        public TranslationResult() {
            UnmappedMembers = new ArrayList<>();
            MappedMembers = new HashMap<>();
            FilteredOutMembers = new HashMap<>();
            DistinctMappedConcepts = new ArrayList<>();
        }
    }

    private class BasecodeRuleMatch {
        public String basecode;
        public I2b2TerminologyRule rule;

        public BasecodeRuleMatch(String basecode, I2b2TerminologyRule rule) {
            this.basecode = basecode;
            this.rule = rule;
        }
    }

    /**
     * Initialize the value set translator
     * @param config
     * @throws Exception
     */
    public void initialize(Config config, OntologyService ontService) throws Exception {
        this.ontService = ontService;
        initializeTerminologyRules(config);
        initializeValueSetRules(config);
        initializeOverrideRules(config);
    }

    public TranslationResult translate(ValueSet valueSet) throws PhemaUserException {
        if (ontService == null) {
            throw new PhemaUserException("No active i2b2 instance was provided.  Please make sure that your username and password work correctly with the configured i2b2 instance.");
        }

        ArrayList<Member> members = valueSet.getMembers();
        ArrayList<Concept> concepts = new ArrayList<>();
        TranslationResult result = new TranslationResult();
        for (Member member : members) {
            BasecodeRuleMatch basecodeRule = translateValueSetMemberToBasecode(member);
            ArrayList<ArrayList<Concept>> filterResults = filterFoundConcepts(basecodeRule);
            ArrayList<Concept> foundConcepts = filterResults.get(0);
            ArrayList<Concept> filteredOutConcepts = filterResults.get(1);
            if (foundConcepts.isEmpty()) {
                result.UnmappedMembers.add(member);
            }
            else {
                result.MappedMembers.put(member, foundConcepts);
                concepts.addAll(foundConcepts);
            }

            // We separately track any concepts that were filtered out so that we can
            // report to the user if their filter is perhaps too restrictive.  Note
            // that this is only set if a filter is defined.
            if (!filteredOutConcepts.isEmpty()) {
                result.FilteredOutMembers.put(member, filteredOutConcepts);
            }
        }

        result.DistinctMappedConcepts = distinctConceptList(concepts);
        return result;
    }

    public ArrayList<Concept> translateAgeRequirement(DataCriteria criterion) throws PhemaUserException {
        ArrayList<Concept> ageConcepts = new ArrayList<>();

        if (!Objects.equals(criterion.getDefinition(), "patient_characteristic_birthdate")) {
            throw new PhemaUserException("A non-patient age criteria was treated as an age requirement.  This may be an internal error within the PhEMA Executer, or may be caused by an error in your HQMF file.");
        }

        TemporalReference temporalReference = criterion.getTemporalReferences().get(0);
        Range range = temporalReference.getRange();
        Value lowRange = null;
        Value highRange = null;
        if (range.getLow() != null && range.getLow().getClass().equals(Value.class)) {
            lowRange = (Value)range.getLow();
        }
        if (range.getHigh() != null && range.getHigh().getClass().equals(Value.class)) {
            highRange = (Value)range.getHigh();
        }<commit_msg>Closes #6 only show distinct value set codes<commit_after>package org.phema.executer.translator;

import com.typesafe.config.Config;
import com.typesafe.config.ConfigObject;
import org.phema.executer.exception.PhemaUserException;
import org.phema.executer.hqmf.v2.DataCriteria;
import org.phema.executer.hqmf.v2.Range;
import org.phema.executer.hqmf.v2.TemporalReference;
import org.phema.executer.hqmf.v2.Value;
import org.phema.executer.i2b2.OntologyService;
import org.phema.executer.i2b2.ProjectManagementService;
import org.phema.executer.models.i2b2.Concept;
import org.phema.executer.models.i2b2.I2b2OverrideRule;
import org.phema.executer.models.i2b2.I2b2TerminologyRule;
import org.phema.executer.models.i2b2.I2b2ValueSetRule;
import org.phema.executer.util.ConfigHelper;
import org.phema.executer.valueSets.models.Member;
import org.phema.executer.valueSets.models.ValueSet;

import java.util.*;
import java.util.stream.Collectors;

/**
 * Created by <NAME> on 12/22/17.
 */
public class ValueSetToI2b2Ontology {
    private static final String DEFAULT_DELIMITER = ":";

    private ArrayList<I2b2TerminologyRule> terminologyRules;
    private ArrayList<I2b2ValueSetRule> valueSetRules;
    private ArrayList<I2b2OverrideRule> overrideRules;
    private OntologyService ontService;

    public class TranslationResult {
        public ArrayList<Member> UnmappedMembers;
        public HashMap<Member, List<Concept>> MappedMembers;
        public HashMap<Member, List<Concept>> FilteredOutMembers;
        public ArrayList<Concept> DistinctMappedConcepts;

        public TranslationResult() {
            UnmappedMembers = new ArrayList<>();
            MappedMembers = new HashMap<>();
            FilteredOutMembers = new HashMap<>();
            DistinctMappedConcepts = new ArrayList<>();
        }
    }

    private class BasecodeRuleMatch {
        public String basecode;
        public I2b2TerminologyRule rule;

        public BasecodeRuleMatch(String basecode, I2b2TerminologyRule rule) {
            this.basecode = basecode;
            this.rule = rule;
        }
    }

    /**
     * Initialize the value set translator
     * @param config
     * @throws Exception
     */
    public void initialize(Config config, OntologyService ontService) throws Exception {
        this.ontService = ontService;
        initializeTerminologyRules(config);
        initializeValueSetRules(config);
        initializeOverrideRules(config);
    }

    public TranslationResult translate(ValueSet valueSet) throws PhemaUserException {
        if (ontService == null) {
            throw new PhemaUserException("No active i2b2 instance was provided.  Please make sure that your username and password work correctly with the configured i2b2 instance.");
        }

        ArrayList<Member> members = valueSet.getMembers();
        ArrayList<Concept> concepts = new ArrayList<>();
        TranslationResult result = new TranslationResult();
        for (Member member : members) {
            BasecodeRuleMatch basecodeRule = translateValueSetMemberToBasecode(member);
            ArrayList<ArrayList<Concept>> filterResults = filterFoundConcepts(basecodeRule);
            ArrayList<Concept> foundConcepts = filterResults.get(0);
            ArrayList<Concept> filteredOutConcepts = filterResults.get(1);
            if (foundConcepts.isEmpty() && !result.UnmappedMembers.contains(member)) {
                result.UnmappedMembers.add(member);
            }
            else {
                result.MappedMembers.put(member, distinctConceptList(foundConcepts));
                concepts.addAll(foundConcepts);
            }

            // We separately track any concepts that were filtered out so that we can
            // report to the user if their filter is perhaps too restrictive.  Note
            // that this is only set if a filter is defined.
            if (!filteredOutConcepts.isEmpty()) {
                result.FilteredOutMembers.put(member, distinctConceptList(filteredOutConcepts));
            }
        }

        result.DistinctMappedConcepts = distinctConceptList(concepts);
        return result;
    }

    public ArrayList<Concept> translateAgeRequirement(DataCriteria criterion) throws PhemaUserException {
        ArrayList<Concept> ageConcepts = new ArrayList<>();

        if (!Objects.equals(criterion.getDefinition(), "patient_characteristic_birthdate")) {
            throw new PhemaUserException("A non-patient age criteria was treated as an age requirement.  This may be an internal error within the PhEMA Executer, or may be caused by an error in your HQMF file.");
        }

        TemporalReference temporalReference = criterion.getTemporalReferences().get(0);
        Range range = temporalReference.getRange();
        Value lowRange = null;
        Value highRange = null;
        if (range.getLow() != null && range.getLow().getClass().equals(Value.class)) {
            lowRange = (Value)range.getLow();
        }
        if (range.getHigh() != null && range.getHigh().getClass().equals(Value.class)) {
            highRange = (Value)range.getHigh();
        }
{'fraction_non_alphanumeric': 0.08974992586735198, 'fraction_numerical': 0.009390135415637046, 'mean_word_length': 3.0963562753036435, 'pattern_counts': {'":': 2, '<': 51, '<?xml version=': 0, '>': 51, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1794385', 'n_tokens_mistral': 2860, 'n_tokens_neox': 2743, 'n_words': 634}