<commit_before>package org.opencds.cqf.cql.file.fhir;

import org.hl7.fhir.dstu3.model.CodeableConcept;
import org.hl7.fhir.dstu3.model.Coding;
import org.hl7.fhir.dstu3.model.DateTimeType;
import org.joda.time.Partial;
import org.opencds.cqf.cql.data.fhir.BaseFhirDataProvider;
import org.opencds.cqf.cql.elm.execution.InEvaluator;
import org.opencds.cqf.cql.elm.execution.IncludesEvaluator;
import org.opencds.cqf.cql.runtime.Code;
import org.opencds.cqf.cql.runtime.DateTime;
import org.opencds.cqf.cql.runtime.Interval;
import org.opencds.cqf.cql.terminology.ValueSetInfo;
import org.opencds.cqf.cql.terminology.fhir.FhirTerminologyProvider;

import com.fasterxml.jackson.databind.ObjectMapper;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.nio.file.InvalidPathException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;

/*
What the heck does this thing do?
  This class is intended to provide the user with the alternative of using a local repository
  for clinical data retrieval instead of an external service.
  NOTE: This class still uses a Terminology service for value set retrieval and evaluating code membership

How do I use it?
  Point the provider to the directory of patients:
    FileBasedFhirProvider provider = new FileBasedFhirProvider(path to root data dir, terminology service endpoint(optional));
    Each subfolder name in the patients directory should be an id for a specific patients
      In each patient folder there should be subfolders contatining clinical information
        (e.g. Condition, Procedure, etc...) for that patient in JSON format (XML support pending)
    Here is a mock directory structure:
    - patients
      - 123
        - Conditions
          - JSON representation of Fhir resources
        - Procedures
        - Encounters
        - etc...
      - 154
        - Observations
        - etc...
      - 209
      - etc...
*/

public class FileBasedFhirProvider extends BaseFhirDataProvider {

  private Path path;
  private FhirTerminologyProvider terminologyProvider;

  public FileBasedFhirProvider (String path, String endpoint) {
    if (path.isEmpty() || path == null) {
      throw new InvalidPathException(path, "Invalid path!");
    }
    this.path = Paths.get(path);
    this.terminologyProvider = endpoint == null ? new FhirTerminologyProvider().withEndpoint("http://fhirtest.uhn.ca/baseDstu3")
                                                : new FhirTerminologyProvider().withEndpoint(endpoint);
  }

  private Path pathToModelJar;
  public Path getpathToModelJar() {
    return pathToModelJar;
  }

  public void setPathToModelJar(Path pathToModelJar) {
    this.pathToModelJar = pathToModelJar;
  }

  public FileBasedFhirProvider withPathToModelJar(Path pathToModelJar) {
    setPathToModelJar(pathToModelJar);
    return this;
  }

  public Iterable<Object> retrieve(String context, Object contextValue, String dataType, String templateId,
                                   String codePath, Iterable<Code> codes, String valueSet, String datePath,
                                   String dateLowPath, String dateHighPath, Interval dateRange) {

    List<Object> results = new ArrayList<>();
    List<String> patientFiles = new ArrayList<>();
    Path toResults = path;

    // default context is Patient
    if (context == null) {
      context = "Patient";
    }

    if (templateId != null && !templateId.equals("")) {
      // TODO: do something?
    }

    if (codePath == null && (codes != null || valueSet != null)) {
        throw new IllegalArgumentException("A code path must be provided when filtering on codes or a valueset.");
    }

    if (context != null && context.equals("Patient") && contextValue != null) {
      toResults = toResults.resolve((String)contextValue);
    }

    // Need the context value (patient id) to resolve the toResults path correctly
    else if (context.equals("Patient") && contextValue == null) {
      toResults = toResults.resolve(getDefaultPatient(toResults));
    }

    if (dataType != null) {
      // TODO: this isn't right - Patient is a valid fhir resource.
      if (!dataType.equals("Patient"))
        toResults = toResults.resolve(dataType.toLowerCase());
    }
    else { // Just in case -- probably redundant error checking...
      throw new IllegalArgumentException("A data type (i.e. Procedure, Valueset, etc...) must be specified for clinical data retrieval");
    }

    // No filtering
    if (dateRange == null && codePath == null) {
      patientFiles = getPatientFiles(toResults, context);
      for (String resource : patientFiles) {
        Object res = null;
        if (getPackageName().equals("com.motivemi.cds2.model")) {
          res = mFhirDeserialize(resource);
        }
        else {
          res = fhirContext.newJsonParser().parseResource(resource);
        }
        results.add(res);
      }
      return results;
    }

    patientFiles = getPatientFiles(toResults, context);

    // filtering
    // NOTE: retrieves can include both date and code filtering,
    // so even though I may include a record if it is within the date range,
    // that record may be excluded later during the code filtering stage
    for (String resource : patientFiles) {
      Object outcome = null;
      Object res = null;
      if (getPackageName().equals("com.motivemi.cds2.model")) {
        res = mFhirDeserialize(resource);
      }
      else {
        res = fhirContext.newJsonParser().parseResource(resource);
      }

      // since retrieves can include both date and code filtering, I need this flag
      // to determine inclusion of codes -- if date is no good -- don't test code
      boolean includeRes = true;

      // dateRange element optionally allows a date range to be provided.
      // The clinical statements returned would be only those clinical statements whose date
      // fell within the range specified.
      if (dateRange != null) {

        // Expand Interval DateTimes to avoid InEvalutor returning null
        // TODO: account for possible null for high or low? - No issues with this yet...
        Interval expanded = new Interval(
                                  DateTime.expandPartialMin((DateTime)dateRange.getLow(), 7), true,
                                  DateTime.expandPartialMin((DateTime)dateRange.getHigh(), 7), true
                                  );
        if (datePath != null) {
          if (dateHighPath != null || dateLowPath != null) {
            throw new IllegalArgumentException("If the datePath is specified, the dateLowPath and dateHighPath attributes must not be present.");
          }

          DateTime date = null;
          Object temp = resolvePath(res, datePath);

          // Hmm... May not need this.
          if (temp instanceof DateTime) {
            date = (DateTime)temp;
          }
          else if (temp instanceof DateTimeType) {
            date = toDateTime((DateTimeType)temp);
          }

          if (date != null && InEvaluator.in(date, expanded)) {
            results.add(res);
          }
          else {
            includeRes = false;
          }
        }

        else {
          if (dateHighPath == null && dateLowPath == null) {
            throw new IllegalArgumentException("If the datePath is not given, either the lowDatePath or highDatePath must be provided.");
          }

          // get the high and low dates if present
          // if not present, set to corresponding value in the expanded Interval
          DateTime highDt = dateHighPath != null ? (DateTime)resolvePath(res, dateHighPath) : (DateTime)expanded.getHigh();
          DateTime lowDt = dateLowPath != null ? (DateTime)resolvePath(res, dateLowPath) : (DateTime)expanded.getLow();

          // the low and high dates are resolved -- create the Interval
          Interval highLowDtInterval = new Interval(lowDt, true, highDt, true);

          // Now the Includes operation
          if ((Boolean)IncludesEvaluator.includes(expanded, highLowDtInterval)) {
              results.add(res);
          }
          else {
              includeRes = false;
          }
        }
      }

      // codePath specifies which property/path of the model contains the Code or Codes for the clinical statement
      if (codePath != null && !codePath.equals("") && includeRes) {
        if (valueSet != null && !valueSet.equals("")) {
          // now we need to get the codes in the resource and check for membership in the valueset
          Object resCodes = resolvePath(res, codePath);
          if (resCodes instanceof Iterable) {
            for (Object codeObj : (Iterable)resCodes) {
              boolean inValSet = checkCodeMembership(codeObj, valueSet);
              if (inValSet && results.indexOf(res) == -1)
                results.add(res);
              else if (!inValSet)
                results.remove(res);
            }
          }
          else if (resCodes instanceof CodeableConcept) {
            if (checkCodeMembership(resCodes, valueSet) && results.indexOf(res) == -1)
              results.add(res);
          }
        }
        else if (codes != null) {
          for (Code code : codes) {
            Object resCodes = resolvePath(res, codePath);
            if (resCodes instanceof Iterable) {
              for (Object codeObj : (Iterable)resCodes) {
                Iterable<Coding> conceptCodes = ((CodeableConcept)codeObj).getCoding();
                for (Coding c : conceptCodes) {
                  if (c.getCodeElement().getValue().equals(code.getCode()) && c.getSystem().equals(code.getSystem()))
                  {
                    if (results.indexOf(res) == -1)
                      results.add(res);
                  }
                  else if (results.indexOf(res) != -1)
                    results.remove(res);
                }
              }
            }
            else if (resCodes instanceof CodeableConcept) {
              for (Coding c : ((CodeableConcept)resCodes).getCoding()) {
                if (c.getCodeElement().getValue().equals(code.getCode()) && c.getSystem().equals(code.getSystem()))
                {
                  if (results.indexOf(res) == -1)
                    results.add(res);
                }
                else if (results.indexOf(res) != -1)
                  results.remove(res);
              }
            }
          }
        }
      }
    } // end of filtering for each loop

    return results;
  }

  public Object mFhirDeserialize(String resource) {
    // ObjectMapper objectMapper = new ObjectMapper();
    // try {
    //   return objectMapper.readValue(resource, FHIRStatement.class);
    // }
    // catch (IOException e) {
    //   // TODO: quick fix here - do something more substantial
    //   return null;
    // }
    return null;
  }

  // If Patient context without patient id, get the first patient
  public String getDefaultPatient(Path evalPath) {
    File file = new File(evalPath.toString());
    String patientId = null;
    if (!file.exists()) {<commit_msg>Adding class loader for model jar<commit_after>package org.opencds.cqf.cql.file.fhir;

import org.hl7.fhir.dstu3.model.CodeableConcept;
import org.hl7.fhir.dstu3.model.Coding;
import org.hl7.fhir.dstu3.model.DateTimeType;
import org.joda.time.Partial;
import org.opencds.cqf.cql.data.fhir.BaseFhirDataProvider;
import org.opencds.cqf.cql.elm.execution.InEvaluator;
import org.opencds.cqf.cql.elm.execution.IncludesEvaluator;
import org.opencds.cqf.cql.runtime.Code;
import org.opencds.cqf.cql.runtime.DateTime;
import org.opencds.cqf.cql.runtime.Interval;
import org.opencds.cqf.cql.terminology.ValueSetInfo;
import org.opencds.cqf.cql.terminology.fhir.FhirTerminologyProvider;

import com.fasterxml.jackson.databind.ObjectMapper;
import java.net.URL;
import java.net.URLClassLoader;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.nio.file.InvalidPathException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;

/*
What the heck does this thing do?
  This class is intended to provide the user with the alternative of using a local repository
  for clinical data retrieval instead of an external service.
  NOTE: This class still uses a Terminology service for value set retrieval and evaluating code membership

How do I use it?
  Point the provider to the directory of patients:
    FileBasedFhirProvider provider = new FileBasedFhirProvider(path to root data dir, terminology service endpoint(optional));
    Each subfolder name in the patients directory should be an id for a specific patients
      In each patient folder there should be subfolders contatining clinical information
        (e.g. Condition, Procedure, etc...) for that patient in JSON format (XML support pending)
    Here is a mock directory structure:
    - patients
      - 123
        - Conditions
          - JSON representation of Fhir resources
        - Procedures
        - Encounters
        - etc...
      - 154
        - Observations
        - etc...
      - 209
      - etc...
*/

public class FileBasedFhirProvider extends BaseFhirDataProvider {

  private Path path;
  private FhirTerminologyProvider terminologyProvider;

  public FileBasedFhirProvider (String path, String endpoint) {
    if (path.isEmpty() || path == null) {
      throw new InvalidPathException(path, "Invalid path!");
    }
    this.path = Paths.get(path);
    this.terminologyProvider = endpoint == null ? new FhirTerminologyProvider().withEndpoint("http://fhirtest.uhn.ca/baseDstu3")
                                                : new FhirTerminologyProvider().withEndpoint(endpoint);
  }

  private URL pathToModelJar;
  public URL getpathToModelJar() {
    return pathToModelJar;
  }

  public void setPathToModelJar(URL pathToModelJar) {
    this.pathToModelJar = pathToModelJar;
  }

  public FileBasedFhirProvider withPathToModelJar(URL pathToModelJar) {
    setPathToModelJar(pathToModelJar);
    return this;
  }

  public Iterable<Object> retrieve(String context, Object contextValue, String dataType, String templateId,
                                   String codePath, Iterable<Code> codes, String valueSet, String datePath,
                                   String dateLowPath, String dateHighPath, Interval dateRange) {

    List<Object> results = new ArrayList<>();
    List<String> patientFiles = new ArrayList<>();
    Path toResults = path;

    // default context is Patient
    if (context == null) {
      context = "Patient";
    }

    if (templateId != null && !templateId.equals("")) {
      // TODO: do something?
    }

    if (codePath == null && (codes != null || valueSet != null)) {
        throw new IllegalArgumentException("A code path must be provided when filtering on codes or a valueset.");
    }

    if (context != null && context.equals("Patient") && contextValue != null) {
      toResults = toResults.resolve((String)contextValue);
    }

    // Need the context value (patient id) to resolve the toResults path correctly
    else if (context.equals("Patient") && contextValue == null) {
      toResults = toResults.resolve(getDefaultPatient(toResults));
    }

    if (dataType != null) {
      // TODO: this isn't right - Patient is a valid fhir resource.
      if (!dataType.equals("Patient"))
        toResults = toResults.resolve(dataType.toLowerCase());
    }
    else { // Just in case -- probably redundant error checking...
      throw new IllegalArgumentException("A data type (i.e. Procedure, Valueset, etc...) must be specified for clinical data retrieval");
    }

    // No filtering
    if (dateRange == null && codePath == null) {
      patientFiles = getPatientFiles(toResults, context);
      for (String resource : patientFiles) {
        Object res = null;
        if (getPackageName().equals("com.motivemi.cds2.model")) {
          res = mFhirDeserialize(resource);
        }
        else {
          res = fhirContext.newJsonParser().parseResource(resource);
        }
        results.add(res);
      }
      return results;
    }

    patientFiles = getPatientFiles(toResults, context);

    // filtering
    // NOTE: retrieves can include both date and code filtering,
    // so even though I may include a record if it is within the date range,
    // that record may be excluded later during the code filtering stage
    for (String resource : patientFiles) {
      Object outcome = null;
      Object res = null;
      if (getPackageName().equals("com.motivemi.cds2.model")) {
        res = mFhirDeserialize(resource);
      }
      else {
        res = fhirContext.newJsonParser().parseResource(resource);
      }

      // since retrieves can include both date and code filtering, I need this flag
      // to determine inclusion of codes -- if date is no good -- don't test code
      boolean includeRes = true;

      // dateRange element optionally allows a date range to be provided.
      // The clinical statements returned would be only those clinical statements whose date
      // fell within the range specified.
      if (dateRange != null) {

        // Expand Interval DateTimes to avoid InEvalutor returning null
        // TODO: account for possible null for high or low? - No issues with this yet...
        Interval expanded = new Interval(
                                  DateTime.expandPartialMin((DateTime)dateRange.getLow(), 7), true,
                                  DateTime.expandPartialMin((DateTime)dateRange.getHigh(), 7), true
                                  );
        if (datePath != null) {
          if (dateHighPath != null || dateLowPath != null) {
            throw new IllegalArgumentException("If the datePath is specified, the dateLowPath and dateHighPath attributes must not be present.");
          }

          DateTime date = null;
          Object temp = resolvePath(res, datePath);

          // Hmm... May not need this.
          if (temp instanceof DateTime) {
            date = (DateTime)temp;
          }
          else if (temp instanceof DateTimeType) {
            date = toDateTime((DateTimeType)temp);
          }

          if (date != null && InEvaluator.in(date, expanded)) {
            results.add(res);
          }
          else {
            includeRes = false;
          }
        }

        else {
          if (dateHighPath == null && dateLowPath == null) {
            throw new IllegalArgumentException("If the datePath is not given, either the lowDatePath or highDatePath must be provided.");
          }

          // get the high and low dates if present
          // if not present, set to corresponding value in the expanded Interval
          DateTime highDt = dateHighPath != null ? (DateTime)resolvePath(res, dateHighPath) : (DateTime)expanded.getHigh();
          DateTime lowDt = dateLowPath != null ? (DateTime)resolvePath(res, dateLowPath) : (DateTime)expanded.getLow();

          // the low and high dates are resolved -- create the Interval
          Interval highLowDtInterval = new Interval(lowDt, true, highDt, true);

          // Now the Includes operation
          if ((Boolean)IncludesEvaluator.includes(expanded, highLowDtInterval)) {
              results.add(res);
          }
          else {
              includeRes = false;
          }
        }
      }

      // codePath specifies which property/path of the model contains the Code or Codes for the clinical statement
      if (codePath != null && !codePath.equals("") && includeRes) {
        if (valueSet != null && !valueSet.equals("")) {
          // now we need to get the codes in the resource and check for membership in the valueset
          Object resCodes = resolvePath(res, codePath);
          if (resCodes instanceof Iterable) {
            for (Object codeObj : (Iterable)resCodes) {
              boolean inValSet = checkCodeMembership(codeObj, valueSet);
              if (inValSet && results.indexOf(res) == -1)
                results.add(res);
              else if (!inValSet)
                results.remove(res);
            }
          }
          else if (resCodes instanceof CodeableConcept) {
            if (checkCodeMembership(resCodes, valueSet) && results.indexOf(res) == -1)
              results.add(res);
          }
        }
        else if (codes != null) {
          for (Code code : codes) {
            Object resCodes = resolvePath(res, codePath);
            if (resCodes instanceof Iterable) {
              for (Object codeObj : (Iterable)resCodes) {
                Iterable<Coding> conceptCodes = ((CodeableConcept)codeObj).getCoding();
                for (Coding c : conceptCodes) {
                  if (c.getCodeElement().getValue().equals(code.getCode()) && c.getSystem().equals(code.getSystem()))
                  {
                    if (results.indexOf(res) == -1)
                      results.add(res);
                  }
                  else if (results.indexOf(res) != -1)
                    results.remove(res);
                }
              }
            }
            else if (resCodes instanceof CodeableConcept) {
              for (Coding c : ((CodeableConcept)resCodes).getCoding()) {
                if (c.getCodeElement().getValue().equals(code.getCode()) && c.getSystem().equals(code.getSystem()))
                {
                  if (results.indexOf(res) == -1)
                    results.add(res);
                }
                else if (results.indexOf(res) != -1)
                  results.remove(res);
              }
            }
          }
        }
      }
    } // end of filtering for each loop

    return results;
  }

  public Object mFhirDeserialize(String resource) {
    URLClassLoader loader = new URLClassLoader(new URL[] {pathToModelJar});
    Class<?> clazz = null;
    try {
      clazz = loader.loadClass(getPackageName() + ".FHIRStatement");
    }
    catch (ClassNotFoundException e) {
      // TODO: quick fix here - report something more substantial
      return null;
    }

    ObjectMapper objectMapper = new ObjectMapper();
    try {
      return objectMapper.readValue(resource, clazz);
    }
    catch (IOException e) {
      // TODO: quick fix here - report something more substantial
      return null;
    }
  }

  // If Patient context without patient id, get the first patient
  public String getDefaultPatient(Path evalPath) {
    File file = new File(evalPath.toString());
    String patientId = null;
    if (!file.exists()) {
{'fraction_non_alphanumeric': 0.09745631497456315, 'fraction_numerical': 0.00230037602300376, 'mean_word_length': 2.282416146362712, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4445666', 'n_tokens_mistral': 6156, 'n_tokens_neox': 5976, 'n_words': 1807}