<commit_before>package com.ligadata.kamanja.copd;

import com.ligadata.FatafatBase.*;
import com.ligadata.messagescontainers.System.*;
import org.joda.time.*;

import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.*;

/**
 * Created by will on 7/1/15.
 */
public class COPDRiskAssessment extends ModelBase {
    // Messages
    private Beneficiary msg = null;
    private JavaRDD<InpatientClaim> inpatientClaimHistoryRDD = null;
    private JavaRDD<OutpatientClaim> outpatientClaimHistoryRDD = null;
    private JavaRDD<HL7> hl7HistoryRDD = null;

    // Containers
    private JavaRDD<CoughCodes> coughCodesRDD = null;
    private JavaRDD<DyspnoeaCodes> dyspnoeaCodesRDD = null;
    private JavaRDD<EnvCodes> envCodesRDD = null;
    private JavaRDD<SmokeCodes> smokeCodesRDD = null;
    private JavaRDD<SputumCodes> sputumCodesRDD = null;

    // Filtered Message Arrays
    private ArrayList<InpatientClaim> inpatientClaimHistory = new ArrayList<>();
    private ArrayList<OutpatientClaim> outpatientClaimHistory = new ArrayList<>();
    private ArrayList<HL7> hl7History = new ArrayList<>();

    // Lookup Arrays
    private ArrayList<String> coughCodes = new ArrayList<String>();
    private ArrayList<String> dyspnoeaCodes = new ArrayList<String>();
    private ArrayList<String> envCodes = new ArrayList<String>();
    private ArrayList<String> smokeCodes = new ArrayList<String>();
    private ArrayList<String> sputumCodes = new ArrayList<String>();

    private String[] partitionKeys = null;

    private ModelContext mdlContext = null;

    private SimpleDateFormat yearMonthDayHourFormat = new SimpleDateFormat("yyyyMMdd");
    private Integer today = null;
    private Integer oneYearAgo = null;
    private TimeRange lookupPeriod = null;

    private void init() {
        msg = (Beneficiary) this.modelContext().msg();
        System.out.println("Executing COPD Risk Assessment against Beneficiary message:");
        System.out.println("\tMessage Name: " + msg.Name());
        System.out.println("\tMessage Version: " + msg.Version());
        System.out.println("\tMessage Desynpuf ID: " + msg.desynpuf_id());

        partitionKeys = msg.PartitionKeyData();

        lookupPeriod = new TimeRange(getOneYearAgo(), getToday());

        // Getting message RDD objects
/*
        // We can call the rddObject with local variable also.
        JavaRDDObject<InpatientClaim> inpJavaRddObject = InpatientClaimFactory.rddObject;
        JavaRDDObject<OutpatientClaim> outpJavaRddObject = OutpatientClaimFactory.rddObject;
        JavaRDDObject<HL7> hl7JavaRddObject = HL7Factory.rddObject;
        inpatientClaimHistoryRDD = inpJavaRddObject.getRDD(partitionKeys, getLookupPeriod());
        outpatientClaimHistoryRDD = outpJavaRddObject.getRDD(partitionKeys, getLookupPeriod());
        hl7HistoryRDD = hl7JavaRddObject.getRDD(partitionKeys, getLookupPeriod());
*/
        inpatientClaimHistoryRDD = InpatientClaimFactory.rddObject.getRDD(partitionKeys, getLookupPeriod());
        outpatientClaimHistoryRDD = OutpatientClaimFactory.rddObject.getRDD(partitionKeys, getLookupPeriod());
        hl7HistoryRDD = HL7Factory.rddObject.getRDD(partitionKeys, getLookupPeriod());
        // Filter claims down to the last year based on clm_thru_dt
        for (Iterator<InpatientClaim> ipClaimIt = inpatientClaimHistoryRDD.iterator(); ipClaimIt.hasNext(); ) {
            InpatientClaim claim = ipClaimIt.next();
            if (isDateBetween(claim.clm_thru_dt())) {
                inpatientClaimHistory.add(claim);
            }
        }

        for (Iterator<OutpatientClaim> opClaimIt = outpatientClaimHistoryRDD.iterator(); opClaimIt.hasNext(); ) {
            OutpatientClaim claim = opClaimIt.next();
            if (isDateBetween(claim.clm_thru_dt())) {
                outpatientClaimHistory.add(claim);
            }
        }

        for (Iterator<HL7> hl7Iterator = hl7HistoryRDD.iterator(); hl7Iterator.hasNext(); ) {
            HL7 claim = hl7Iterator.next();
            if (isDateBetween(claim.clm_thru_dt())) {
                hl7History.add(claim);
            }
        }

        // Getting lookup tables
        coughCodesRDD = CoughCodesFactory.rddObject.getRDD(new TimeRange(0, getToday()));
        dyspnoeaCodesRDD = DyspnoeaCodesFactory.rddObject.getRDD(new TimeRange(0, getToday()));
        envCodesRDD = EnvCodesFactory.rddObject.getRDD(new TimeRange(0, getToday()));
        smokeCodesRDD = SmokeCodesFactory.rddObject.getRDD(new TimeRange(0, getToday()));
        sputumCodesRDD = SputumCodesFactory.rddObject.getRDD(new TimeRange(0, getToday()));

        for (Iterator<CoughCodes> coughCodeIt = coughCodesRDD.iterator(); coughCodeIt.hasNext(); ) {
            coughCodes.add(coughCodeIt.next().icd9code());
        }
        System.out.print("Cough Codes:\t");
        for (String code : coughCodes) {
            System.out.print(code + ", ");
        }

        for (Iterator<DyspnoeaCodes> dyspCodeIt = dyspnoeaCodesRDD.iterator(); dyspCodeIt.hasNext(); ) {
            dyspnoeaCodes.add(dyspCodeIt.next().icd9code());
        }
        System.out.print("\nDyspnea Codes:\t");
        for (String code : dyspnoeaCodes) {
            System.out.print(code + ", ");
        }<commit_msg>Minor update on top of Pokuri's changes to get getRDD() on containers to get all containers without a timerange.<commit_after>package com.ligadata.kamanja.copd;

import com.ligadata.FatafatBase.*;
import com.ligadata.messagescontainers.System.*;
import org.joda.time.*;

import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.*;

/**
 * Created by will on 7/1/15.
 */

public class COPDRiskAssessment extends ModelBase {
    // Messages
    private Beneficiary msg = null;
    private JavaRDD<InpatientClaim> inpatientClaimHistoryRDD = null;
    private JavaRDD<OutpatientClaim> outpatientClaimHistoryRDD = null;
    private JavaRDD<HL7> hl7HistoryRDD = null;

    // Containers
    private JavaRDD<CoughCodes> coughCodesRDD = null;
    private JavaRDD<DyspnoeaCodes> dyspnoeaCodesRDD = null;
    private JavaRDD<EnvCodes> envCodesRDD = null;
    private JavaRDD<SmokeCodes> smokeCodesRDD = null;
    private JavaRDD<SputumCodes> sputumCodesRDD = null;

    // Filtered Message Arrays
    private ArrayList<InpatientClaim> inpatientClaimHistory = new ArrayList<>();
    private ArrayList<OutpatientClaim> outpatientClaimHistory = new ArrayList<>();
    private ArrayList<HL7> hl7History = new ArrayList<>();

    // Lookup Arrays
    private ArrayList<String> coughCodes = new ArrayList<>();
    private ArrayList<String> dyspnoeaCodes = new ArrayList<>();
    private ArrayList<String> envCodes = new ArrayList<>();
    private ArrayList<String> smokeCodes = new ArrayList<>();
    private ArrayList<String> sputumCodes = new ArrayList<>();

    private String[] partitionKeys = null;

    private ModelContext mdlContext = null;

    private SimpleDateFormat yearMonthDayHourFormat = new SimpleDateFormat("yyyyMMdd");
    private Integer today = null;
    private Integer oneYearAgo = null;
    private TimeRange lookupPeriod = null;

    private void init() {
        msg = (Beneficiary) this.modelContext().msg();
        System.out.println("Executing COPD Risk Assessment against Beneficiary message:");
        System.out.println("\tMessage Name: " + msg.Name());
        System.out.println("\tMessage Version: " + msg.Version());
        System.out.println("\tMessage Desynpuf ID: " + msg.desynpuf_id());

        partitionKeys = msg.PartitionKeyData();

        lookupPeriod = new TimeRange(getOneYearAgo(), getToday());

        // Getting message RDD objects
        inpatientClaimHistoryRDD = InpatientClaimFactory.rddObject.getRDD(partitionKeys, getLookupPeriod());
        outpatientClaimHistoryRDD = OutpatientClaimFactory.rddObject.getRDD(partitionKeys, getLookupPeriod());
        hl7HistoryRDD = HL7Factory.rddObject.getRDD(partitionKeys, getLookupPeriod());

        // Filter claims down to the last year based on clm_thru_dt
        for (Iterator<InpatientClaim> ipClaimIt = inpatientClaimHistoryRDD.iterator(); ipClaimIt.hasNext(); ) {
            InpatientClaim claim = ipClaimIt.next();
            if (isDateBetween(claim.clm_thru_dt())) {
                inpatientClaimHistory.add(claim);
            }
        }

        for (Iterator<OutpatientClaim> opClaimIt = outpatientClaimHistoryRDD.iterator(); opClaimIt.hasNext(); ) {
            OutpatientClaim claim = opClaimIt.next();
            if (isDateBetween(claim.clm_thru_dt())) {
                outpatientClaimHistory.add(claim);
            }
        }

        for (Iterator<HL7> hl7Iterator = hl7HistoryRDD.iterator(); hl7Iterator.hasNext(); ) {
            HL7 claim = hl7Iterator.next();
            if (isDateBetween(claim.clm_thru_dt())) {
                hl7History.add(claim);
            }
        }

        // Getting lookup tables
        coughCodesRDD = CoughCodesFactory.rddObject.getRDD();
        dyspnoeaCodesRDD = DyspnoeaCodesFactory.rddObject.getRDD();
        envCodesRDD = EnvCodesFactory.rddObject.getRDD();
        smokeCodesRDD = SmokeCodesFactory.rddObject.getRDD();
        sputumCodesRDD = SputumCodesFactory.rddObject.getRDD();

        for (Iterator<CoughCodes> coughCodeIt = coughCodesRDD.iterator(); coughCodeIt.hasNext(); ) {
            coughCodes.add(coughCodeIt.next().icd9code());
        }
        System.out.print("Cough Codes:\t");
        for (String code : coughCodes) {
            System.out.print(code + ", ");
        }

        for (Iterator<DyspnoeaCodes> dyspCodeIt = dyspnoeaCodesRDD.iterator(); dyspCodeIt.hasNext(); ) {
            dyspnoeaCodes.add(dyspCodeIt.next().icd9code());
        }
        System.out.print("\nDyspnea Codes:\t");
        for (String code : dyspnoeaCodes) {
            System.out.print(code + ", ");
        }
{'fraction_non_alphanumeric': 0.10900425877104036, 'fraction_numerical': 0.004867166903265057, 'mean_word_length': 3.362229102167183, 'pattern_counts': {'":': 0, '<': 64, '<?xml version=': 0, '>': 64, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6035446', 'n_tokens_mistral': 2945, 'n_tokens_neox': 2920, 'n_words': 567}