<commit_before>package org.endeavourhealth.transform.enterprise.transforms;

import OpenPseudonymiser.Crypto;
import com.google.common.base.Strings;
import org.endeavourhealth.common.fhir.FhirUri;
import org.endeavourhealth.common.fhir.IdentifierHelper;
import org.endeavourhealth.common.utility.Resources;
import org.endeavourhealth.core.data.ehr.models.ResourceByExchangeBatch;
import org.endeavourhealth.core.rdbms.eds.PatientLinkHelper;
import org.endeavourhealth.core.rdbms.eds.PatientLinkPair;
import org.endeavourhealth.core.rdbms.reference.PostcodeHelper;
import org.endeavourhealth.core.rdbms.reference.PostcodeReference;
import org.endeavourhealth.core.rdbms.transform.EnterpriseAgeUpdater;
import org.endeavourhealth.core.rdbms.transform.EnterpriseIdHelper;
import org.endeavourhealth.core.rdbms.transform.HouseholdHelper;
import org.endeavourhealth.core.rdbms.transform.PseudoIdHelper;
import org.endeavourhealth.transform.enterprise.outputModels.OutputContainer;
import org.hl7.fhir.instance.model.Address;
import org.hl7.fhir.instance.model.DateType;
import org.hl7.fhir.instance.model.Enumerations;
import org.hl7.fhir.instance.model.Patient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.math.BigDecimal;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Map;
import java.util.TreeMap;

public class PatientTransformer extends AbstractTransformer {
    private static final Logger LOG = LoggerFactory.getLogger(PatientTransformer.class);

    private static final String PSEUDO_KEY_NHS_NUMBER = "NHSNumber";
    private static final String PSEUDO_KEY_PATIENT_NUMBER = "PatientNumber";
    private static final String PSEUDO_KEY_DATE_OF_BIRTH = "DOB";
    private static final String PSEUDO_SALT_RESOURCE = "Endeavour Enterprise - East London.EncryptedSalt";

    private static byte[] saltBytes = null;

    public void transform(ResourceByExchangeBatch resource,
                          OutputContainer data,
                          Map<String, ResourceByExchangeBatch> otherResources,
                          Long enterpriseOrganisationId,
                          Long nullEnterprisePatientId,
                          Long nullEnterprisePersonId,
                          String configName) throws Exception {

        org.endeavourhealth.transform.enterprise.outputModels.Patient model = data.getPatients();

        Long enterpriseId = mapId(resource, model);
        if (enterpriseId == null) {
            return;

        } else if (resource.getIsDeleted()) {
            model.writeDelete(enterpriseId.longValue());

        } else {
            Patient fhirPatient = (Patient)deserialiseResouce(resource);

            String discoveryPersonId = PatientLinkHelper.getPersonId(fhirPatient.getId());

            //when the person ID table was populated, patients who had been deleted weren't added,
            //so we'll occasionally get null for some patients. If this happens, just do what would have
            //been done originally and assign an ID
            if (Strings.isNullOrEmpty(discoveryPersonId)) {
                PatientLinkPair pair = PatientLinkHelper.updatePersonId(fhirPatient);
                discoveryPersonId = pair.getNewPersonId();
            }

            Long enterprisePersonId = EnterpriseIdHelper.findOrCreateEnterprisePersonId(discoveryPersonId, configName);

            long id;
            long organizationId;
            long personId;
            int patientGenderId;
            String pseudoId = null;
            String nhsNumber = null;
            Integer ageYears = null;
            Integer ageMonths = null;
            Integer ageWeeks = null;
            Date dateOfBirth = null;
            Date dateOfDeath = null;
            String postcode = null;
            String postcodePrefix = null;
            Long householdId = null;
            String lsoaCode = null;
            String msoaCode = null;
            BigDecimal townsendScore = null;

            id = enterpriseId.longValue();
            organizationId = enterpriseOrganisationId.longValue();
            personId = enterprisePersonId.longValue();

            //Calendar cal = Calendar.getInstance();

            dateOfBirth = fhirPatient.getBirthDate();
            /*cal.setTime(dob);
            int yearOfBirth = cal.get(Calendar.YEAR);
            model.setYearOfBirth(yearOfBirth);*/

            if (fhirPatient.hasDeceasedDateTimeType()) {
                dateOfDeath = fhirPatient.getDeceasedDateTimeType().getValue();
                /*cal.setTime(dod);
                int yearOfDeath = cal.get(Calendar.YEAR);
                model.setYearOfDeath(new Integer(yearOfDeath));*/
            } else if (fhirPatient.hasDeceased()
                    && fhirPatient.getDeceased() instanceof DateType) {
                //should always be a DATE TIME type, but a bug in the CSV->FHIR transform
                //means we've got data with a DATE type too
                DateType d = (DateType)fhirPatient.getDeceased();
                dateOfDeath = d.getValue();
            }

            patientGenderId = fhirPatient.getGender().ordinal();


            if (fhirPatient.hasAddress()) {
                for (Address address: fhirPatient.getAddress()) {
                    if (address.getUse().equals(Address.AddressUse.HOME)) {
                        postcode = address.getPostalCode();
                        postcodePrefix = findPostcodePrefix(postcode);
                        householdId = HouseholdHelper.findOrCreateHouseholdId(address);
                    }
                }
            }

            //if we've found a postcode, then get the LSOA etc. for it
            if (!Strings.isNullOrEmpty(postcode)) {
                PostcodeReference postcodeReference = PostcodeHelper.getPostcodeReference(postcode);
                if (postcodeReference != null) {
                    lsoaCode = postcodeReference.getLsoaCode();
                    msoaCode = postcodeReference.getMsoaCode();
                    townsendScore = postcodeReference.getTownsendScore();
                }
            }


            if (model.isPseduonymised()) {

                //if pseudonymised, all non-male/non-female genders should be treated as female
                if (fhirPatient.getGender() != Enumerations.AdministrativeGender.FEMALE
                        && fhirPatient.getGender() != Enumerations.AdministrativeGender.MALE) {
                    patientGenderId = Enumerations.AdministrativeGender.FEMALE.ordinal();
                }

                pseudoId = pseudonomise(fhirPatient);

                //only persist the pseudo ID if it's non-null
                if (!Strings.isNullOrEmpty(pseudoId)) {
                    PseudoIdHelper.storePseudoId(fhirPatient.getId(), pseudoId);
                }

                Integer[] ageValues = EnterpriseAgeUpdater.calculateAgeValues(id, dateOfBirth, configName);
                ageYears = ageValues[EnterpriseAgeUpdater.UNIT_YEARS];<commit_msg>Adding handler for Homerton addresses w/o uses in FHIR->Enterprise<commit_after>package org.endeavourhealth.transform.enterprise.transforms;

import OpenPseudonymiser.Crypto;
import com.google.common.base.Strings;
import org.endeavourhealth.common.fhir.FhirUri;
import org.endeavourhealth.common.fhir.IdentifierHelper;
import org.endeavourhealth.common.utility.Resources;
import org.endeavourhealth.core.data.ehr.models.ResourceByExchangeBatch;
import org.endeavourhealth.core.rdbms.eds.PatientLinkHelper;
import org.endeavourhealth.core.rdbms.eds.PatientLinkPair;
import org.endeavourhealth.core.rdbms.reference.PostcodeHelper;
import org.endeavourhealth.core.rdbms.reference.PostcodeReference;
import org.endeavourhealth.core.rdbms.transform.EnterpriseAgeUpdater;
import org.endeavourhealth.core.rdbms.transform.EnterpriseIdHelper;
import org.endeavourhealth.core.rdbms.transform.HouseholdHelper;
import org.endeavourhealth.core.rdbms.transform.PseudoIdHelper;
import org.endeavourhealth.transform.enterprise.outputModels.OutputContainer;
import org.hl7.fhir.instance.model.Address;
import org.hl7.fhir.instance.model.DateType;
import org.hl7.fhir.instance.model.Enumerations;
import org.hl7.fhir.instance.model.Patient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.math.BigDecimal;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Map;
import java.util.TreeMap;

public class PatientTransformer extends AbstractTransformer {
    private static final Logger LOG = LoggerFactory.getLogger(PatientTransformer.class);

    private static final String PSEUDO_KEY_NHS_NUMBER = "NHSNumber";
    private static final String PSEUDO_KEY_PATIENT_NUMBER = "PatientNumber";
    private static final String PSEUDO_KEY_DATE_OF_BIRTH = "DOB";
    private static final String PSEUDO_SALT_RESOURCE = "Endeavour Enterprise - East London.EncryptedSalt";

    private static byte[] saltBytes = null;

    public void transform(ResourceByExchangeBatch resource,
                          OutputContainer data,
                          Map<String, ResourceByExchangeBatch> otherResources,
                          Long enterpriseOrganisationId,
                          Long nullEnterprisePatientId,
                          Long nullEnterprisePersonId,
                          String configName) throws Exception {

        org.endeavourhealth.transform.enterprise.outputModels.Patient model = data.getPatients();

        Long enterpriseId = mapId(resource, model);
        if (enterpriseId == null) {
            return;

        } else if (resource.getIsDeleted()) {
            model.writeDelete(enterpriseId.longValue());

        } else {
            Patient fhirPatient = (Patient)deserialiseResouce(resource);

            String discoveryPersonId = PatientLinkHelper.getPersonId(fhirPatient.getId());

            //when the person ID table was populated, patients who had been deleted weren't added,
            //so we'll occasionally get null for some patients. If this happens, just do what would have
            //been done originally and assign an ID
            if (Strings.isNullOrEmpty(discoveryPersonId)) {
                PatientLinkPair pair = PatientLinkHelper.updatePersonId(fhirPatient);
                discoveryPersonId = pair.getNewPersonId();
            }

            Long enterprisePersonId = EnterpriseIdHelper.findOrCreateEnterprisePersonId(discoveryPersonId, configName);

            long id;
            long organizationId;
            long personId;
            int patientGenderId;
            String pseudoId = null;
            String nhsNumber = null;
            Integer ageYears = null;
            Integer ageMonths = null;
            Integer ageWeeks = null;
            Date dateOfBirth = null;
            Date dateOfDeath = null;
            String postcode = null;
            String postcodePrefix = null;
            Long householdId = null;
            String lsoaCode = null;
            String msoaCode = null;
            BigDecimal townsendScore = null;

            id = enterpriseId.longValue();
            organizationId = enterpriseOrganisationId.longValue();
            personId = enterprisePersonId.longValue();

            //Calendar cal = Calendar.getInstance();

            dateOfBirth = fhirPatient.getBirthDate();
            /*cal.setTime(dob);
            int yearOfBirth = cal.get(Calendar.YEAR);
            model.setYearOfBirth(yearOfBirth);*/

            if (fhirPatient.hasDeceasedDateTimeType()) {
                dateOfDeath = fhirPatient.getDeceasedDateTimeType().getValue();
                /*cal.setTime(dod);
                int yearOfDeath = cal.get(Calendar.YEAR);
                model.setYearOfDeath(new Integer(yearOfDeath));*/
            } else if (fhirPatient.hasDeceased()
                    && fhirPatient.getDeceased() instanceof DateType) {
                //should always be a DATE TIME type, but a bug in the CSV->FHIR transform
                //means we've got data with a DATE type too
                DateType d = (DateType)fhirPatient.getDeceased();
                dateOfDeath = d.getValue();
            }

            patientGenderId = fhirPatient.getGender().ordinal();


            if (fhirPatient.hasAddress()) {
                for (Address address: fhirPatient.getAddress()) {
                    if (address.getUse() != null //got Homerton data will null address use
                            && address.getUse().equals(Address.AddressUse.HOME)) {
                        postcode = address.getPostalCode();
                        postcodePrefix = findPostcodePrefix(postcode);
                        householdId = HouseholdHelper.findOrCreateHouseholdId(address);
                        break;
                    }
                }
            }

            //if we've found a postcode, then get the LSOA etc. for it
            if (!Strings.isNullOrEmpty(postcode)) {
                PostcodeReference postcodeReference = PostcodeHelper.getPostcodeReference(postcode);
                if (postcodeReference != null) {
                    lsoaCode = postcodeReference.getLsoaCode();
                    msoaCode = postcodeReference.getMsoaCode();
                    townsendScore = postcodeReference.getTownsendScore();
                }
            }

            if (model.isPseduonymised()) {

                //if pseudonymised, all non-male/non-female genders should be treated as female
                if (fhirPatient.getGender() != Enumerations.AdministrativeGender.FEMALE
                        && fhirPatient.getGender() != Enumerations.AdministrativeGender.MALE) {
                    patientGenderId = Enumerations.AdministrativeGender.FEMALE.ordinal();
                }

                pseudoId = pseudonomise(fhirPatient);

                //only persist the pseudo ID if it's non-null
                if (!Strings.isNullOrEmpty(pseudoId)) {
                    PseudoIdHelper.storePseudoId(fhirPatient.getId(), pseudoId);
                }

                Integer[] ageValues = EnterpriseAgeUpdater.calculateAgeValues(id, dateOfBirth, configName);
                ageYears = ageValues[EnterpriseAgeUpdater.UNIT_YEARS];
{'fraction_non_alphanumeric': 0.08105448154657294, 'fraction_numerical': 0.0008435852372583479, 'mean_word_length': 2.3370865587614356, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6102431', 'n_tokens_mistral': 3932, 'n_tokens_neox': 3625, 'n_words': 767}