<commit_before>package ca.intelliware.ihtsdo.mlds.service.affiliatesimport;

import java.io.IOException;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.List;

import javax.annotation.Resource;
import javax.persistence.EntityManager;
import javax.persistence.PersistenceContext;
import javax.transaction.Transactional;

import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.joda.time.LocalDate;
import org.springframework.stereotype.Service;

import ca.intelliware.ihtsdo.mlds.domain.Affiliate;
import ca.intelliware.ihtsdo.mlds.domain.AffiliateDetails;
import ca.intelliware.ihtsdo.mlds.domain.AffiliateType;
import ca.intelliware.ihtsdo.mlds.domain.Application;
import ca.intelliware.ihtsdo.mlds.domain.Application.ApplicationType;
import ca.intelliware.ihtsdo.mlds.domain.ApprovalState;
import ca.intelliware.ihtsdo.mlds.domain.CommercialUsage;
import ca.intelliware.ihtsdo.mlds.domain.CommercialUsagePeriod;
import ca.intelliware.ihtsdo.mlds.domain.PrimaryApplication;
import ca.intelliware.ihtsdo.mlds.domain.UsageContext;
import ca.intelliware.ihtsdo.mlds.repository.AffiliateDetailsRepository;
import ca.intelliware.ihtsdo.mlds.repository.AffiliateRepository;
import ca.intelliware.ihtsdo.mlds.repository.ApplicationRepository;
import ca.intelliware.ihtsdo.mlds.repository.CommercialUsageRepository;

@Service
@Transactional
public class AffiliatesImporterService {

	@Resource ApplicationRepository applicationRepository;
	@Resource AffiliateRepository affiliateRepository;
	@Resource AffiliateDetailsRepository affiliateDetailsRepository;
	@Resource CommercialUsageRepository commercialUsageRepository;
	
	@Resource AffiliatesMapper affiliatesMapper;
	
	@PersistenceContext
    private EntityManager entityManager;
	
	public ImportResult importFromCSV(String contents) throws IOException {
		ImportResult result = new ImportResult();
		long start = System.currentTimeMillis();
		try {
			importContents(contents, result);
		} catch (Exception e) {
			result.addException(e);
		}
		result.durationMillis = System.currentTimeMillis() - start;
		return result;
	}

	private void importContents(String contents, ImportResult result) throws IOException {
		List<LineRecord> lines = parseFile(contents);
		result.readRecords = lines.size();
		validateLines(lines, result);
		if (result.success) {
			processAffiliateRecords(lines, result);
		}
		
	}

	private void processAffiliateRecords(List<LineRecord> lines, ImportResult result) {
		result.importedRecords = 0;
		for (int i = 0; i < lines.size(); i++) {
			LineRecord lineRecord = lines.get(i);
			if (!lineRecord.header && !lineRecord.isBlank) {
				try {
					processLineRecord(lineRecord, result);
					result.importedRecords += 1;
				} catch (Exception e) {
					result.addError(lineRecord, "Failed to populate record: "+e);
				}
			}
		}
	}

	void processLineRecord(LineRecord record, ImportResult result) throws IllegalArgumentException, IllegalAccessException, InstantiationException {
		Affiliate affiliate = findExistingAffiliateForUpdate(record);
		if (affiliate == null) {
			createApprovedAffiliate(record,result);
		} else {
			updateAffiliate(affiliate,record,result);
		}
		clearSession();
	}

	/** For bulk performance clear pending changes from hibernate session */
	private void clearSession() {
		entityManager.flush();
		entityManager.clear();
	}

	private Affiliate findExistingAffiliateForUpdate(LineRecord record) throws IllegalAccessException, InstantiationException {
		Affiliate tmpAffiliate = new Affiliate();
		PrimaryApplication tmpApplication = new PrimaryApplication();
		record.setValuesOfMatchingClass(tmpAffiliate, Affiliate.class, affiliatesMapper);
		record.setValuesOfMatchingClass(tmpApplication, Application.class, affiliatesMapper);
		Affiliate affiliate = affiliateRepository.findByImportKeyAndHomeMember(tmpAffiliate.getImportKey(),tmpApplication.getMember());
		return affiliate;
	}

	private void updateAffiliate(Affiliate affiliate, LineRecord record, ImportResult result) throws IllegalAccessException, InstantiationException {
		record.setValuesOfMatchingClass(affiliate, Affiliate.class, affiliatesMapper);
		record.setValuesOfMatchingClass(affiliate.getAffiliateDetails(), AffiliateDetails.class, affiliatesMapper);
	}

	private void createApprovedAffiliate(LineRecord record, ImportResult result) throws IllegalArgumentException, IllegalAccessException, InstantiationException {
		
		//FIXME use services for much of this where possible... with a create and approve steps
		
		PrimaryApplication application = createApprovedPrimaryApplication(record);
		application = applicationRepository.save(application);
		
		CommercialUsage commercialUsage = createCommercialUsage(record, application.getType());
		commercialUsage = commercialUsageRepository.save(commercialUsage);
		
		application.setCommercialUsage(commercialUsage);
		application = applicationRepository.save(application);

		AffiliateDetails affiliateDetails = createPopulateAffiliateDetails(record);
		affiliateDetails = affiliateDetailsRepository.save(affiliateDetails);
		
		Affiliate affiliate = createAffiliate(record, application, affiliateDetails);
		affiliate = affiliateRepository.save(affiliate);
		
		result.sourceMemberKey = affiliate.getHomeMember().getKey();
	}

	private Affiliate createAffiliate(LineRecord record, PrimaryApplication application, AffiliateDetails affiliateDetails) throws IllegalAccessException, InstantiationException {
		Affiliate affiliate = new Affiliate();
		
		affiliate.setAffiliateDetails(affiliateDetails);
		populateWithAll(affiliate, record, Affiliate.class);<commit_msg>MLDS-325 notes<commit_after>package ca.intelliware.ihtsdo.mlds.service.affiliatesimport;

import java.io.IOException;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.List;

import javax.annotation.Resource;
import javax.persistence.EntityManager;
import javax.persistence.PersistenceContext;
import javax.transaction.Transactional;

import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.joda.time.LocalDate;
import org.springframework.stereotype.Service;

import ca.intelliware.ihtsdo.mlds.domain.Affiliate;
import ca.intelliware.ihtsdo.mlds.domain.AffiliateDetails;
import ca.intelliware.ihtsdo.mlds.domain.AffiliateType;
import ca.intelliware.ihtsdo.mlds.domain.Application;
import ca.intelliware.ihtsdo.mlds.domain.Application.ApplicationType;
import ca.intelliware.ihtsdo.mlds.domain.ApprovalState;
import ca.intelliware.ihtsdo.mlds.domain.CommercialUsage;
import ca.intelliware.ihtsdo.mlds.domain.CommercialUsagePeriod;
import ca.intelliware.ihtsdo.mlds.domain.PrimaryApplication;
import ca.intelliware.ihtsdo.mlds.domain.UsageContext;
import ca.intelliware.ihtsdo.mlds.repository.AffiliateDetailsRepository;
import ca.intelliware.ihtsdo.mlds.repository.AffiliateRepository;
import ca.intelliware.ihtsdo.mlds.repository.ApplicationRepository;
import ca.intelliware.ihtsdo.mlds.repository.CommercialUsageRepository;

@Service
@Transactional
public class AffiliatesImporterService {

	@Resource ApplicationRepository applicationRepository;
	@Resource AffiliateRepository affiliateRepository;
	@Resource AffiliateDetailsRepository affiliateDetailsRepository;
	@Resource CommercialUsageRepository commercialUsageRepository;
	
	@Resource AffiliatesMapper affiliatesMapper;
	
	@PersistenceContext
    private EntityManager entityManager;
	
	public ImportResult importFromCSV(String contents) throws IOException {
		ImportResult result = new ImportResult();
		long start = System.currentTimeMillis();
		try {
			importContents(contents, result);
		} catch (Exception e) {
			result.addException(e);
		}
		result.durationMillis = System.currentTimeMillis() - start;
		return result;
	}

	private void importContents(String contents, ImportResult result) throws IOException {
		List<LineRecord> lines = parseFile(contents);
		result.readRecords = lines.size();
		validateLines(lines, result);
		if (result.success) {
			processAffiliateRecords(lines, result);
		}
		
	}

	private void processAffiliateRecords(List<LineRecord> lines, ImportResult result) {
		result.importedRecords = 0;
		for (int i = 0; i < lines.size(); i++) {
			LineRecord lineRecord = lines.get(i);
			if (!lineRecord.header && !lineRecord.isBlank) {
				try {
					processLineRecord(lineRecord, result);
					result.importedRecords += 1;
				} catch (Exception e) {
					result.addError(lineRecord, "Failed to populate record: "+e);
				}
			}
		}
	}

	void processLineRecord(LineRecord record, ImportResult result) throws IllegalArgumentException, IllegalAccessException, InstantiationException {
		Affiliate affiliate = findExistingAffiliateForUpdate(record);
		if (affiliate == null) {
			createApprovedAffiliate(record,result);
		} else {
			updateAffiliate(affiliate,record,result);
		}
		clearSession();
	}

	/** For bulk performance clear pending changes from hibernate session */
	private void clearSession() {
		entityManager.flush();
		entityManager.clear();
	}

	private Affiliate findExistingAffiliateForUpdate(LineRecord record) throws IllegalAccessException, InstantiationException {
		Affiliate tmpAffiliate = new Affiliate();
		PrimaryApplication tmpApplication = new PrimaryApplication();
		record.setValuesOfMatchingClass(tmpAffiliate, Affiliate.class, affiliatesMapper);
		record.setValuesOfMatchingClass(tmpApplication, Application.class, affiliatesMapper);
		Affiliate affiliate = affiliateRepository.findByImportKeyAndHomeMember(tmpAffiliate.getImportKey(),tmpApplication.getMember());
		return affiliate;
	}

	private void updateAffiliate(Affiliate affiliate, LineRecord record, ImportResult result) throws IllegalAccessException, InstantiationException {
		record.setValuesOfMatchingClass(affiliate, Affiliate.class, affiliatesMapper);
		record.setValuesOfMatchingClass(affiliate.getAffiliateDetails(), AffiliateDetails.class, affiliatesMapper);
		// FIXME MLDS-325 MB what other objects should we update?
		// FIXME MLDS-325 MB create an "Import" application for history?
	}

	private void createApprovedAffiliate(LineRecord record, ImportResult result) throws IllegalArgumentException, IllegalAccessException, InstantiationException {
		
		//FIXME use services for much of this where possible... with a create and approve steps
		
		PrimaryApplication application = createApprovedPrimaryApplication(record);
		application = applicationRepository.save(application);
		
		CommercialUsage commercialUsage = createCommercialUsage(record, application.getType());
		commercialUsage = commercialUsageRepository.save(commercialUsage);
		
		application.setCommercialUsage(commercialUsage);
		application = applicationRepository.save(application);

		AffiliateDetails affiliateDetails = createPopulateAffiliateDetails(record);
		affiliateDetails = affiliateDetailsRepository.save(affiliateDetails);
		
		Affiliate affiliate = createAffiliate(record, application, affiliateDetails);
		affiliate = affiliateRepository.save(affiliate);
		
		result.sourceMemberKey = affiliate.getHomeMember().getKey();
	}

	private Affiliate createAffiliate(LineRecord record, PrimaryApplication application, AffiliateDetails affiliateDetails) throws IllegalAccessException, InstantiationException {
		Affiliate affiliate = new Affiliate();
		
		affiliate.setAffiliateDetails(affiliateDetails);
		populateWithAll(affiliate, record, Affiliate.class);
{'fraction_non_alphanumeric': 0.08514175708785439, 'fraction_numerical': 0.001312565628281414, 'mean_word_length': 6.573889993373095, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3756356', 'n_tokens_mistral': 3437, 'n_tokens_neox': 3178, 'n_words': 628}