<commit_before>package org.slc.sli.ingestion.transformation;

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.slc.sli.domain.NeutralCriteria;
import org.slc.sli.domain.NeutralQuery;
import org.slc.sli.ingestion.FileType;
import org.slc.sli.ingestion.NeutralRecord;
import org.slc.sli.ingestion.NeutralRecordFileWriter;
import org.slc.sli.ingestion.landingzone.IngestionFileEntry;
import org.slc.sli.ingestion.util.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;

/**
 * Transformer for Assessment Entities
 * 
 * @author ifaybyshev
 * 
 */
public class AssessmentCombiner extends AbstractTransformationStrategy {
    
    public static final String SUB_OBJECTIVE_REFS = "subObjectiveRefs";
    
    private static final Logger LOG = LoggerFactory.getLogger(AssessmentCombiner.class);
    
    private Map<String, Map<Object, NeutralRecord>> collections;
    
    private List<NeutralRecord> transformedAssessments = new ArrayList<NeutralRecord>();
    
    private final FileUtils fileUtils;
    
    @Autowired
    public AssessmentCombiner(FileUtils fileUtils) {
        this.collections = new HashMap<String, Map<Object, NeutralRecord>>();
        this.fileUtils = fileUtils;
    }
    
    /**
     * The chaining of transformation steps. This implementation assumes that all data will be
     * processed in "one-go"
     * 
     */
    @Override
    public void performTransformation() {
        loadData();
        transform();
        persist();
    }
    
    public void loadData() {
        LOG.info("Loading data for transformation.");
        
        loadCollectionFromDb("assessment");
        LOG.info("Assessment is loaded into local storage.  Total Count = " + collections.get("assessment").size());
        
        loadCollectionFromDb("assessmentFamily");
        LOG.info("AssessmentFamily is loaded into local storage.  Total Count = "
                + collections.get("assessmentFamily").size());
    }
    
    public void transform() {
        LOG.debug("Transforming data: Injecting assessmentFamilies into assessment");
        
        for (Map.Entry<Object, NeutralRecord> neutralRecordEntry : collections.get("assessment").entrySet()) {
            NeutralRecord neutralRecord = neutralRecordEntry.getValue();
            
            // get the key of parent
            Map<String, Object> attrs = neutralRecord.getAttributes();
            String parentFamilyId = (String) attrs.get("parentAssessmentFamilyId");
            attrs.remove("parentAssessmentFamilyId");
            String familyHierarchyName = "";
            familyHierarchyName = getAssocationFamilyMap(parentFamilyId, new HashMap<String, Map<String, Object>>(),
                    familyHierarchyName);
            
            attrs.put("assessmentFamilyHierarchyName", familyHierarchyName);
            
            @SuppressWarnings("unchecked")
            List<String> objectiveAssessmentRefs = (List<String>) attrs.get("objectiveAssessmentRefs");
            attrs.remove("objectiveAssessmentRefs");
            List<Map<String, Object>> objectiveAssessments = new ArrayList<Map<String, Object>>();
            if (objectiveAssessmentRefs != null && !(objectiveAssessmentRefs.isEmpty())) {
                
                for (String objectiveAssessmentRef : objectiveAssessmentRefs) {
                    
                    objectiveAssessments.add(getObjectiveAssessment(objectiveAssessmentRef));
                }
                attrs.put("objectiveAssessment", objectiveAssessments);
            }
            
            String assessmentPeriodDescriptorRef = (String) attrs.get("periodDescriptorRef");
            attrs.remove("periodDescriptorRef");
            if (assessmentPeriodDescriptorRef != null) {
                
                attrs.put("assessmentPeriodDescriptor", getAssessmentPeriodDescriptor(assessmentPeriodDescriptorRef));
                
            }
            neutralRecord.setAttributes(attrs);
            transformedAssessments.add(neutralRecord);
        }
        <commit_msg>missing external id<commit_after>package org.slc.sli.ingestion.transformation;

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.slc.sli.domain.NeutralCriteria;
import org.slc.sli.domain.NeutralQuery;
import org.slc.sli.ingestion.FileType;
import org.slc.sli.ingestion.NeutralRecord;
import org.slc.sli.ingestion.NeutralRecordFileWriter;
import org.slc.sli.ingestion.landingzone.IngestionFileEntry;
import org.slc.sli.ingestion.util.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;

/**
 * Transformer for Assessment Entities
 * 
 * @author ifaybyshev
 * 
 */
public class AssessmentCombiner extends AbstractTransformationStrategy {
    
    public static final String SUB_OBJECTIVE_REFS = "subObjectiveRefs";
    
    private static final Logger LOG = LoggerFactory.getLogger(AssessmentCombiner.class);
    
    private Map<String, Map<Object, NeutralRecord>> collections;
    
    private List<NeutralRecord> transformedAssessments = new ArrayList<NeutralRecord>();
    
    private final FileUtils fileUtils;
    
    @Autowired
    public AssessmentCombiner(FileUtils fileUtils) {
        this.collections = new HashMap<String, Map<Object, NeutralRecord>>();
        this.fileUtils = fileUtils;
    }
    
    /**
     * The chaining of transformation steps. This implementation assumes that all data will be
     * processed in "one-go"
     * 
     */
    @Override
    public void performTransformation() {
        loadData();
        transform();
        persist();
    }
    
    public void loadData() {
        LOG.info("Loading data for transformation.");
        
        loadCollectionFromDb("assessment");
        LOG.info("Assessment is loaded into local storage.  Total Count = " + collections.get("assessment").size());
        
        loadCollectionFromDb("assessmentFamily");
        LOG.info("AssessmentFamily is loaded into local storage.  Total Count = "
                + collections.get("assessmentFamily").size());
    }
    
    public void transform() {
        LOG.debug("Transforming data: Injecting assessmentFamilies into assessment");
        
        for (Map.Entry<Object, NeutralRecord> neutralRecordEntry : collections.get("assessment").entrySet()) {
            NeutralRecord neutralRecord = neutralRecordEntry.getValue();
            
            // get the key of parent
            Map<String, Object> attrs = neutralRecord.getAttributes();
            String parentFamilyId = (String) attrs.get("parentAssessmentFamilyId");
            attrs.remove("parentAssessmentFamilyId");
            String familyHierarchyName = "";
            familyHierarchyName = getAssocationFamilyMap(parentFamilyId, new HashMap<String, Map<String, Object>>(),
                    familyHierarchyName);
            
            attrs.put("assessmentFamilyHierarchyName", familyHierarchyName);
            
            @SuppressWarnings("unchecked")
            List<String> objectiveAssessmentRefs = (List<String>) attrs.get("objectiveAssessmentRefs");
            attrs.remove("objectiveAssessmentRefs");
            List<Map<String, Object>> objectiveAssessments = new ArrayList<Map<String, Object>>();
            if (objectiveAssessmentRefs != null && !(objectiveAssessmentRefs.isEmpty())) {
                
                for (String objectiveAssessmentRef : objectiveAssessmentRefs) {
                    
                    objectiveAssessments.add(getObjectiveAssessment(objectiveAssessmentRef));
                }
                attrs.put("objectiveAssessment", objectiveAssessments);
            }
            
            String assessmentPeriodDescriptorRef = (String) attrs.get("periodDescriptorRef");
            attrs.remove("periodDescriptorRef");
            if (assessmentPeriodDescriptorRef != null) {
                
                attrs.put("assessmentPeriodDescriptor", getAssessmentPeriodDescriptor(assessmentPeriodDescriptorRef));
                
            }
            neutralRecord.setAttributes(attrs);
            transformedAssessments.add(neutralRecord);
            neutralRecord.setLocalId(attrs.get("assessmentTitle"));
        }
        
{'fraction_non_alphanumeric': 0.09647557439214811, 'fraction_numerical': 0.00044612982377871963, 'mean_word_length': 2.8650862068965517, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6278030', 'n_tokens_mistral': 2368, 'n_tokens_neox': 2325, 'n_words': 445}