<commit_before>package org.slc.sli.manager.impl;

import java.text.DateFormat;
import java.text.NumberFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;

import org.apache.commons.lang.WordUtils;
import org.joda.time.DateTime;
import org.joda.time.MutableDateTime;
import org.joda.time.format.DateTimeFormat;
import org.joda.time.format.DateTimeFormatter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;

import org.slc.sli.entity.Config;
import org.slc.sli.entity.GenericEntity;
import org.slc.sli.entity.util.GenericEntityEnhancer;
import org.slc.sli.manager.ApiClientManager;
import org.slc.sli.manager.EntityManager;
import org.slc.sli.manager.PopulationManager;
import org.slc.sli.util.Constants;
import org.slc.sli.util.TimedLogic;

/**
 * PopulationManager facilitates creation of logical aggregations of EdFi
 * entities/associations such as a student summary comprised of student profile,
 * enrollment, program, and assessment information in order to deliver the
 * Population Summary interaction.
 *
 * @author <NAME> <EMAIL>
 *
 */
public class PopulationManagerImpl extends ApiClientManager implements PopulationManager {

    private static final String ATTENDANCE_TARDY = "Tardy";
    private static final String ATTENDANCE_ABSENCE = "Absence";

    private static final String STUDENT_CACHE = "user.student";

    private static Logger log = LoggerFactory.getLogger(PopulationManagerImpl.class);

    @Autowired
    private EntityManager entityManager;

    public PopulationManagerImpl() {

    }

    /*
    * (non-Javadoc)
    *
    * @see
    * org.slc.sli.manager.PopulationManagerI#getAssessments(java.lang.String,
    * java.util.List)
    */
    @Override
    public List<GenericEntity> getAssessments(String token, List<GenericEntity> studentSummaries) {
        Set<GenericEntity> assessments = new TreeSet<GenericEntity>(new Comparator<GenericEntity>() {
            @Override
            public int compare(GenericEntity att1, GenericEntity att2) {
                return (att2.getString("id")).compareTo(att1.getString("id"));
            }
        });
        for (GenericEntity studentSummary : studentSummaries) {
            List<Map<String, Object>> studentAssessments = (List<Map<String, Object>>) studentSummary
                    .get(Constants.ATTR_STUDENT_ASSESSMENTS);

            for (Map<String, Object> studentAssessment : studentAssessments) {
                try {
                    GenericEntity assessment = new GenericEntity((Map) studentAssessment.get("assessments"));
                    assessments.add(assessment);
                } catch (ClassCastException cce) {
                    log.warn(cce.getMessage());
                }
            }
        }
        return new ArrayList<GenericEntity>(assessments);
    }

    /*
    * (non-Javadoc)
    *
    * @see
    * org.slc.sli.manager.PopulationManagerI#getStudentSummaries(java.lang.
    * String, java.util.List, org.slc.sli.config.ViewConfig, java.lang.String,
    * java.lang.String)
    */
    @Override
    public List<GenericEntity> getStudentSummaries(String token, List<String> studentIds,
                                                   String sessionId, String sectionId) {

        long startTime = System.nanoTime();
        // Initialize student summaries

        List<GenericEntity> studentSummaries = entityManager.getStudents(token, sectionId, studentIds);
        log.warn("@@@@@@@@@@@@@@@@@@ Benchmark for student section view: {}", (System.nanoTime() - startTime) * 1.0e-9);

        return studentSummaries;
    }


    /*
    * (non-Javadoc)
    *
    * @see
    * org.slc.sli.manager.PopulationManagerI#getListOfStudents(java.lang.String
    * , java.lang.Object, org.slc.sli.entity.Config.Data)
    */
    @Override
    public GenericEntity getListOfStudents(String token, Object sectionId, Config.Data config) {

        // get student summary data
        List<GenericEntity> studentSummaries = getStudentSummaries(token, null, null, (String) sectionId);

        // apply assmt filters and flatten assmt data structure for easy
        // fetching
        applyAssessmentFilters(studentSummaries, config);

        // data enhancements
        enhanceListOfStudents(studentSummaries, (String) sectionId);

        GenericEntity result = new GenericEntity();
        result.put(Constants.ATTR_STUDENTS, studentSummaries);

        return result;
    }

    /**
     * Make enhancements that make it easier for front-end javascript to use the
     * data
     *
     * @param studentSummaries
     */
    public void enhanceListOfStudents(List<GenericEntity> studentSummaries, String sectionId) {
        if (studentSummaries != null) {
            for (GenericEntity student : studentSummaries) {
                if (student == null) {
                    continue;
                }

                // clean out some unneeded gunk
                scrubStudentData(student);

                // add full name
                addFullName(student);

                // add the final grade
                addFinalGrades(student, sectionId);

                // add the grade book
                addCurrentSemesterGrades(student, sectionId);

                // transform assessment score format
                transformAssessmentFormat(student);

                // tally up attendance data
                tallyAttendanceData(student);

            }

        }

    }

    /**
     * Create an attribute for the full student name (first name + last name)
     *
     * @param student
     */
    public void addFullName(GenericEntity student) {

        Map name = (Map) student.get(Constants.ATTR_NAME);
        if (name != null) {
            String fullName = (String) name.get(Constants.ATTR_FIRST_NAME) + " "
                    + (String) name.get(Constants.ATTR_LAST_SURNAME);
            name.put(Constants.ATTR_FULL_NAME, fullName);
        }
    }

    /**
     * Tally up individual attendance events. Front-end needs to show aggregated
     * results.
     *
     * @param student
     */
    public void tallyAttendanceData(GenericEntity student) {

        Map<String, Object> attendanceBody = (Map<String, Object>) student.get(Constants.ATTR_STUDENT_ATTENDANCES);
        if (attendanceBody != null) {

            List<Map<String, Object>> attendances = (List<Map<String, Object>>) attendanceBody
                    .get(Constants.ATTR_STUDENT_ATTENDANCES);
            int absenceCount = 0;
            int tardyCount = 0;
            if (attendances != null && attendances.size() > 0) {
                for (Map attendance : attendances) {

                    String eventCategory = (String) attendance.get(Constants.ATTR_ATTENDANCE_EVENT_CATEGORY);

                    if (eventCategory.contains(ATTENDANCE_ABSENCE)) {
                        absenceCount++;

                    } else if (eventCategory.contains(ATTENDANCE_TARDY)) {
                        tardyCount++;
                    }
                }

                int attendanceRate = Math
                        .round((((float) (attendances.size() - absenceCount)) / attendances.size()) * 100);
                int tardyRate = Math.round((((float) tardyCount) / attendances.size()) * 100);

                attendanceBody.remove(Constants.ATTR_STUDENT_ATTENDANCES);
                attendanceBody.put(Constants.ATTR_ABSENCE_COUNT, absenceCount);
                attendanceBody.put(Constants.ATTR_TARDY_COUNT, tardyCount);
                attendanceBody.put(Constants.ATTR_ATTENDANCE_RATE, attendanceRate);
                attendanceBody.put(Constants.ATTR_TARDY_RATE, tardyRate);
            }
        }
    }

    /**
     * Modify the data structure for assessments, for front-end convenience
     *
     * @param student
     */
    public void transformAssessmentFormat(GenericEntity student) {

        Map studentAssmtAssocs = (Map) student.get(Constants.ATTR_ASSESSMENTS);
        if (studentAssmtAssocs == null) {
            return;
        }

        Collection<Map> assmtResults = studentAssmtAssocs.values();
        for (Map assmtResult : assmtResults) {

            if (assmtResult == null) {
                continue;
            }

            // for each score result, create a new attribute that makes the
            // score easily accessible
            // without looping through this list
            List<Map> scoreResults = (List<Map>) assmtResult.get(Constants.ATTR_SCORE_RESULTS);
            if (scoreResults != null) {

                for (Map scoreResult : scoreResults) {

                    String type = (String) scoreResult.get(Constants.ATTR_ASSESSMENT_REPORTING_METHOD);
                    String result = (String) scoreResult.get(Constants.ATTR_RESULT);
                    assmtResult.put(type, result);
                }
            }

            // create a new attribute "perfLevel"
            List<List<Map>> perfLevelsDescs = (List<List<Map>>) assmtResult
                    .get(Constants.ATTR_PERFORMANCE_LEVEL_DESCRIPTOR);

            if (perfLevelsDescs != null) {
                for (List<Map> perfLevelsDesc : perfLevelsDescs) {
                    if (perfLevelsDesc != null && perfLevelsDesc.size() > 0) {

                        String perfLevel = (String) perfLevelsDesc.get(0).get(Constants.ATTR_CODE_VALUE);
                        assmtResult.put(Constants.ATTR_PERF_LEVEL, perfLevel);

                    }
                }
            }
        }
    }

    /**
     * Clean out some student data, so we don't pass too much unnecessary stuff
     * to the front-end
     *
     * @param student
     */
    public void scrubStudentData(GenericEntity student) {

        student.remove(Constants.ATTR_LINKS);
    }

    /**
     * Grabs the subject area from the data based on the section ID.
     *
     * @param stuSectAssocs
     * @param sectionId
     * @return
     */
    @SuppressWarnings({ "unchecked", "rawtypes" })
    private String getSubjectArea(List<Map<String, Object>> stuSectAssocs, String sectionId) {
        String subjectArea = null;
        for (Map<String, Object> assoc : stuSectAssocs) {
            if (sectionId.equalsIgnoreCase((String) assoc.get(Constants.ATTR_SECTION_ID))) {
                Map<String, Object> sections = (Map) assoc.get(Constants.ATTR_SECTIONS);
                subjectArea = getSubjectArea(sections);
                break;
            }
        }

        return subjectArea;
    }
    
    private String getSubjectArea(Map<String, Object> sections) {
        String subjectArea = null;
        
        if(sections == null){
            return subjectArea;
        }
        
        Map<String, Object> courses = (Map) sections.get(Constants.ATTR_COURSES);
        if(courses == null) {
            return subjectArea;
        }
        
        return (String) courses.get(Constants.ATTR_SUBJECTAREA);
    }

    /**
     * Extracts grades from transcriptAssociationRecord based on sections in the
     * past. For each section where a transcript with final letter grade
     * exist, the grade is added to the list of grades for the semester.
     *
     * @param student
     * @param interSections
     * @param stuTransAssocs
     */
    @SuppressWarnings({ "unchecked", "rawtypes" })
    private void addSemesterFinalGrades(GenericEntity student, List<Map<String, Object>> interSections,
                                        List<Map<String, Object>> stuTransAssocs) {

        // Iterate through the course Id's and grab transcripts grades, once
        // we have NUMBER_OF_SEMESTERS transcript grades, we're done
        for (Map<String, Object> section : interSections) {
            String courseId = (String) section.get(Constants.ATTR_COURSE_ID);

            // Find the correct course. If that course is found in
            // the transcript, then record that letter grade to the
            // semesterScores.
            for (Map<String, Object> assoc : stuTransAssocs) {
                if (courseId.equalsIgnoreCase((String) assoc.get(Constants.ATTR_COURSE_ID))) {
                    String finalLetterGrade = (String) assoc.get(Constants.ATTR_FINAL_LETTER_GRADE);
                    String courseTitle = (String) ((Map) section.get(Constants.ATTR_COURSES))
                            .get(Constants.ATTR_COURSE_TITLE);
                    if (finalLetterGrade != null) {
                        String semesterString = buildSemesterYearString(section);
                        Map<String, Object> grade = new LinkedHashMap<String, Object>();
                        grade.put(Constants.SECTION_LETTER_GRADE, finalLetterGrade);
                        grade.put(Constants.SECTION_COURSE, courseTitle);
                        List<Map<String, Object>> semesterScores = (List<Map<String, Object>>) student
                                .get(semesterString);
                        if (semesterScores == null) {
                            semesterScores = new ArrayList<Map<String, Object>>();
                        }
                        semesterScores.add(grade);
                        student.put(semesterString, semesterScores);
                        break;
                    }
                }
            }
        }
    }

    /**
     * This method adds the final grades of a student to the student data. It
     * will only grab the latest two grades. Ideally we would filter on subject
     * area, but there is currently no subject area data in the SDS.
     *
     * @param student
     */
    @SuppressWarnings({ "unchecked", "rawtypes" })
    private void addFinalGrades(GenericEntity student, String sectionId) {
        try {
            Map<String, Object> transcripts = (Map<String, Object>) student.get(Constants.ATTR_TRANSCRIPT);
            if (transcripts == null) {
                return;
            }

            /*
             * For each student section association, we have to determine if it is in the same
             * subject area as the sectionid passed.  If it is then we add it to our List.
             * Once we have a list of sections. We can grab all of the semester grades
             * for those sections whose subject area intersect.
             */
            List<Map<String, Object>> stuSectAssocs = (List<Map<String, Object>>) transcripts
                    .get(Constants.ATTR_STUDENT_SECTION_ASSOC);
            if(stuSectAssocs == null) {
                return;
            }
            
            String subjectArea = getSubjectArea(stuSectAssocs, sectionId);
            List<Map<String, Object>> interSections = new ArrayList<Map<String, Object>>();
            for (Map<String, Object> assoc : stuSectAssocs) {
                Map<String, Object> sections = (Map<String, Object>) assoc.get(Constants.ATTR_SECTIONS);
                // This case will catch if the subjectArea is null
                if (subjectArea == null || subjectArea.equalsIgnoreCase(getSubjectArea(sections))) {
                    interSections.add(sections);
                }
            }

            addSemesterFinalGrades(student, interSections,
                    (List<Map<String, Object>>) transcripts.get(Constants.ATTR_STUDENT_TRANSCRIPT_ASSOC));

        } catch (ClassCastException ex) {
            log.error("Error occured processing Final Grades", ex);
            Map<String, Object> grade = new LinkedHashMap<String, Object>();
            student.put(Constants.ATTR_SCORE_RESULTS, grade.put(Constants.ATTR_FINAL_LETTER_GRADE, "?"));
        } catch (NullPointerException ex) {
            log.error("Error occured processing Final Grades", ex);
            Map<String, Object> grade = new LinkedHashMap<String, Object>();
            student.put(Constants.ATTR_SCORE_RESULTS, grade.put(Constants.ATTR_FINAL_LETTER_GRADE, "?"));
        }
    }

    /**
     * Returns the term and the year as a string for a given student Section association.<commit_msg>Fixed Checkstyle<commit_after>package org.slc.sli.manager.impl;

import java.text.DateFormat;
import java.text.NumberFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;

import org.apache.commons.lang.WordUtils;
import org.joda.time.DateTime;
import org.joda.time.MutableDateTime;
import org.joda.time.format.DateTimeFormat;
import org.joda.time.format.DateTimeFormatter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;

import org.slc.sli.entity.Config;
import org.slc.sli.entity.GenericEntity;
import org.slc.sli.entity.util.GenericEntityEnhancer;
import org.slc.sli.manager.ApiClientManager;
import org.slc.sli.manager.EntityManager;
import org.slc.sli.manager.PopulationManager;
import org.slc.sli.util.Constants;
import org.slc.sli.util.TimedLogic;

/**
 * PopulationManager facilitates creation of logical aggregations of EdFi
 * entities/associations such as a student summary comprised of student profile,
 * enrollment, program, and assessment information in order to deliver the
 * Population Summary interaction.
 *
 * @author <NAME> <EMAIL>
 *
 */
public class PopulationManagerImpl extends ApiClientManager implements PopulationManager {

    private static final String ATTENDANCE_TARDY = "Tardy";
    private static final String ATTENDANCE_ABSENCE = "Absence";

    private static final String STUDENT_CACHE = "user.student";

    private static Logger log = LoggerFactory.getLogger(PopulationManagerImpl.class);

    @Autowired
    private EntityManager entityManager;

    public PopulationManagerImpl() {

    }

    /*
    * (non-Javadoc)
    *
    * @see
    * org.slc.sli.manager.PopulationManagerI#getAssessments(java.lang.String,
    * java.util.List)
    */
    @Override
    public List<GenericEntity> getAssessments(String token, List<GenericEntity> studentSummaries) {
        Set<GenericEntity> assessments = new TreeSet<GenericEntity>(new Comparator<GenericEntity>() {
            @Override
            public int compare(GenericEntity att1, GenericEntity att2) {
                return (att2.getString("id")).compareTo(att1.getString("id"));
            }
        });
        for (GenericEntity studentSummary : studentSummaries) {
            List<Map<String, Object>> studentAssessments = (List<Map<String, Object>>) studentSummary
                    .get(Constants.ATTR_STUDENT_ASSESSMENTS);

            for (Map<String, Object> studentAssessment : studentAssessments) {
                try {
                    GenericEntity assessment = new GenericEntity((Map) studentAssessment.get("assessments"));
                    assessments.add(assessment);
                } catch (ClassCastException cce) {
                    log.warn(cce.getMessage());
                }
            }
        }
        return new ArrayList<GenericEntity>(assessments);
    }

    /*
    * (non-Javadoc)
    *
    * @see
    * org.slc.sli.manager.PopulationManagerI#getStudentSummaries(java.lang.
    * String, java.util.List, org.slc.sli.config.ViewConfig, java.lang.String,
    * java.lang.String)
    */
    @Override
    public List<GenericEntity> getStudentSummaries(String token, List<String> studentIds,
                                                   String sessionId, String sectionId) {

        long startTime = System.nanoTime();
        // Initialize student summaries

        List<GenericEntity> studentSummaries = entityManager.getStudents(token, sectionId, studentIds);
        log.warn("@@@@@@@@@@@@@@@@@@ Benchmark for student section view: {}", (System.nanoTime() - startTime) * 1.0e-9);

        return studentSummaries;
    }


    /*
    * (non-Javadoc)
    *
    * @see
    * org.slc.sli.manager.PopulationManagerI#getListOfStudents(java.lang.String
    * , java.lang.Object, org.slc.sli.entity.Config.Data)
    */
    @Override
    public GenericEntity getListOfStudents(String token, Object sectionId, Config.Data config) {

        // get student summary data
        List<GenericEntity> studentSummaries = getStudentSummaries(token, null, null, (String) sectionId);

        // apply assmt filters and flatten assmt data structure for easy
        // fetching
        applyAssessmentFilters(studentSummaries, config);

        // data enhancements
        enhanceListOfStudents(studentSummaries, (String) sectionId);

        GenericEntity result = new GenericEntity();
        result.put(Constants.ATTR_STUDENTS, studentSummaries);

        return result;
    }

    /**
     * Make enhancements that make it easier for front-end javascript to use the
     * data
     *
     * @param studentSummaries
     */
    public void enhanceListOfStudents(List<GenericEntity> studentSummaries, String sectionId) {
        if (studentSummaries != null) {
            for (GenericEntity student : studentSummaries) {
                if (student == null) {
                    continue;
                }

                // clean out some unneeded gunk
                scrubStudentData(student);

                // add full name
                addFullName(student);

                // add the final grade
                addFinalGrades(student, sectionId);

                // add the grade book
                addCurrentSemesterGrades(student, sectionId);

                // transform assessment score format
                transformAssessmentFormat(student);

                // tally up attendance data
                tallyAttendanceData(student);

            }

        }

    }

    /**
     * Create an attribute for the full student name (first name + last name)
     *
     * @param student
     */
    public void addFullName(GenericEntity student) {

        Map name = (Map) student.get(Constants.ATTR_NAME);
        if (name != null) {
            String fullName = (String) name.get(Constants.ATTR_FIRST_NAME) + " "
                    + (String) name.get(Constants.ATTR_LAST_SURNAME);
            name.put(Constants.ATTR_FULL_NAME, fullName);
        }
    }

    /**
     * Tally up individual attendance events. Front-end needs to show aggregated
     * results.
     *
     * @param student
     */
    public void tallyAttendanceData(GenericEntity student) {

        Map<String, Object> attendanceBody = (Map<String, Object>) student.get(Constants.ATTR_STUDENT_ATTENDANCES);
        if (attendanceBody != null) {

            List<Map<String, Object>> attendances = (List<Map<String, Object>>) attendanceBody
                    .get(Constants.ATTR_STUDENT_ATTENDANCES);
            int absenceCount = 0;
            int tardyCount = 0;
            if (attendances != null && attendances.size() > 0) {
                for (Map attendance : attendances) {

                    String eventCategory = (String) attendance.get(Constants.ATTR_ATTENDANCE_EVENT_CATEGORY);

                    if (eventCategory.contains(ATTENDANCE_ABSENCE)) {
                        absenceCount++;

                    } else if (eventCategory.contains(ATTENDANCE_TARDY)) {
                        tardyCount++;
                    }
                }

                int attendanceRate = Math
                        .round((((float) (attendances.size() - absenceCount)) / attendances.size()) * 100);
                int tardyRate = Math.round((((float) tardyCount) / attendances.size()) * 100);

                attendanceBody.remove(Constants.ATTR_STUDENT_ATTENDANCES);
                attendanceBody.put(Constants.ATTR_ABSENCE_COUNT, absenceCount);
                attendanceBody.put(Constants.ATTR_TARDY_COUNT, tardyCount);
                attendanceBody.put(Constants.ATTR_ATTENDANCE_RATE, attendanceRate);
                attendanceBody.put(Constants.ATTR_TARDY_RATE, tardyRate);
            }
        }
    }

    /**
     * Modify the data structure for assessments, for front-end convenience
     *
     * @param student
     */
    public void transformAssessmentFormat(GenericEntity student) {

        Map studentAssmtAssocs = (Map) student.get(Constants.ATTR_ASSESSMENTS);
        if (studentAssmtAssocs == null) {
            return;
        }

        Collection<Map> assmtResults = studentAssmtAssocs.values();
        for (Map assmtResult : assmtResults) {

            if (assmtResult == null) {
                continue;
            }

            // for each score result, create a new attribute that makes the
            // score easily accessible
            // without looping through this list
            List<Map> scoreResults = (List<Map>) assmtResult.get(Constants.ATTR_SCORE_RESULTS);
            if (scoreResults != null) {

                for (Map scoreResult : scoreResults) {

                    String type = (String) scoreResult.get(Constants.ATTR_ASSESSMENT_REPORTING_METHOD);
                    String result = (String) scoreResult.get(Constants.ATTR_RESULT);
                    assmtResult.put(type, result);
                }
            }

            // create a new attribute "perfLevel"
            List<List<Map>> perfLevelsDescs = (List<List<Map>>) assmtResult
                    .get(Constants.ATTR_PERFORMANCE_LEVEL_DESCRIPTOR);

            if (perfLevelsDescs != null) {
                for (List<Map> perfLevelsDesc : perfLevelsDescs) {
                    if (perfLevelsDesc != null && perfLevelsDesc.size() > 0) {

                        String perfLevel = (String) perfLevelsDesc.get(0).get(Constants.ATTR_CODE_VALUE);
                        assmtResult.put(Constants.ATTR_PERF_LEVEL, perfLevel);

                    }
                }
            }
        }
    }

    /**
     * Clean out some student data, so we don't pass too much unnecessary stuff
     * to the front-end
     *
     * @param student
     */
    public void scrubStudentData(GenericEntity student) {

        student.remove(Constants.ATTR_LINKS);
    }

    /**
     * Grabs the subject area from the data based on the section ID.
     * 
     * @param stuSectAssocs
     * @param sectionId
     * @return
     */
    @SuppressWarnings({ "unchecked", "rawtypes" })
    private String getSubjectArea(List<Map<String, Object>> stuSectAssocs, String sectionId) {
        String subjectArea = null;
        for (Map<String, Object> assoc : stuSectAssocs) {
            if (sectionId.equalsIgnoreCase((String) assoc.get(Constants.ATTR_SECTION_ID))) {
                Map<String, Object> sections = (Map) assoc.get(Constants.ATTR_SECTIONS);
                subjectArea = getSubjectArea(sections);
                break;
            }
        }
        
        return subjectArea;
    }
    
    /**
     * Grabs the Subject Area from a section.
     * 
     * @param sections
     * @return
     */
    @SuppressWarnings({ "unchecked", "rawtypes" })
    private String getSubjectArea(Map<String, Object> sections) {
        String subjectArea = null;
        
        if (sections == null) {
            return subjectArea;
        }
        
        Map<String, Object> courses = (Map) sections.get(Constants.ATTR_COURSES);
        if (courses == null) {
            return subjectArea;
        }
        
        return (String) courses.get(Constants.ATTR_SUBJECTAREA);
    }

    /**
     * Extracts grades from transcriptAssociationRecord based on sections in the
     * past. For each section where a transcript with final letter grade
     * exist, the grade is added to the list of grades for the semester.
     *
     * @param student
     * @param interSections
     * @param stuTransAssocs
     */
    @SuppressWarnings({ "unchecked", "rawtypes" })
    private void addSemesterFinalGrades(GenericEntity student, List<Map<String, Object>> interSections,
                                        List<Map<String, Object>> stuTransAssocs) {

        // Iterate through the course Id's and grab transcripts grades, once
        // we have NUMBER_OF_SEMESTERS transcript grades, we're done
        for (Map<String, Object> section : interSections) {
            String courseId = (String) section.get(Constants.ATTR_COURSE_ID);

            // Find the correct course. If that course is found in
            // the transcript, then record that letter grade to the
            // semesterScores.
            for (Map<String, Object> assoc : stuTransAssocs) {
                if (courseId.equalsIgnoreCase((String) assoc.get(Constants.ATTR_COURSE_ID))) {
                    String finalLetterGrade = (String) assoc.get(Constants.ATTR_FINAL_LETTER_GRADE);
                    String courseTitle = (String) ((Map) section.get(Constants.ATTR_COURSES))
                            .get(Constants.ATTR_COURSE_TITLE);
                    if (finalLetterGrade != null) {
                        String semesterString = buildSemesterYearString(section);
                        Map<String, Object> grade = new LinkedHashMap<String, Object>();
                        grade.put(Constants.SECTION_LETTER_GRADE, finalLetterGrade);
                        grade.put(Constants.SECTION_COURSE, courseTitle);
                        List<Map<String, Object>> semesterScores = (List<Map<String, Object>>) student
                                .get(semesterString);
                        if (semesterScores == null) {
                            semesterScores = new ArrayList<Map<String, Object>>();
                        }
                        semesterScores.add(grade);
                        student.put(semesterString, semesterScores);
                        break;
                    }
                }
            }
        }
    }

    /**
     * This method adds the final grades of a student to the student data. It
     * will only grab the latest two grades. Ideally we would filter on subject
     * area, but there is currently no subject area data in the SDS.
     *
     * @param student
     */
    @SuppressWarnings({ "unchecked" })
    private void addFinalGrades(GenericEntity student, String sectionId) {
        try {
            Map<String, Object> transcripts = (Map<String, Object>) student.get(Constants.ATTR_TRANSCRIPT);
            if (transcripts == null) {
                return;
            }

            /*
             * For each student section association, we have to determine if it is in the same
             * subject area as the sectionid passed. If it is then we add it to our List.
             * Once we have a list of sections. We can grab all of the semester grades
             * for those sections whose subject area intersect.
             */
            List<Map<String, Object>> stuSectAssocs = (List<Map<String, Object>>) transcripts
                    .get(Constants.ATTR_STUDENT_SECTION_ASSOC);
            if (stuSectAssocs == null) {
                return;
            }
            
            String subjectArea = getSubjectArea(stuSectAssocs, sectionId);
            List<Map<String, Object>> interSections = new ArrayList<Map<String, Object>>();
            for (Map<String, Object> assoc : stuSectAssocs) {
                Map<String, Object> sections = (Map<String, Object>) assoc.get(Constants.ATTR_SECTIONS);
                // This case will catch if the subjectArea is null
                if (subjectArea == null || subjectArea.equalsIgnoreCase(getSubjectArea(sections))) {
                    interSections.add(sections);
                }
            }

            addSemesterFinalGrades(student, interSections,
                    (List<Map<String, Object>>) transcripts.get(Constants.ATTR_STUDENT_TRANSCRIPT_ASSOC));

        } catch (ClassCastException ex) {
            log.error("Error occured processing Final Grades", ex);
            Map<String, Object> grade = new LinkedHashMap<String, Object>();
            student.put(Constants.ATTR_SCORE_RESULTS, grade.put(Constants.ATTR_FINAL_LETTER_GRADE, "?"));
        } catch (NullPointerException ex) {
            log.error("Error occured processing Final Grades", ex);
            Map<String, Object> grade = new LinkedHashMap<String, Object>();
            student.put(Constants.ATTR_SCORE_RESULTS, grade.put(Constants.ATTR_FINAL_LETTER_GRADE, "?"));
        }
    }

    /**
     * Returns the term and the year as a string for a given student Section association.
{'fraction_non_alphanumeric': 0.09377455424599578, 'fraction_numerical': 0.0012088244182532487, 'mean_word_length': 2.162970751290384, 'pattern_counts': {'":': 0, '<': 145, '<?xml version=': 0, '>': 149, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5328598', 'n_tokens_mistral': 9180, 'n_tokens_neox': 8714, 'n_words': 2079}