<commit_before>package org.slc.sli.sample.transform;

import java.io.File;
import java.io.IOException;
import java.io.PrintStream;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import javax.xml.bind.JAXBContext;
import javax.xml.bind.JAXBException;
import javax.xml.bind.Marshaller;
import javax.xml.bind.PropertyException;

import org.slc.sli.sample.entities.AcademicSubjectType;
import org.slc.sli.sample.entities.ComplexObjectType;
import org.slc.sli.sample.entities.ContentStandardType;
import org.slc.sli.sample.entities.GradeLevelType;
import org.slc.sli.sample.entities.InterchangeAssessmentMetadata;
import org.slc.sli.sample.entities.LearningStandard;
import org.slc.sli.sample.entities.LearningStandardId;

/**
 * @author dliu
 * 
 *         Transform the Common Core Standard Math csv to edfi xml
 */
public class CCSMathCSV2XMLTransformer {
    // read CCS csv file
    private CcsCsvReader ccsReader;
    
    // read identifier csv file
    private CSVReader identifiersReader;
    
    // map between dotNotation and Guids
    private Map<String, String> identifiersMap = new HashMap<String, String>();

    // input csv files
    private static final String ccsCSVFile = "data/CC_Standards_6.25.10-Math.csv";
    private static final String identifiersCSVFile = "data/E0330_ccss_identifiers.csv";

    // output Ed-Fi xml file
    private static final String interchangeCCSFile = "data/InterchangeAssessmentMetadata.xml";
    private static final String outputPath = "data/";
    
    private String getCopyright() {
        if (ccsReader != null) {
            return ccsReader.getCopyright();
        }
        return null;
    }

    /**
     * main method
     * 
     * @param args
     * @throws Exception
     */
    public static void main(String[] args) throws Exception {
        CCSMathCSV2XMLTransformer transformer = new CCSMathCSV2XMLTransformer();
        transformer.loadData();
        
        PrintStream ps = new PrintStream(new File(interchangeCCSFile));
        transformer.printInterchangeCCS(ps);
        
        SchemaValidator.check(outputPath);
        System.out.println(transformer.getCopyright());
    }
    
    /**
     * open csv files and create CSV reader for each file
     * and load the first record for each reader
     * 
     * @throws IOException
     */
    private void loadData() throws Exception {
        // load CCS data
        ccsReader = new CcsCsvReader(ccsCSVFile);
        
        // load identifier data for mapping between dotNotation and Guid
        identifiersReader = new CSVReader(identifiersCSVFile);
        while (identifiersReader.getCurrentRecord() != null) {
            Map<String, String> currentRecord = identifiersReader.getCurrentRecord();
            identifiersMap.put(currentRecord.get("Dot notation").replaceAll("\\.", "").replaceAll("-", ""),
                    currentRecord.get("GUID"));
            identifiersReader.getNextRecord();
        }

    }
    
    /**
     * Iterate through common core standard csv records in the CSV files,
     * converts them into JAXB java objects, and then marshals them into SLI-EdFi xml file.
     * 
     * @param ps
     * @throws JAXBException
     */
    private void printInterchangeCCS(PrintStream ps) throws JAXBException, IOException {
        int learningStandardCounter = 0;
        
        Marshaller marshaller = getMarshaller();
        InterchangeAssessmentMetadata interchangeCCS = new InterchangeAssessmentMetadata();
        
        List<ComplexObjectType> learningStandards = interchangeCCS
                .getAssessmentFamilyOrAssessmentOrAssessmentPeriodDescriptor();

        // process student
        while (ccsReader.getCurrentRecord() != null) {
            learningStandards.add(this.getLearningStandard());
            ccsReader.getNextRecord();
            learningStandardCounter++;
        }
        
        marshaller.marshal(interchangeCCS, ps);
        
        System.out.println("Total " + learningStandardCounter + " LearningStandards are exported.");
    }
    
    private Marshaller getMarshaller() throws JAXBException, PropertyException {
        JAXBContext context = JAXBContext.newInstance(LearningStandard.class);
        Marshaller marshaller = context.createMarshaller();
        marshaller.setProperty("jaxb.formatted.output", Boolean.TRUE);
        return marshaller;
    }
    
    private LearningStandard getLearningStandard() {
        
        Map<String, String> learningStandardRecord = ccsReader.getCurrentRecord();
        LearningStandard learningStandard = new LearningStandard();
        LearningStandardId learningStandardId = new LearningStandardId();
        String id = learningStandardRecord.get("ID");
        learningStandardId.setIdentificationCode(getLSIdentificationCode(id));
        learningStandard.setLearningStandardId(learningStandardId);
        learningStandard.setContentStandard(ContentStandardType.STATE_STANDARD);
        String description = learningStandardRecord.get("State Standard");
        if (description == null || description.equals("")) {
            System.out.println("no description for" + id);
        }
        learningStandard.setDescription(learningStandardRecord.get("State Standard"));
        learningStandard.setGradeLevel(getLSGradeLevel(id));<commit_msg>change learning standards na,e<commit_after>package org.slc.sli.sample.transform;

import java.io.File;
import java.io.IOException;
import java.io.PrintStream;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import javax.xml.bind.JAXBContext;
import javax.xml.bind.JAXBException;
import javax.xml.bind.Marshaller;
import javax.xml.bind.PropertyException;

import org.slc.sli.sample.entities.AcademicSubjectType;
import org.slc.sli.sample.entities.ComplexObjectType;
import org.slc.sli.sample.entities.ContentStandardType;
import org.slc.sli.sample.entities.GradeLevelType;
import org.slc.sli.sample.entities.InterchangeAssessmentMetadata;
import org.slc.sli.sample.entities.LearningStandard;
import org.slc.sli.sample.entities.LearningStandardId;

/**
 * @author dliu
 * 
 *         Transform the Common Core Standard Math csv to edfi xml
 */
@SuppressWarnings("restriction")
public class CCSMathCSV2XMLTransformer {
    // read CCS csv file
    private CcsCsvReader ccsReader;
    
    // read identifier csv file
    private CSVReader identifiersReader;
    
    // map between dotNotation and Guids
    private Map<String, String> identifiersMap = new HashMap<String, String>();

    // input csv files
    private static final String ccsCSVFile = "data/CC_Standards_6.25.10-Math.csv";
    private static final String identifiersCSVFile = "data/E0330_ccss_identifiers.csv";

    // output Ed-Fi xml file
    private static final String interchangeCCSFile = "data/InterchangeAssessmentMetadata.xml";
    private static final String outputPath = "data/";
    
    private String getCopyright() {
        if (ccsReader != null) {
            return ccsReader.getCopyright();
        }
        return null;
    }

    /**
     * main method
     * 
     * @param args
     * @throws Exception
     */
    public static void main(String[] args) throws Exception {
        CCSMathCSV2XMLTransformer transformer = new CCSMathCSV2XMLTransformer();
        transformer.loadData();
        
        PrintStream ps = new PrintStream(new File(interchangeCCSFile));
        transformer.printInterchangeCCS(ps);
        
        SchemaValidator.check(outputPath);
        System.out.println(transformer.getCopyright());
    }
    
    /**
     * open csv files and create CSV reader for each file
     * and load the first record for each reader
     * 
     * @throws IOException
     */
    private void loadData() throws Exception {
        // load CCS data
        ccsReader = new CcsCsvReader(ccsCSVFile);
        
        // load identifier data for mapping between dotNotation and Guid
        identifiersReader = new CSVReader(identifiersCSVFile);
        while (identifiersReader.getCurrentRecord() != null) {
            Map<String, String> currentRecord = identifiersReader.getCurrentRecord();
            identifiersMap.put(currentRecord.get("Dot notation").replaceAll("\\.", "").replaceAll("-", ""),
                    currentRecord.get("GUID"));
            identifiersReader.getNextRecord();
        }

    }
    
    /**
     * Iterate through common core standard csv records in the CSV files,
     * converts them into JAXB java objects, and then marshals them into SLI-EdFi xml file.
     * 
     * @param ps
     * @throws JAXBException
     */
    private void printInterchangeCCS(PrintStream ps) throws JAXBException, IOException {
        int learningStandardCounter = 0;
        
        Marshaller marshaller = getMarshaller();
        InterchangeAssessmentMetadata interchangeCCS = new InterchangeAssessmentMetadata();
        
        List<ComplexObjectType> ccsCollection = interchangeCCS
                .getAssessmentFamilyOrAssessmentOrAssessmentPeriodDescriptor();

        // process learningStandard
        while (ccsReader.getCurrentRecord() != null) {
            ccsCollection.add(this.getLearningStandard());
            ccsReader.getNextRecord();
            learningStandardCounter++;
        }
        
        marshaller.marshal(interchangeCCS, ps);
        
        System.out.println("Total " + learningStandardCounter + " LearningStandards are exported.");
    }
    
    private Marshaller getMarshaller() throws JAXBException, PropertyException {
        JAXBContext context = JAXBContext.newInstance(LearningStandard.class);
        Marshaller marshaller = context.createMarshaller();
        marshaller.setProperty("jaxb.formatted.output", Boolean.TRUE);
        return marshaller;
    }
    
    private LearningStandard getLearningStandard() {
        
        Map<String, String> learningStandardRecord = ccsReader.getCurrentRecord();
        LearningStandard learningStandard = new LearningStandard();
        LearningStandardId learningStandardId = new LearningStandardId();
        String id = learningStandardRecord.get("ID");
        learningStandardId.setIdentificationCode(getLSIdentificationCode(id));
        learningStandard.setLearningStandardId(learningStandardId);
        learningStandard.setContentStandard(ContentStandardType.STATE_STANDARD);
        String description = learningStandardRecord.get("State Standard");
        if (description == null || description.equals("")) {
            System.out.println("no description for" + id);
        }
        learningStandard.setDescription(learningStandardRecord.get("State Standard"));
        learningStandard.setGradeLevel(getLSGradeLevel(id));
{'fraction_non_alphanumeric': 0.0900046490004649, 'fraction_numerical': 0.002417480241748024, 'mean_word_length': 3.0696178584941354, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 16}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1893594', 'n_tokens_mistral': 2906, 'n_tokens_neox': 2844, 'n_words': 639}