<commit_before>package org.slc.sli.ingestion;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;

import org.junit.Test;
import org.junit.runner.RunWith;
import org.slc.sli.domain.School;
import org.slc.sli.ingestion.landingzone.IngestionFileEntry;
import org.slc.sli.ingestion.processors.EdFiProcessor;
import org.slc.sli.ingestion.processors.PersistenceProcessor;
import org.slc.sli.ingestion.util.MD5;
import org.slc.sli.repository.SchoolRepository;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.repository.PagingAndSortingRepository;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.TestExecutionListeners;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
import org.springframework.test.context.support.DependencyInjectionTestExecutionListener;
import org.springframework.test.context.support.DirtiesContextTestExecutionListener;
import org.xml.sax.SAXException;

<<<<<<< HEAD
import org.slc.sli.dal.repository.EntityRepository;
import org.slc.sli.domain.Entity;
import org.slc.sli.domain.MongoEntity;
import org.slc.sli.domain.School;
import org.slc.sli.ingestion.landingzone.IngestionFileEntry;
import org.slc.sli.ingestion.processors.EdFiProcessor;
import org.slc.sli.ingestion.processors.PersistenceProcessor;
import org.slc.sli.ingestion.util.MD5;
import org.slc.sli.repository.SchoolRepository;

=======
>>>>>>> 6861cffb23e19f0f9b177c2a715143990c2569bf
@RunWith(SpringJUnit4ClassRunner.class)
@ContextConfiguration(locations = { "/spring/applicationContext-test.xml" })
@TestExecutionListeners({ DependencyInjectionTestExecutionListener.class, DirtiesContextTestExecutionListener.class })
public class SchoolIngestionTest {

    @Autowired
    private EdFiProcessor edFiProcessor;

    @Autowired
    private PersistenceProcessor persistenceProcessor;

    // @Autowired
    // private SchoolRepository schoolRepository;

    @Autowired
    private EntityRepository repository;

    private static String schoolEntityType = "school";

    @Test
    public void testSchoolIngestionPersistence() throws IOException, SAXException {

        repository.deleteAll(schoolEntityType);

        int numberOfSchools = 2;
        List<NeutralRecord> neutralRecords = createSchoolIngestionNeutralRecords(numberOfSchools);

        File neutralRecordsFile = IngestionTest.createNeutralRecordsFile(neutralRecords);

        File ingestionPersistenceProcessorOutputFile = IngestionTest.createTempFile();

        persistenceProcessor.processIngestionStream(neutralRecordsFile, ingestionPersistenceProcessorOutputFile);

        verifySchools(repository, numberOfSchools);

    }

    @Test
    public void testSchoolInterchangeXmlParsing() throws IOException, SAXException {

        repository.deleteAll(schoolEntityType);

        int numberOfSchools = 2;
        String xmlRecords = createSchoolInterchangeXml(numberOfSchools);

        File inputFile = IngestionTest.createTestFile(xmlRecords);

        // Create Ingestion File Entry
        IngestionFileEntry inputFileEntry = new IngestionFileEntry(FileFormat.CSV, FileType.CSV_SCHOOL,
                inputFile.getName(), MD5.calculate(inputFile));
        inputFileEntry.setFile(inputFile);
<<<<<<< HEAD

        File ingestionEdFiProcessorOutputFile = IngestionTest.createTempFile();

        edFiProcessor.processIngestionStream(inputFileEntry, ingestionEdFiProcessorOutputFile);

        File ingestionPersistenceProcessorOutputFile = IngestionTest.createTempFile();

        persistenceProcessor.processIngestionStream(ingestionEdFiProcessorOutputFile,
=======
        
        edFiProcessor.processFileEntry(inputFileEntry);
        
        File ingestionPersistenceProcessorOutputFile = IngestionTest.createTempFile();
        
        persistenceProcessor.processIngestionStream(inputFileEntry.getNeutralRecordFile(),
>>>>>>> 6861cffb23e19f0f9b177c2a715143990c2569bf
                ingestionPersistenceProcessorOutputFile);

        verifySchools(repository, 0);

    }

    @Test
    public void testSchoolInterchangeCsvParsing() throws IOException, SAXException {

        repository.deleteAll(schoolEntityType);

        int numberOfSchools = 2;
        String csvRecords = createSchoolInterchangeCsv(numberOfSchools);

        File inputFile = IngestionTest.createTestFile(IngestionTest.INGESTION_FILE_PREFIX,
                IngestionTest.INGESTION_CSV_FILE_SUFFIX, csvRecords);

        // Create Ingestion File Entry
        IngestionFileEntry inputFileEntry = new IngestionFileEntry(FileFormat.CSV, FileType.CSV_SCHOOL,
                inputFile.getName(), MD5.calculate(inputFile));
        inputFileEntry.setFile(inputFile);
<<<<<<< HEAD

        File ingestionEdFiProcessorOutputFile = IngestionTest.createTempFile();

        edFiProcessor.processIngestionStream(inputFileEntry, ingestionEdFiProcessorOutputFile);

        File ingestionPersistenceProcessorOutputFile = IngestionTest.createTempFile();

        persistenceProcessor.processIngestionStream(ingestionEdFiProcessorOutputFile,
=======
        
        edFiProcessor.processFileEntry(inputFileEntry);
        
        File ingestionPersistenceProcessorOutputFile = IngestionTest.createTempFile();
        
        persistenceProcessor.processIngestionStream(inputFileEntry.getNeutralRecordFile(),
>>>>>>> 6861cffb23e19f0f9b177c2a715143990c2569bf
                ingestionPersistenceProcessorOutputFile);

        verifySchools(repository, 0);

    }

    @Test
    public void testSchoolInterchangeCsvFileParsing() throws IOException, SAXException {

        repository.deleteAll(schoolEntityType);

        File inputFile = IngestionTest.getFile("smooks/InterchangeSchool.csv");

        // Create Ingestion File Entry
        IngestionFileEntry inputFileEntry = new IngestionFileEntry(FileFormat.CSV, FileType.CSV_SCHOOL,
                inputFile.getName(), MD5.calculate(inputFile));
        inputFileEntry.setFile(inputFile);
<<<<<<< HEAD

        File ingestionEdFiProcessorOutputFile = IngestionTest.createTempFile();

        edFiProcessor.processIngestionStream(inputFileEntry, ingestionEdFiProcessorOutputFile);

        File ingestionPersistenceProcessorOutputFile = IngestionTest.createTempFile();

        persistenceProcessor.processIngestionStream(ingestionEdFiProcessorOutputFile,
=======
        
        edFiProcessor.processFileEntry(inputFileEntry);
        
        File ingestionPersistenceProcessorOutputFile = IngestionTest.createTempFile();
        
        persistenceProcessor.processIngestionStream(inputFileEntry.getNeutralRecordFile(),
>>>>>>> 6861cffb23e19f0f9b177c2a715143990c2569bf
                ingestionPersistenceProcessorOutputFile);

        assertEquals(2, IngestionTest.getTotalCountOfEntityInRepository(repository, schoolEntityType));
<commit_msg>add school test code into the commit<commit_after>package org.slc.sli.ingestion;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;

import org.junit.Test;
import org.junit.runner.RunWith;
import org.slc.sli.ingestion.landingzone.IngestionFileEntry;
import org.slc.sli.ingestion.processors.EdFiProcessor;
import org.slc.sli.ingestion.processors.PersistenceProcessor;
import org.slc.sli.ingestion.util.MD5;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.TestExecutionListeners;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
import org.springframework.test.context.support.DependencyInjectionTestExecutionListener;
import org.springframework.test.context.support.DirtiesContextTestExecutionListener;
import org.xml.sax.SAXException;

import org.slc.sli.dal.repository.EntityRepository;
import org.slc.sli.domain.Entity;
import org.slc.sli.domain.MongoEntity;

@RunWith(SpringJUnit4ClassRunner.class)
@ContextConfiguration(locations = { "/spring/applicationContext-test.xml" })
@TestExecutionListeners({ DependencyInjectionTestExecutionListener.class, DirtiesContextTestExecutionListener.class })
public class SchoolIngestionTest {

    @Autowired
    private EdFiProcessor edFiProcessor;

    @Autowired
    private PersistenceProcessor persistenceProcessor;

    // @Autowired
    // private SchoolRepository schoolRepository;

    @Autowired
    private EntityRepository repository;

    private static String schoolEntityType = "school";

    @Test
    public void testSchoolIngestionPersistence() throws IOException, SAXException {

        repository.deleteAll(schoolEntityType);

        int numberOfSchools = 2;
        List<NeutralRecord> neutralRecords = createSchoolIngestionNeutralRecords(numberOfSchools);

        File neutralRecordsFile = IngestionTest.createNeutralRecordsFile(neutralRecords);

        File ingestionPersistenceProcessorOutputFile = IngestionTest.createTempFile();

        persistenceProcessor.processIngestionStream(neutralRecordsFile, ingestionPersistenceProcessorOutputFile);

        verifySchools(repository, numberOfSchools);

    }

    @Test
    public void testSchoolInterchangeXmlParsing() throws IOException, SAXException {

        repository.deleteAll(schoolEntityType);

        int numberOfSchools = 2;
        String xmlRecords = createSchoolInterchangeXml(numberOfSchools);

        File inputFile = IngestionTest.createTestFile(xmlRecords);

        // Create Ingestion File Entry
        IngestionFileEntry inputFileEntry = new IngestionFileEntry(FileFormat.CSV, FileType.CSV_SCHOOL,
                inputFile.getName(), MD5.calculate(inputFile));
        inputFileEntry.setFile(inputFile);

        edFiProcessor.processFileEntry(inputFileEntry);

        File ingestionPersistenceProcessorOutputFile = IngestionTest.createTempFile();

        persistenceProcessor.processIngestionStream(inputFileEntry.getNeutralRecordFile(),
                ingestionPersistenceProcessorOutputFile);

        verifySchools(repository, 0);

    }

    @Test
    public void testSchoolInterchangeCsvParsing() throws IOException, SAXException {

        repository.deleteAll(schoolEntityType);

        int numberOfSchools = 2;
        String csvRecords = createSchoolInterchangeCsv(numberOfSchools);

        File inputFile = IngestionTest.createTestFile(IngestionTest.INGESTION_FILE_PREFIX,
                IngestionTest.INGESTION_CSV_FILE_SUFFIX, csvRecords);

        // Create Ingestion File Entry
        IngestionFileEntry inputFileEntry = new IngestionFileEntry(FileFormat.CSV, FileType.CSV_SCHOOL,
                inputFile.getName(), MD5.calculate(inputFile));
        inputFileEntry.setFile(inputFile);

        edFiProcessor.processFileEntry(inputFileEntry);

        File ingestionPersistenceProcessorOutputFile = IngestionTest.createTempFile();

        persistenceProcessor.processIngestionStream(inputFileEntry.getNeutralRecordFile(),
                ingestionPersistenceProcessorOutputFile);

        verifySchools(repository, 0);

    }

    @Test
    public void testSchoolInterchangeCsvFileParsing() throws IOException, SAXException {

        repository.deleteAll(schoolEntityType);

        File inputFile = IngestionTest.getFile("smooks/InterchangeSchool.csv");

        // Create Ingestion File Entry
        IngestionFileEntry inputFileEntry = new IngestionFileEntry(FileFormat.CSV, FileType.CSV_SCHOOL,
                inputFile.getName(), MD5.calculate(inputFile));
        inputFileEntry.setFile(inputFile);

        edFiProcessor.processFileEntry(inputFileEntry);

        File ingestionPersistenceProcessorOutputFile = IngestionTest.createTempFile();

        persistenceProcessor.processIngestionStream(inputFileEntry.getNeutralRecordFile(),
                ingestionPersistenceProcessorOutputFile);

        assertEquals(2, IngestionTest.getTotalCountOfEntityInRepository(repository, schoolEntityType));

{'fraction_non_alphanumeric': 0.07910569105691057, 'fraction_numerical': 0.01097560975609756, 'mean_word_length': 4.596451319381256, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2406858', 'n_tokens_mistral': 3568, 'n_tokens_neox': 3299, 'n_words': 491}