<commit_before>package edu.harvard.iq.dataverse.ingest.tabulardata.impl.plugins.dta;

import edu.harvard.iq.dataverse.datavariable.DataVariable;
import edu.harvard.iq.dataverse.datavariable.VariableCategory;
import edu.harvard.iq.dataverse.ingest.tabulardata.TabularDataIngest;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.util.List;
import org.junit.Test;
import static org.junit.Assert.assertEquals;
import org.junit.Ignore;

public class DTA117FileReaderTest {

    DTA117FileReader instance = new DTA117FileReader(null);
    File nullDataFile = null;

    @Ignore
    @Test
    public void testBen() throws IOException {
        TabularDataIngest result = instance.read(new BufferedInputStream(new FileInputStream(new File("/tmp/15aa6802ee5-5d2ed1bf55a5.dta"))), nullDataFile);
        assertEquals("application/x-stata", result.getDataTable().getOriginalFileFormat());
        assertEquals("STATA 13", result.getDataTable().getOriginalFormatVersion());
        assertEquals(25, result.getDataTable().getDataVariables().size());
        // TODO: add more assertions on the result.
    }

    @Test
    public void testAuto() throws IOException {
        // From https://www.stata-press.com/data/r13/auto.dta
        // `strings` shows "<stata_dta><header><release>117</release>"
        TabularDataIngest result = instance.read(new BufferedInputStream(new FileInputStream(new File("scripts/search/data/tabular/stata13-auto.dta"))), nullDataFile);
        assertEquals("application/x-stata", result.getDataTable().getOriginalFileFormat());
        assertEquals("STATA 13", result.getDataTable().getOriginalFormatVersion());
        assertEquals(12, result.getDataTable().getDataVariables().size());
        DataVariable foreign = result.getDataTable().getDataVariables().get(11);
        assertEquals(2, foreign.getCategories().size());
        List<VariableCategory> origins = (List) foreign.getCategories();
        assertEquals("Domestic", origins.get(0).getLabel());
        assertEquals("Foreign", origins.get(1).getLabel());<commit_msg>better name for characteristics test, added TODO #2301<commit_after>package edu.harvard.iq.dataverse.ingest.tabulardata.impl.plugins.dta;

import edu.harvard.iq.dataverse.datavariable.DataVariable;
import edu.harvard.iq.dataverse.datavariable.VariableCategory;
import edu.harvard.iq.dataverse.ingest.tabulardata.TabularDataIngest;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.util.List;
import org.junit.Test;
import static org.junit.Assert.assertEquals;
import org.junit.Ignore;

public class DTA117FileReaderTest {

    DTA117FileReader instance = new DTA117FileReader(null);
    File nullDataFile = null;

    // TODO: Can we create a small file to check into the code base that exercises the characteristics issue?
    @Ignore
    @Test
    public void testCharacteristics() throws IOException {
        TabularDataIngest result = instance.read(new BufferedInputStream(new FileInputStream(new File("/tmp/15aa6802ee5-5d2ed1bf55a5.dta"))), nullDataFile);
        assertEquals("application/x-stata", result.getDataTable().getOriginalFileFormat());
        assertEquals("STATA 13", result.getDataTable().getOriginalFormatVersion());
        assertEquals(441, result.getDataTable().getDataVariables().size());
    }

    @Test
    public void testAuto() throws IOException {
        // From https://www.stata-press.com/data/r13/auto.dta
        // `strings` shows "<stata_dta><header><release>117</release>"
        TabularDataIngest result = instance.read(new BufferedInputStream(new FileInputStream(new File("scripts/search/data/tabular/stata13-auto.dta"))), nullDataFile);
        assertEquals("application/x-stata", result.getDataTable().getOriginalFileFormat());
        assertEquals("STATA 13", result.getDataTable().getOriginalFormatVersion());
        assertEquals(12, result.getDataTable().getDataVariables().size());
        DataVariable foreign = result.getDataTable().getDataVariables().get(11);
        assertEquals(2, foreign.getCategories().size());
        List<VariableCategory> origins = (List) foreign.getCategories();
        assertEquals("Domestic", origins.get(0).getLabel());
        assertEquals("Foreign", origins.get(1).getLabel());
{'fraction_non_alphanumeric': 0.12585812356979406, 'fraction_numerical': 0.020366132723112128, 'mean_word_length': 4.706266318537859, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '145845', 'n_tokens_mistral': 1264, 'n_tokens_neox': 1227, 'n_words': 219}