<commit_before>package org.commcare.cases.util.test;

import org.commcare.cases.model.Case;
import org.javarosa.core.services.storage.IStorageIterator;
import org.javarosa.core.services.storage.IStorageUtilityIndexed;
import org.javarosa.core.services.storage.util.DummyIndexedStorageUtility;
import org.javarosa.core.util.DataUtil;
import org.junit.Assert;

import org.commcare.cases.util.CasePurgeFilter;
import org.commcare.core.parse.ParseUtils;
import org.commcare.core.sandbox.SandboxUtils;
import org.commcare.util.mocks.MockDataUtils;
import org.commcare.util.mocks.MockUserDataSandbox;
import org.javarosa.core.util.DAG;
import org.junit.Test;

import java.util.Arrays;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Hashtable;
import java.util.Set;
import java.util.Vector;

import static org.junit.Assert.fail;

/**
 * Quick test to be able to restore a set of user data
 * and ensure users and groups are properly being included
 * in case purges.
 *
 * @author ctsims
 */
public class CasePurgeRegressions {

    @Test
    public void testSimpleExtensions() throws Exception {
        MockUserDataSandbox sandbox;
        Vector<String> owners;
        sandbox = MockDataUtils.getStaticStorage();

        ParseUtils.parseIntoSandbox(this.getClass().getClassLoader().
                getResourceAsStream("case_purge/simple_extension_test.xml"), sandbox);
        owners = SandboxUtils.extractEntityOwners(sandbox);

        CasePurgeFilter purger = new CasePurgeFilter(sandbox.getCaseStorage(), owners);
        int removedCases = sandbox.getCaseStorage().removeAll(purger).size();

        if (removedCases > 0) {
            throw new RuntimeException("Incorrectly removed cases");
        }
    }

    /**
     * Test correct validation of a graph where 1 case indexes a non-existent node
     */
    @Test
    public void testValidateCaseGraphBeforePurge_simple() throws Exception {
        MockUserDataSandbox sandbox = MockDataUtils.getStaticStorage();
        ParseUtils.parseIntoSandbox(this.getClass().getClassLoader().
                getResourceAsStream("case_purge/validate_case_graph_test_simple.xml"), sandbox);
        IStorageUtilityIndexed<Case> storage = sandbox.getCaseStorage();

        HashMap<String, Integer> caseIdsToRecordIds = createCaseIdsMap(storage);
        CasePurgeFilter filter = new CasePurgeFilter(storage);

        Set<String> nodesExpectedToBeLeft = new HashSet<>();
        nodesExpectedToBeLeft.add("case_one");
        nodesExpectedToBeLeft.add("case_two");

        Set<String[]> edgesExpectedToBeLeft = new HashSet<>();
        edgesExpectedToBeLeft.add(new String[]{"case_two", "case_one"});

        // Check that the edges and nodes still present in the graph are as expected
        DAG<String, int[], String> internalCaseGraph = filter.getInternalCaseGraph();
        checkProperNodesPresent(nodesExpectedToBeLeft, internalCaseGraph);
        checkProperEdgesPresent(edgesExpectedToBeLeft, internalCaseGraph);

        // Check that the correct cases were actually purged
        Vector<Integer> expectedToRemove = new Vector<>();
        expectedToRemove.add(caseIdsToRecordIds.get("case_three"));
        Vector<Integer> removed = storage.removeAll(filter);
        checkProperCasesRemoved(expectedToRemove, removed);
    }

    /**
     * Test correct validation of a graph where 2 different cases index the same non-existent node,
     * and both of those cases have child nodes
     */
    @Test
    public void testValidateCaseGraphBeforePurge_complex() throws Exception {
        MockUserDataSandbox sandbox = MockDataUtils.getStaticStorage();
        ParseUtils.parseIntoSandbox(this.getClass().getClassLoader().
                getResourceAsStream("case_purge/validate_case_graph_test_complex.xml"), sandbox);
        IStorageUtilityIndexed<Case> storage = sandbox.getCaseStorage();

        HashMap<String, Integer> caseIdsToRecordIds = createCaseIdsMap(storage);
        CasePurgeFilter filter = new CasePurgeFilter(storage);

        Set<String> nodesExpectedToBeLeft = new HashSet<>();
        nodesExpectedToBeLeft.add("case_one");
        nodesExpectedToBeLeft.add("case_two");

        Set<String[]> edgesExpectedToBeLeft = new HashSet<>();
        edgesExpectedToBeLeft.add(new String[]{"case_two", "case_one"});

        // Check that the edges and nodes still present in the graph are as expected
        DAG<String, int[], String> internalCaseGraph = filter.getInternalCaseGraph();
        checkProperNodesPresent(nodesExpectedToBeLeft, internalCaseGraph);
        //checkProperEdgesPresent(edgesExpectedToBeLeft, internalCaseGraph);

        // Check that the correct cases were actually purged
        Vector<Integer> expectedToRemove = new Vector<>();
        expectedToRemove.add(caseIdsToRecordIds.get("case_three"));
        expectedToRemove.add(caseIdsToRecordIds.get("case_four"));
        expectedToRemove.add(caseIdsToRecordIds.get("case_five"));
        expectedToRemove.add(caseIdsToRecordIds.get("case_six"));
        expectedToRemove.add(caseIdsToRecordIds.get("case_seven"));
        Vector<Integer> removed = storage.removeAll(filter);
        //checkProperCasesRemoved(expectedToRemove, removed);
    }

    /**
     * Test correct validation of a graph where 1 case indexes 2 other cases - 1 valid and 1 that
     * does not exist
     */
    @Test
    public void testValidateCaseGraphBeforePurge_multipleParents() throws Exception {
        MockUserDataSandbox sandbox = MockDataUtils.getStaticStorage();
        ParseUtils.parseIntoSandbox(this.getClass().getClassLoader().
                getResourceAsStream("case_purge/validate_case_graph_test_multiple_parents.xml"),
                sandbox);
        IStorageUtilityIndexed<Case> storage = sandbox.getCaseStorage();

        HashMap<String, Integer> caseIdsToRecordIds = createCaseIdsMap(storage);
        CasePurgeFilter filter = new CasePurgeFilter(storage);

        Set<String> nodesExpectedToBeLeft = new HashSet<>();
        nodesExpectedToBeLeft.add("case_two");
        nodesExpectedToBeLeft.add("case_three");

        Set<String[]> edgesExpectedToBeLeft = new HashSet<>();
        edgesExpectedToBeLeft.add(new String[]{"case_three", "case_two"});

        // Check that the edges and nodes still present in the graph are as expected
        DAG<String, int[], String> internalCaseGraph = filter.getInternalCaseGraph();
        checkProperNodesPresent(nodesExpectedToBeLeft, internalCaseGraph);
        checkProperEdgesPresent(edgesExpectedToBeLeft, internalCaseGraph);

        // Check that the correct cases were actually purged<commit_msg>uncomment lines<commit_after>package org.commcare.cases.util.test;

import org.commcare.cases.model.Case;
import org.javarosa.core.services.storage.IStorageIterator;
import org.javarosa.core.services.storage.IStorageUtilityIndexed;
import org.junit.Assert;

import org.commcare.cases.util.CasePurgeFilter;
import org.commcare.core.parse.ParseUtils;
import org.commcare.core.sandbox.SandboxUtils;
import org.commcare.util.mocks.MockDataUtils;
import org.commcare.util.mocks.MockUserDataSandbox;
import org.javarosa.core.util.DAG;
import org.junit.Test;

import java.util.Arrays;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Hashtable;
import java.util.Set;
import java.util.Vector;

/**
 * Quick test to be able to restore a set of user data
 * and ensure users and groups are properly being included
 * in case purges.
 *
 * @author ctsims
 */
public class CasePurgeRegressions {

    @Test
    public void testSimpleExtensions() throws Exception {
        MockUserDataSandbox sandbox;
        Vector<String> owners;
        sandbox = MockDataUtils.getStaticStorage();

        ParseUtils.parseIntoSandbox(this.getClass().getClassLoader().
                getResourceAsStream("case_purge/simple_extension_test.xml"), sandbox);
        owners = SandboxUtils.extractEntityOwners(sandbox);

        CasePurgeFilter purger = new CasePurgeFilter(sandbox.getCaseStorage(), owners);
        int removedCases = sandbox.getCaseStorage().removeAll(purger).size();

        if (removedCases > 0) {
            throw new RuntimeException("Incorrectly removed cases");
        }
    }

    /**
     * Test correct validation of a graph where 1 case indexes a non-existent node
     */
    @Test
    public void testValidateCaseGraphBeforePurge_simple() throws Exception {
        MockUserDataSandbox sandbox = MockDataUtils.getStaticStorage();
        ParseUtils.parseIntoSandbox(this.getClass().getClassLoader().
                getResourceAsStream("case_purge/validate_case_graph_test_simple.xml"), sandbox);
        IStorageUtilityIndexed<Case> storage = sandbox.getCaseStorage();

        HashMap<String, Integer> caseIdsToRecordIds = createCaseIdsMap(storage);
        CasePurgeFilter filter = new CasePurgeFilter(storage);

        Set<String> nodesExpectedToBeLeft = new HashSet<>();
        nodesExpectedToBeLeft.add("case_one");
        nodesExpectedToBeLeft.add("case_two");

        Set<String[]> edgesExpectedToBeLeft = new HashSet<>();
        edgesExpectedToBeLeft.add(new String[]{"case_two", "case_one"});

        // Check that the edges and nodes still present in the graph are as expected
        DAG<String, int[], String> internalCaseGraph = filter.getInternalCaseGraph();
        checkProperNodesPresent(nodesExpectedToBeLeft, internalCaseGraph);
        checkProperEdgesPresent(edgesExpectedToBeLeft, internalCaseGraph);

        // Check that the correct cases were actually purged
        Vector<Integer> expectedToRemove = new Vector<>();
        expectedToRemove.add(caseIdsToRecordIds.get("case_three"));
        Vector<Integer> removed = storage.removeAll(filter);
        checkProperCasesRemoved(expectedToRemove, removed);
    }

    /**
     * Test correct validation of a graph where 2 different cases index the same non-existent node,
     * and both of those cases have child nodes
     */
    @Test
    public void testValidateCaseGraphBeforePurge_complex() throws Exception {
        MockUserDataSandbox sandbox = MockDataUtils.getStaticStorage();
        ParseUtils.parseIntoSandbox(this.getClass().getClassLoader().
                getResourceAsStream("case_purge/validate_case_graph_test_complex.xml"), sandbox);
        IStorageUtilityIndexed<Case> storage = sandbox.getCaseStorage();

        HashMap<String, Integer> caseIdsToRecordIds = createCaseIdsMap(storage);
        CasePurgeFilter filter = new CasePurgeFilter(storage);

        Set<String> nodesExpectedToBeLeft = new HashSet<>();
        nodesExpectedToBeLeft.add("case_one");
        nodesExpectedToBeLeft.add("case_two");

        Set<String[]> edgesExpectedToBeLeft = new HashSet<>();
        edgesExpectedToBeLeft.add(new String[]{"case_two", "case_one"});

        // Check that the edges and nodes still present in the graph are as expected
        DAG<String, int[], String> internalCaseGraph = filter.getInternalCaseGraph();
        checkProperNodesPresent(nodesExpectedToBeLeft, internalCaseGraph);
        checkProperEdgesPresent(edgesExpectedToBeLeft, internalCaseGraph);

        // Check that the correct cases were actually purged
        Vector<Integer> expectedToRemove = new Vector<>();
        expectedToRemove.add(caseIdsToRecordIds.get("case_three"));
        expectedToRemove.add(caseIdsToRecordIds.get("case_four"));
        expectedToRemove.add(caseIdsToRecordIds.get("case_five"));
        expectedToRemove.add(caseIdsToRecordIds.get("case_six"));
        expectedToRemove.add(caseIdsToRecordIds.get("case_seven"));
        Vector<Integer> removed = storage.removeAll(filter);
        checkProperCasesRemoved(expectedToRemove, removed);
    }

    /**
     * Test correct validation of a graph where 1 case indexes 2 other cases - 1 valid and 1 that
     * does not exist
     */
    @Test
    public void testValidateCaseGraphBeforePurge_multipleParents() throws Exception {
        MockUserDataSandbox sandbox = MockDataUtils.getStaticStorage();
        ParseUtils.parseIntoSandbox(this.getClass().getClassLoader().
                getResourceAsStream("case_purge/validate_case_graph_test_multiple_parents.xml"),
                sandbox);
        IStorageUtilityIndexed<Case> storage = sandbox.getCaseStorage();

        HashMap<String, Integer> caseIdsToRecordIds = createCaseIdsMap(storage);
        CasePurgeFilter filter = new CasePurgeFilter(storage);

        Set<String> nodesExpectedToBeLeft = new HashSet<>();
        nodesExpectedToBeLeft.add("case_two");
        nodesExpectedToBeLeft.add("case_three");

        Set<String[]> edgesExpectedToBeLeft = new HashSet<>();
        edgesExpectedToBeLeft.add(new String[]{"case_three", "case_two"});

        // Check that the edges and nodes still present in the graph are as expected
        DAG<String, int[], String> internalCaseGraph = filter.getInternalCaseGraph();
        checkProperNodesPresent(nodesExpectedToBeLeft, internalCaseGraph);
        checkProperEdgesPresent(edgesExpectedToBeLeft, internalCaseGraph);

        // Check that the correct cases were actually purged
{'fraction_non_alphanumeric': 0.09446427215430177, 'fraction_numerical': 0.0010631027412863544, 'mean_word_length': 3.865164388622091, 'pattern_counts': {'":': 0, '<': 59, '<?xml version=': 0, '>': 61, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3227901', 'n_tokens_mistral': 3639, 'n_tokens_neox': 3557, 'n_words': 771}