<commit_before>/*
 * Autopsy Forensic Browser
 *
 * Copyright 2017-2018 Basis Technology Corp.
 * Contact: carrier <at> sleuthkit <dot> org
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.sleuthkit.autopsy.ingest;

import java.io.File;
import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import static junit.framework.Assert.assertFalse;
import org.netbeans.junit.NbModuleSuite;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.casemodule.CaseActionException;
import org.sleuthkit.autopsy.casemodule.CaseDetails;
import junit.framework.Test;
import org.apache.commons.io.FileUtils;
import org.netbeans.junit.NbTestCase;
import org.openide.util.Exceptions;
import org.python.icu.impl.Assert;
import org.sleuthkit.autopsy.casemodule.ImageDSProcessor;
import org.sleuthkit.autopsy.casemodule.LocalFilesDSProcessor;
import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException;
import org.sleuthkit.autopsy.casemodule.services.FileManager;
import org.sleuthkit.autopsy.datasourceprocessors.AutoIngestDataSourceProcessor;
import org.sleuthkit.autopsy.ingest.IngestJobSettings.IngestType;
import org.sleuthkit.autopsy.modules.embeddedfileextractor.EmbeddedFileExtractorModuleFactory;
import org.sleuthkit.autopsy.modules.filetypeid.FileTypeIdModuleFactory;
import org.sleuthkit.autopsy.modules.interestingitems.FilesSet;
import org.sleuthkit.autopsy.modules.interestingitems.FilesSet.Rule;
import org.sleuthkit.autopsy.modules.interestingitems.FilesSet.Rule.ExtensionCondition;
import org.sleuthkit.autopsy.modules.interestingitems.FilesSet.Rule.FullNameCondition;
import org.sleuthkit.autopsy.modules.interestingitems.FilesSet.Rule.MetaTypeCondition;
import org.sleuthkit.autopsy.modules.interestingitems.FilesSet.Rule.ParentPathCondition;
import org.sleuthkit.autopsy.modules.photoreccarver.PhotoRecCarverIngestModuleFactory;
import org.sleuthkit.autopsy.testutils.DataSourceProcessorRunner;
import org.sleuthkit.autopsy.testutils.DataSourceProcessorRunner.ProcessorCallback;
import org.sleuthkit.autopsy.testutils.IngestJobRunner;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.Content;
import org.sleuthkit.datamodel.TskCoreException;
import org.sleuthkit.datamodel.TskData;
import org.sleuthkit.datamodel.TskData.TSK_DB_FILES_TYPE_ENUM;

public class IngestFileFiltersTest extends NbTestCase {

    private final Path IMAGE_PATH = Paths.get(this.getDataDir().toString(),"filter_test1.img");
    private final Path ZIPFILE_PATH = Paths.get(this.getDataDir().toString(), "local_files_test.zip");
    
    public static Test suite() {
        NbModuleSuite.Configuration conf = NbModuleSuite.createConfiguration(IngestFileFiltersTest.class).
                clusters(".*").
                enableModules(".*");
        return conf.suite();
    }

    public IngestFileFiltersTest(String name) {
        super(name);
    }

    @Override
    public void tearDown() {
        try {
            Case.closeCurrentCase();
            //Seems like we need some time to close the case.
            try {
                Thread.sleep(2000);
            } catch (Exception ex) {

            }
            assertFalse(Case.isCaseOpen());
        } catch (CaseActionException ex) {
            Exceptions.printStackTrace(ex);
            Assert.fail(ex);
        }
    }
    
    public void testBasicDir() {
        Path casePath = Paths.get(System.getProperty("java.io.tmpdir"), "testBasicDir");
        createCase(casePath);
        ImageDSProcessor dataSourceProcessor = new ImageDSProcessor();
        addDataSourceToCase(dataSourceProcessor, IMAGE_PATH);

        HashMap<String, Rule> rule = new HashMap<>();
        rule.put("Rule", new Rule("testFileType", null, new MetaTypeCondition(MetaTypeCondition.Type.FILES), new ParentPathCondition("dir1"), null, null, null));
        //Filter for dir1 and no unallocated space
        FilesSet dirFilter = new FilesSet("Filter", "Filter to find all files in dir1.", false, true, rule);        

        try {
            Case openCase = Case.getOpenCase();
            ArrayList<IngestModuleTemplate> templates =  new ArrayList<>();
            templates.add(getIngestModuleTemplate(new FileTypeIdModuleFactory()));
            runIngestJob(openCase.getDataSources(), templates, dirFilter);
            FileManager fileManager = openCase.getServices().getFileManager();
            List<AbstractFile> results = fileManager.findFiles("file.jpg", "dir1");
            String mimeType = results.get(0).getMIMEType();
            assertEquals("image/jpeg", mimeType);
            
            results = fileManager.findFiles("%%");
          
            for (AbstractFile file : results) {
                //All files in dir1 should have MIME type, except '.' '..' and slack files
                if (file.getParentPath().equalsIgnoreCase("/dir1/")) {
                    if (!(file.getName().equals(".") || file.getName().equals("..") || file.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.SLACK)) {
                        String errMsg = String.format("File %s (objId=%d) unexpectedly passed by the file filter.", file.getName(), file.getId());
                        assertTrue(errMsg, file.getMIMEType() != null && !file.getMIMEType().isEmpty());
                    }
                } else { //All files not in dir1 shouldn't have MIME type
                    String errMsg = String.format("File %s (objId=%d) unexpectedly passed by the file filter.", file.getName(), file.getId());
                    assertTrue(errMsg, file.getMIMEType() == null);
                }
            }
        } catch (NoCurrentCaseException | TskCoreException ex) {
            Exceptions.printStackTrace(ex);
            Assert.fail(ex);
        }
        
    }
    
    public void testExtAndDirWithOneRule() {
        Path casePath = Paths.get(System.getProperty("java.io.tmpdir"), "testBasicDir");
        createCase(casePath);
        ImageDSProcessor dataSourceProcessor = new ImageDSProcessor();
        addDataSourceToCase(dataSourceProcessor, IMAGE_PATH);

        HashMap<String, Rule> rules = new HashMap<>();
        rules.put("Rule", new Rule("testExtAndDirWithOneRule", new ExtensionCondition("jpg"), new MetaTypeCondition(MetaTypeCondition.Type.FILES), new ParentPathCondition("dir1"), null, null, null));
        //Build the filter that ignore unallocated space and with one rule
        FilesSet filesExtDirsFilter = new FilesSet("Filter", "Filter to find all jpg files in dir1.", false, true, rules);
        
        try {
            Case openCase = Case.getOpenCase();
            ArrayList<IngestModuleTemplate> templates =  new ArrayList<>();
            templates.add(getIngestModuleTemplate(new FileTypeIdModuleFactory()));
            runIngestJob(openCase.getDataSources(), templates, filesExtDirsFilter); 
            FileManager fileManager = Case.getOpenCase().getServices().getFileManager();            
            List<AbstractFile> results = fileManager.findFiles("%%");
            assertEquals(62, results.size());
            for (AbstractFile file : results) {
                //Files with jpg extension in dir1 should have MIME Type
                if (file.getParentPath().equalsIgnoreCase("/dir1/") && file.getNameExtension().equalsIgnoreCase("jpg")) {
                    String errMsg = String.format("File %s (objId=%d) unexpectedly blocked by the file filter.", file.getName(), file.getId());
                    assertTrue(errMsg, file.getMIMEType() != null && !file.getMIMEType().isEmpty());
                } else { //All others shouldn't have MIME Type
                    String errMsg = String.format("File %s (objId=%d) unexpectedly passed by the file filter.", file.getName(), file.getId());
                    assertTrue(errMsg, file.getMIMEType() == null);
                }
            }
        } catch (NoCurrentCaseException | TskCoreException ex) {
            Exceptions.printStackTrace(ex);
            Assert.fail(ex);
        }
    }

    public void testExtAndDirWithTwoRules() {
        Path casePath = Paths.get(System.getProperty("java.io.tmpdir"), "testBasicDir");
        createCase(casePath);
        ImageDSProcessor dataSourceProcessor = new ImageDSProcessor();
        addDataSourceToCase(dataSourceProcessor, IMAGE_PATH);

        HashMap<String, Rule> rules = new HashMap<>();
        rules.put("rule1", new Rule("FindJpgExtention", new ExtensionCondition("jpg"), new MetaTypeCondition(MetaTypeCondition.Type.FILES), null, null, null, null));
        rules.put("rule2", new Rule("FindDir1Directory", null, new MetaTypeCondition(MetaTypeCondition.Type.FILES), new ParentPathCondition("dir1"), null, null, null));
        //Build the filter that ingnore unallocated space and with 2 rules
        FilesSet filesExtDirsFilter = new FilesSet("Filter", "Filter to find all files in dir1 and all files with jpg extention.", false, true, rules);        
          
        try {
            Case openCase = Case.getOpenCase();
            ArrayList<IngestModuleTemplate> templates =  new ArrayList<>();
            templates.add(getIngestModuleTemplate(new FileTypeIdModuleFactory()));
            runIngestJob(openCase.getDataSources(), templates, filesExtDirsFilter); 
            FileManager fileManager = Case.getOpenCase().getServices().getFileManager();           
            List<AbstractFile> results = fileManager.findFiles("%%");  
            assertEquals(62, results.size());
            for (AbstractFile file : results) {               
                if (file.getNameExtension().equalsIgnoreCase("jpg")) { //All files with .jpg extension should have MIME type
                    String errMsg = String.format("File %s (objId=%d) unexpectedly blocked by the file filter.", file.getName(), file.getId());
                    assertTrue(errMsg, file.getMIMEType() != null && !file.getMIMEType().isEmpty()); 
                } else if (file.getParentPath().equalsIgnoreCase("/dir1/")) { 
                    //All files in dir1 should have MIME type except '.' '..' slack files
                    if (file.getName().equals(".") || file.getName().equals("..") || file.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.SLACK) {
                        String errMsg = String.format("File %s (objId=%d) unexpectedly passed by the file filter.", file.getName(), file.getId());
                        assertTrue(errMsg, file.getMIMEType() == null);
                    } else {
                        String errMsg = String.format("File %s (objId=%d) unexpectedly blocked by the file filter.", file.getName(), file.getId());
                        assertTrue(errMsg, file.getMIMEType() != null && !file.getMIMEType().isEmpty());
                    }
                } else { //All files that are not in dir1 or not with .jpg extension should not have MIME type
                    String errMsg = String.format("File %s (objId=%d) unexpectedly passed by the file filter.", file.getName(), file.getId());
                    assertTrue(errMsg, file.getMIMEType() == null);                     
                }
            }
         } catch (NoCurrentCaseException | TskCoreException ex) {
            Exceptions.printStackTrace(ex);
            Assert.fail(ex);
        }
   }
   
    public void testFullFileNameRule() {
        Path casePath = Paths.get(System.getProperty("java.io.tmpdir"), "testFullFileNameRule");
        createCase(casePath);
        ImageDSProcessor dataSourceProcessor = new ImageDSProcessor();
        addDataSourceToCase(dataSourceProcessor, IMAGE_PATH);

        HashMap<String, Rule> rules = new HashMap<>();
        rules.put("rule", new Rule("FindFileWithFullName", new FullNameCondition("file.docx"), new MetaTypeCondition(MetaTypeCondition.Type.FILES), null, null, null, null));
        //Build the filter to find file: file.docx
        FilesSet fullNameFilter = new FilesSet("Filter", "Filter to find file.docx.", false, true, rules);
                 
        try {
            Case openCase = Case.getOpenCase();
            ArrayList<IngestModuleTemplate> templates =  new ArrayList<>();
            templates.add(getIngestModuleTemplate(new FileTypeIdModuleFactory()));

            runIngestJob(openCase.getDataSources(), templates, fullNameFilter); 
            FileManager fileManager = Case.getOpenCase().getServices().getFileManager();           
            List<AbstractFile> results = fileManager.findFiles("%%");
            assertEquals(62, results.size());
            for (AbstractFile file : results) {
                //Only file.docx has MIME Type
                if (file.getName().equalsIgnoreCase("file.docx")) {
                    String errMsg = String.format("File %s (objId=%d) unexpectedly blocked by the file filter.", file.getName(), file.getId());
                    assertTrue(errMsg, file.getMIMEType() != null && !file.getMIMEType().isEmpty());
                } else {
                    String errMsg = String.format("File %s (objId=%d) unexpectedly passed by the file filter.", file.getName(), file.getId());
                    assertTrue(errMsg, file.getMIMEType() == null);
                }
            }
        } catch (NoCurrentCaseException | TskCoreException ex) {
            Exceptions.printStackTrace(ex);
            Assert.fail(ex);
        }
    }

    public void testCarvingWithExtRuleAndUnallocSpace() {
        Path casePath = Paths.get(System.getProperty("java.io.tmpdir"), "testCarvingWithExtRuleAndUnallocSpace");
        createCase(casePath);
        ImageDSProcessor dataSourceProcessor = new ImageDSProcessor();
        addDataSourceToCase(dataSourceProcessor, IMAGE_PATH);

        HashMap<String, Rule> rules = new HashMap<>();
        rules.put("rule1", new Rule("FindJpgExtention", new ExtensionCondition("jpg"), new MetaTypeCondition(MetaTypeCondition.Type.FILES), null, null, null, null));
        rules.put("rule2", new Rule("FindGifExtention", new ExtensionCondition("gif"), new MetaTypeCondition(MetaTypeCondition.Type.FILES), null, null, null, null));
        
        //Build the filter to find files with .jpg and .gif extension and unallocated space
        FilesSet extensionFilter = new FilesSet("Filter", "Filter to files with .jpg and .gif extension.", false, false, rules);
                  
        try {
            Case openCase = Case.getOpenCase();
            ArrayList<IngestModuleTemplate> templates =  new ArrayList<>();
            templates.add(getIngestModuleTemplate(new FileTypeIdModuleFactory()));
            templates.add(getIngestModuleTemplate(new PhotoRecCarverIngestModuleFactory()));
            runIngestJob(openCase.getDataSources(), templates, extensionFilter); 
            FileManager fileManager = Case.getOpenCase().getServices().getFileManager();
            List<AbstractFile> results = fileManager.findFiles("%%");
            assertEquals(70, results.size()); 
            int carvedJpgGifFiles = 0;
            for (AbstractFile file : results) {
                if (file.getNameExtension().equalsIgnoreCase("jpg") || file.getNameExtension().equalsIgnoreCase("gif")) { //Unalloc file and .jpg files in dir1, dir2, $CarvedFiles, root directory should have MIME type
                    String errMsg = String.format("File %s (objId=%d) unexpectedly blocked by the file filter.", file.getName(), file.getId());
                    assertTrue(errMsg, file.getMIMEType() != null && !file.getMIMEType().isEmpty());
                    
                    if (file.getParentPath().equalsIgnoreCase("/$CarvedFiles/")) {
                        carvedJpgGifFiles++;
                    }
                } else if (file.getName().startsWith("Unalloc_")) {
                    String errMsg = String.format("File %s (objId=%d) unexpectedly blocked by the file filter.", file.getName(), file.getId());
                    assertTrue(errMsg, file.getMIMEType() != null && !file.getMIMEType().isEmpty());
                } else { //All other files should not have MIME type. 
                    String errMsg = String.format("File %s (objId=%d) unexpectedly passed by the file filter.", file.getName(), file.getId());
                    assertTrue(errMsg, file.getMIMEType() == null);
                }
            }
            //Make sure we have carved jpg/gif files
            assertEquals(2, carvedJpgGifFiles);

        } catch (NoCurrentCaseException | TskCoreException ex) {
            Exceptions.printStackTrace(ex);
            Assert.fail(ex);
        }
    }
  
    public void testCarvingNoUnallocatedSpace() {
        Path casePath = Paths.get(System.getProperty("java.io.tmpdir"), "testCarvingNoUnallocatedSpace");
        createCase(casePath);
        ImageDSProcessor dataSourceProcessor = new ImageDSProcessor();
        addDataSourceToCase(dataSourceProcessor, IMAGE_PATH);

        HashMap<String, Rule> rules = new HashMap<>();
        rules.put("rule1", new Rule("FindJpgExtention", new ExtensionCondition("jpg"), new MetaTypeCondition(MetaTypeCondition.Type.FILES), null, null, null, null));
        rules.put("rule2", new Rule("FindGifExtention", new ExtensionCondition("gif"), new MetaTypeCondition(MetaTypeCondition.Type.FILES), null, null, null, null));
        
        //Build the filter to find files with .jpg and .gif extension
        FilesSet extensionFilter = new FilesSet("Filter", "Filter to files with .jpg and .gif extension.", false, true, rules);        
          
        try {
            Case openCase = Case.getOpenCase();
            ArrayList<IngestModuleTemplate> templates =  new ArrayList<>();
            templates.add(getIngestModuleTemplate(new FileTypeIdModuleFactory()));
            templates.add(getIngestModuleTemplate(new PhotoRecCarverIngestModuleFactory()));
            IngestJobSettings ingestJobSettings = new IngestJobSettings(IngestFileFiltersTest.class.getCanonicalName(), IngestType.FILES_ONLY, templates, extensionFilter);
            try {
                List<IngestModuleError> errs = IngestJobRunner.runIngestJob(openCase.getDataSources(), ingestJobSettings);
                //Ingest fails because Carving wants unallocated space
                assertEquals(1, errs.size());
                assertEquals("PhotoRec Carver", errs.get(0).getModuleDisplayName());
            } catch (InterruptedException ex) {
                Exceptions.printStackTrace(ex);
                Assert.fail(ex);
            }        
        } catch (NoCurrentCaseException | TskCoreException ex) {
            Exceptions.printStackTrace(ex);
            Assert.fail(ex);
        }
    }

    public void testEmbeddedModule() {
        Path casePath = Paths.get(System.getProperty("java.io.tmpdir"), "testEmbeddedModule");
        createCase(casePath);
        LocalFilesDSProcessor dataSourceProcessor = new LocalFilesDSProcessor();
        addDataSourceToCase(dataSourceProcessor, ZIPFILE_PATH);
        
        //Build the filter to find jpg files
        HashMap<String, Rule> rules = new HashMap<>();
        //Extension condition for jpg files
        rules.put("rule1", new Rule("FindJpgExtention", new ExtensionCondition("jpg"), new MetaTypeCondition(MetaTypeCondition.Type.FILES), null, null, null, null));
        //Extension condition for zip files, because we want test jpg extension filter for extracted files from a zip file 
        rules.put("rule2", new Rule("ZipExtention", new ExtensionCondition("zip"), new MetaTypeCondition(MetaTypeCondition.Type.FILES), null, null, null, null));
        FilesSet embeddedFilter = new FilesSet("Filter", "Filter to files with .jpg extension.", false, false, rules);
                  
        try {
            Case openCase = Case.getOpenCase();
            ArrayList<IngestModuleTemplate> templates =  new ArrayList<>();
            templates.add(getIngestModuleTemplate(new FileTypeIdModuleFactory()));
            templates.add(getIngestModuleTemplate(new EmbeddedFileExtractorModuleFactory()));
            runIngestJob(openCase.getDataSources(), templates, embeddedFilter);
            FileManager fileManager = Case.getOpenCase().getServices().getFileManager();
            //get all .jpg files in zip file
            List<AbstractFile> results = fileManager.findFiles("%%");
            assertEquals(39, results.size());
            int numTypeJpgFiles = 0;
            for (AbstractFile file : results) {
                if (file.getNameExtension().equalsIgnoreCase("jpg") || file.getNameExtension().equalsIgnoreCase("zip")) {
                    String errMsg = String.format("File %s (objId=%d) unexpectedly blocked by the file filter.", file.getName(), file.getId());
                    assertTrue(errMsg, file.getMIMEType() != null && !file.getMIMEType().isEmpty());
                    numTypeJpgFiles++;
                } else if (file.isDir() && (file.getType() == TSK_DB_FILES_TYPE_ENUM.DERIVED || file.getType() == TSK_DB_FILES_TYPE_ENUM.LOCAL)) {
                    String errMsg = String.format("File %s (objId=%d) unexpectedly blocked by the file filter.", file.getName(), file.getId());
                    assertTrue(errMsg, file.getMIMEType() != null && !file.getMIMEType().isEmpty());
                } else {
                    String errMsg = String.format("File %s (objId=%d) unexpectedly passed by the file filter.", file.getName(), file.getId());
                    assertTrue(errMsg, file.getMIMEType() == null);
                }
            }
            //Make sure 10 jpg files and 1 zip file have been typed
            assertEquals(11, numTypeJpgFiles);
        } catch (NoCurrentCaseException | TskCoreException ex) {
            Exceptions.printStackTrace(ex);
            Assert.fail(ex);
        }
    }

    private void runIngestJob(List<Content> datasources, ArrayList<IngestModuleTemplate> templates, FilesSet filter) {
        IngestJobSettings ingestJobSettings = new IngestJobSettings(IngestFileFiltersTest.class.getCanonicalName(), IngestType.FILES_ONLY, templates, filter);
        try {
            List<IngestModuleError> errs = IngestJobRunner.runIngestJob(datasources, ingestJobSettings);
            for (IngestModuleError err : errs) {
                System.out.println(String.format("Error: %s: %s.", err.getModuleDisplayName(), err.toString()));
            }
            assertEquals(0, errs.size());
        } catch (InterruptedException ex) {
            Exceptions.printStackTrace(ex);
            Assert.fail(ex);
        }        
    }
        
    private IngestModuleTemplate getIngestModuleTemplate(IngestModuleFactoryAdapter factory) {
        IngestModuleIngestJobSettings settings = factory.getDefaultIngestJobSettings();
        IngestModuleTemplate template = new IngestModuleTemplate(factory, settings);
        template.setEnabled(true);
        return template;
    }

    private void createCase(Path casePath) {
        // Delete the test directory, if it exists
        if (casePath.toFile().exists()) {
            try {
                FileUtils.deleteDirectory(casePath.toFile());
            } catch (IOException ex) {
                Exceptions.printStackTrace(ex);
                Assert.fail(ex);
            }
        }
        assertFalse("Unable to delete existing test directory", casePath.toFile().exists());
 
        // Create the test directory
        casePath.toFile().mkdirs();
        assertTrue("Unable to create test directory", casePath.toFile().exists());

        try {
            Case.createAsCurrentCase(Case.CaseType.SINGLE_USER_CASE, casePath.toString(), new CaseDetails("IngestFiltersTest"));
        } catch (CaseActionException ex) {
            Exceptions.printStackTrace(ex);
            Assert.fail(ex);
        }        
        assertTrue(casePath.toFile().exists());
    }
        
    private void addDataSourceToCase(AutoIngestDataSourceProcessor dataSourceProcessor, Path dataSourcePath) {
        try {
            ProcessorCallback callBack = DataSourceProcessorRunner.runDataSourceProcessor(dataSourceProcessor, dataSourcePath);
            List<Content> dataSourceContent = callBack.getDataSourceContent();
            assertEquals(1, dataSourceContent.size());
            List<String> errorMessages = callBack.getErrorMessages();
            assertEquals(0, errorMessages.size());
        } catch (AutoIngestDataSourceProcessor.AutoIngestDataSourceProcessorException | InterruptedException ex) {
            Exceptions.printStackTrace(ex);
            Assert.fail(ex);
            
        }        
    }
}
<commit_msg>3599: Enable number 7 test case.<commit_after>/*
 * Autopsy Forensic Browser
 *
 * Copyright 2017-2018 Basis Technology Corp.
 * Contact: carrier <at> sleuthkit <dot> org
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.sleuthkit.autopsy.ingest;

import java.io.File;
import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import static junit.framework.Assert.assertFalse;
import org.netbeans.junit.NbModuleSuite;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.casemodule.CaseActionException;
import org.sleuthkit.autopsy.casemodule.CaseDetails;
import junit.framework.Test;
import org.apache.commons.io.FileUtils;
import org.netbeans.junit.NbTestCase;
import org.openide.util.Exceptions;
import org.python.icu.impl.Assert;
import org.sleuthkit.autopsy.casemodule.ImageDSProcessor;
import org.sleuthkit.autopsy.casemodule.LocalFilesDSProcessor;
import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException;
import org.sleuthkit.autopsy.casemodule.services.FileManager;
import org.sleuthkit.autopsy.datasourceprocessors.AutoIngestDataSourceProcessor;
import org.sleuthkit.autopsy.ingest.IngestJobSettings.IngestType;
import org.sleuthkit.autopsy.modules.embeddedfileextractor.EmbeddedFileExtractorModuleFactory;
import org.sleuthkit.autopsy.modules.filetypeid.FileTypeIdModuleFactory;
import org.sleuthkit.autopsy.modules.interestingitems.FilesSet;
import org.sleuthkit.autopsy.modules.interestingitems.FilesSet.Rule;
import org.sleuthkit.autopsy.modules.interestingitems.FilesSet.Rule.ExtensionCondition;
import org.sleuthkit.autopsy.modules.interestingitems.FilesSet.Rule.FullNameCondition;
import org.sleuthkit.autopsy.modules.interestingitems.FilesSet.Rule.MetaTypeCondition;
import org.sleuthkit.autopsy.modules.interestingitems.FilesSet.Rule.ParentPathCondition;
import org.sleuthkit.autopsy.modules.photoreccarver.PhotoRecCarverIngestModuleFactory;
import org.sleuthkit.autopsy.testutils.DataSourceProcessorRunner;
import org.sleuthkit.autopsy.testutils.DataSourceProcessorRunner.ProcessorCallback;
import org.sleuthkit.autopsy.testutils.IngestJobRunner;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.Content;
import org.sleuthkit.datamodel.TskCoreException;
import org.sleuthkit.datamodel.TskData;
import org.sleuthkit.datamodel.TskData.TSK_DB_FILES_TYPE_ENUM;

public class IngestFileFiltersTest extends NbTestCase {

    private final Path IMAGE_PATH = Paths.get(this.getDataDir().toString(),"filter_test1.img");
    private final Path ZIPFILE_PATH = Paths.get(this.getDataDir().toString(), "local_files_test.zip");
    
    public static Test suite() {
        NbModuleSuite.Configuration conf = NbModuleSuite.createConfiguration(IngestFileFiltersTest.class).
                clusters(".*").
                enableModules(".*");
        return conf.suite();
    }

    public IngestFileFiltersTest(String name) {
        super(name);
    }

    @Override
    public void tearDown() {
        try {
            Case.closeCurrentCase();
            //Seems like we need some time to close the case.
            try {
                Thread.sleep(2000);
            } catch (Exception ex) {

            }
            assertFalse(Case.isCaseOpen());
        } catch (CaseActionException ex) {
            Exceptions.printStackTrace(ex);
            Assert.fail(ex);
        }
    }
    
    public void testBasicDir() {
        Path casePath = Paths.get(System.getProperty("java.io.tmpdir"), "testBasicDir");
        createCase(casePath);
        ImageDSProcessor dataSourceProcessor = new ImageDSProcessor();
        addDataSourceToCase(dataSourceProcessor, IMAGE_PATH);

        HashMap<String, Rule> rule = new HashMap<>();
        rule.put("Rule", new Rule("testFileType", null, new MetaTypeCondition(MetaTypeCondition.Type.FILES), new ParentPathCondition("dir1"), null, null, null));
        //Filter for dir1 and no unallocated space
        FilesSet dirFilter = new FilesSet("Filter", "Filter to find all files in dir1.", false, true, rule);        

        try {
            Case openCase = Case.getOpenCase();
            ArrayList<IngestModuleTemplate> templates =  new ArrayList<>();
            templates.add(getIngestModuleTemplate(new FileTypeIdModuleFactory()));
            runIngestJob(openCase.getDataSources(), templates, dirFilter);
            FileManager fileManager = openCase.getServices().getFileManager();
            List<AbstractFile> results = fileManager.findFiles("file.jpg", "dir1");
            String mimeType = results.get(0).getMIMEType();
            assertEquals("image/jpeg", mimeType);
            
            results = fileManager.findFiles("%%");
          
            for (AbstractFile file : results) {
                //All files in dir1 should have MIME type, except '.' '..' and slack files
                if (file.getParentPath().equalsIgnoreCase("/dir1/")) {
                    if (!(file.getName().equals(".") || file.getName().equals("..") || file.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.SLACK)) {
                        String errMsg = String.format("File %s (objId=%d) unexpectedly passed by the file filter.", file.getName(), file.getId());
                        assertTrue(errMsg, file.getMIMEType() != null && !file.getMIMEType().isEmpty());
                    }
                } else { //All files not in dir1 shouldn't have MIME type
                    String errMsg = String.format("File %s (objId=%d) unexpectedly passed by the file filter.", file.getName(), file.getId());
                    assertTrue(errMsg, file.getMIMEType() == null);
                }
            }
        } catch (NoCurrentCaseException | TskCoreException ex) {
            Exceptions.printStackTrace(ex);
            Assert.fail(ex);
        }
        
    }
    
    public void testExtAndDirWithOneRule() {
        Path casePath = Paths.get(System.getProperty("java.io.tmpdir"), "testExtAndDirWithOneRule");
        createCase(casePath);
        ImageDSProcessor dataSourceProcessor = new ImageDSProcessor();
        addDataSourceToCase(dataSourceProcessor, IMAGE_PATH);

        HashMap<String, Rule> rules = new HashMap<>();
        rules.put("Rule", new Rule("testExtAndDirWithOneRule", new ExtensionCondition("jpg"), new MetaTypeCondition(MetaTypeCondition.Type.FILES), new ParentPathCondition("dir1"), null, null, null));
        //Build the filter that ignore unallocated space and with one rule
        FilesSet filesExtDirsFilter = new FilesSet("Filter", "Filter to find all jpg files in dir1.", false, true, rules);
        
        try {
            Case openCase = Case.getOpenCase();
            ArrayList<IngestModuleTemplate> templates =  new ArrayList<>();
            templates.add(getIngestModuleTemplate(new FileTypeIdModuleFactory()));
            runIngestJob(openCase.getDataSources(), templates, filesExtDirsFilter); 
            FileManager fileManager = Case.getOpenCase().getServices().getFileManager();            
            List<AbstractFile> results = fileManager.findFiles("%%");
            assertEquals(62, results.size());
            for (AbstractFile file : results) {
                //Files with jpg extension in dir1 should have MIME Type
                if (file.getParentPath().equalsIgnoreCase("/dir1/") && file.getNameExtension().equalsIgnoreCase("jpg")) {
                    String errMsg = String.format("File %s (objId=%d) unexpectedly blocked by the file filter.", file.getName(), file.getId());
                    assertTrue(errMsg, file.getMIMEType() != null && !file.getMIMEType().isEmpty());
                } else { //All others shouldn't have MIME Type
                    String errMsg = String.format("File %s (objId=%d) unexpectedly passed by the file filter.", file.getName(), file.getId());
                    assertTrue(errMsg, file.getMIMEType() == null);
                }
            }
        } catch (NoCurrentCaseException | TskCoreException ex) {
            Exceptions.printStackTrace(ex);
            Assert.fail(ex);
        }
    }

    public void testExtAndDirWithTwoRules() {
        Path casePath = Paths.get(System.getProperty("java.io.tmpdir"), "testExtAndDirWithTwoRules");
        createCase(casePath);
        ImageDSProcessor dataSourceProcessor = new ImageDSProcessor();
        addDataSourceToCase(dataSourceProcessor, IMAGE_PATH);

        HashMap<String, Rule> rules = new HashMap<>();
        rules.put("rule1", new Rule("FindJpgExtention", new ExtensionCondition("jpg"), new MetaTypeCondition(MetaTypeCondition.Type.FILES), null, null, null, null));
        rules.put("rule2", new Rule("FindDir1Directory", null, new MetaTypeCondition(MetaTypeCondition.Type.FILES), new ParentPathCondition("dir1"), null, null, null));
        //Build the filter that ingnore unallocated space and with 2 rules
        FilesSet filesExtDirsFilter = new FilesSet("Filter", "Filter to find all files in dir1 and all files with jpg extention.", false, true, rules);        
          
        try {
            Case openCase = Case.getOpenCase();
            ArrayList<IngestModuleTemplate> templates =  new ArrayList<>();
            templates.add(getIngestModuleTemplate(new FileTypeIdModuleFactory()));
            runIngestJob(openCase.getDataSources(), templates, filesExtDirsFilter); 
            FileManager fileManager = Case.getOpenCase().getServices().getFileManager();           
            List<AbstractFile> results = fileManager.findFiles("%%");  
            assertEquals(62, results.size());
            for (AbstractFile file : results) {               
                if (file.getNameExtension().equalsIgnoreCase("jpg")) { //All files with .jpg extension should have MIME type
                    String errMsg = String.format("File %s (objId=%d) unexpectedly blocked by the file filter.", file.getName(), file.getId());
                    assertTrue(errMsg, file.getMIMEType() != null && !file.getMIMEType().isEmpty()); 
                } else if (file.getParentPath().equalsIgnoreCase("/dir1/")) { 
                    //All files in dir1 should have MIME type except '.' '..' slack files
                    if (file.getName().equals(".") || file.getName().equals("..") || file.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.SLACK) {
                        String errMsg = String.format("File %s (objId=%d) unexpectedly passed by the file filter.", file.getName(), file.getId());
                        assertTrue(errMsg, file.getMIMEType() == null);
                    } else {
                        String errMsg = String.format("File %s (objId=%d) unexpectedly blocked by the file filter.", file.getName(), file.getId());
                        assertTrue(errMsg, file.getMIMEType() != null && !file.getMIMEType().isEmpty());
                    }
                } else { //All files that are not in dir1 or not with .jpg extension should not have MIME type
                    String errMsg = String.format("File %s (objId=%d) unexpectedly passed by the file filter.", file.getName(), file.getId());
                    assertTrue(errMsg, file.getMIMEType() == null);                     
                }
            }
         } catch (NoCurrentCaseException | TskCoreException ex) {
            Exceptions.printStackTrace(ex);
            Assert.fail(ex);
        }
   }
   
    public void testFullFileNameRule() {
        Path casePath = Paths.get(System.getProperty("java.io.tmpdir"), "testFullFileNameRule");
        createCase(casePath);
        ImageDSProcessor dataSourceProcessor = new ImageDSProcessor();
        addDataSourceToCase(dataSourceProcessor, IMAGE_PATH);

        HashMap<String, Rule> rules = new HashMap<>();
        rules.put("rule", new Rule("FindFileWithFullName", new FullNameCondition("file.docx"), new MetaTypeCondition(MetaTypeCondition.Type.FILES), null, null, null, null));
        //Build the filter to find file: file.docx
        FilesSet fullNameFilter = new FilesSet("Filter", "Filter to find file.docx.", false, true, rules);
                 
        try {
            Case openCase = Case.getOpenCase();
            ArrayList<IngestModuleTemplate> templates =  new ArrayList<>();
            templates.add(getIngestModuleTemplate(new FileTypeIdModuleFactory()));

            runIngestJob(openCase.getDataSources(), templates, fullNameFilter); 
            FileManager fileManager = Case.getOpenCase().getServices().getFileManager();           
            List<AbstractFile> results = fileManager.findFiles("%%");
            assertEquals(62, results.size());
            for (AbstractFile file : results) {
                //Only file.docx has MIME Type
                if (file.getName().equalsIgnoreCase("file.docx")) {
                    String errMsg = String.format("File %s (objId=%d) unexpectedly blocked by the file filter.", file.getName(), file.getId());
                    assertTrue(errMsg, file.getMIMEType() != null && !file.getMIMEType().isEmpty());
                } else {
                    String errMsg = String.format("File %s (objId=%d) unexpectedly passed by the file filter.", file.getName(), file.getId());
                    assertTrue(errMsg, file.getMIMEType() == null);
                }
            }
        } catch (NoCurrentCaseException | TskCoreException ex) {
            Exceptions.printStackTrace(ex);
            Assert.fail(ex);
        }
    }

    public void testCarvingWithExtRuleAndUnallocSpace() {
        Path casePath = Paths.get(System.getProperty("java.io.tmpdir"), "testCarvingWithExtRuleAndUnallocSpace");
        createCase(casePath);
        ImageDSProcessor dataSourceProcessor = new ImageDSProcessor();
        addDataSourceToCase(dataSourceProcessor, IMAGE_PATH);

        HashMap<String, Rule> rules = new HashMap<>();
        rules.put("rule1", new Rule("FindJpgExtention", new ExtensionCondition("jpg"), new MetaTypeCondition(MetaTypeCondition.Type.FILES), null, null, null, null));
        rules.put("rule2", new Rule("FindGifExtention", new ExtensionCondition("gif"), new MetaTypeCondition(MetaTypeCondition.Type.FILES), null, null, null, null));
        
        //Build the filter to find files with .jpg and .gif extension and unallocated space
        FilesSet extensionFilter = new FilesSet("Filter", "Filter to files with .jpg and .gif extension.", false, false, rules);
                  
        try {
            Case openCase = Case.getOpenCase();
            ArrayList<IngestModuleTemplate> templates =  new ArrayList<>();
            templates.add(getIngestModuleTemplate(new FileTypeIdModuleFactory()));
            templates.add(getIngestModuleTemplate(new PhotoRecCarverIngestModuleFactory()));
            runIngestJob(openCase.getDataSources(), templates, extensionFilter); 
            FileManager fileManager = Case.getOpenCase().getServices().getFileManager();
            List<AbstractFile> results = fileManager.findFiles("%%");
            assertEquals(70, results.size()); 
            int carvedJpgGifFiles = 0;
            for (AbstractFile file : results) {
                if (file.getNameExtension().equalsIgnoreCase("jpg") || file.getNameExtension().equalsIgnoreCase("gif")) { //Unalloc file and .jpg files in dir1, dir2, $CarvedFiles, root directory should have MIME type
                    String errMsg = String.format("File %s (objId=%d) unexpectedly blocked by the file filter.", file.getName(), file.getId());
                    assertTrue(errMsg, file.getMIMEType() != null && !file.getMIMEType().isEmpty());
                    
                    if (file.getParentPath().equalsIgnoreCase("/$CarvedFiles/")) {
                        carvedJpgGifFiles++;
                    }
                } else if (file.getName().startsWith("Unalloc_")) {
                    String errMsg = String.format("File %s (objId=%d) unexpectedly blocked by the file filter.", file.getName(), file.getId());
                    assertTrue(errMsg, file.getMIMEType() != null && !file.getMIMEType().isEmpty());
                } else { //All other files should not have MIME type. 
                    String errMsg = String.format("File %s (objId=%d) unexpectedly passed by the file filter.", file.getName(), file.getId());
                    assertTrue(errMsg, file.getMIMEType() == null);
                }
            }
            //Make sure we have carved jpg/gif files
            assertEquals(2, carvedJpgGifFiles);

        } catch (NoCurrentCaseException | TskCoreException ex) {
            Exceptions.printStackTrace(ex);
            Assert.fail(ex);
        }
    }
  
    public void testCarvingNoUnallocatedSpace() {
        Path casePath = Paths.get(System.getProperty("java.io.tmpdir"), "testCarvingNoUnallocatedSpace");
        createCase(casePath);
        ImageDSProcessor dataSourceProcessor = new ImageDSProcessor();
        addDataSourceToCase(dataSourceProcessor, IMAGE_PATH);

        HashMap<String, Rule> rules = new HashMap<>();
        rules.put("rule1", new Rule("FindJpgExtention", new ExtensionCondition("jpg"), new MetaTypeCondition(MetaTypeCondition.Type.FILES), null, null, null, null));
        rules.put("rule2", new Rule("FindGifExtention", new ExtensionCondition("gif"), new MetaTypeCondition(MetaTypeCondition.Type.FILES), null, null, null, null));
        
        //Build the filter to find files with .jpg and .gif extension
        FilesSet extensionFilter = new FilesSet("Filter", "Filter to files with .jpg and .gif extension.", false, true, rules);        
          
        try {
            Case openCase = Case.getOpenCase();
            ArrayList<IngestModuleTemplate> templates =  new ArrayList<>();
            templates.add(getIngestModuleTemplate(new FileTypeIdModuleFactory()));
            templates.add(getIngestModuleTemplate(new PhotoRecCarverIngestModuleFactory()));
            IngestJobSettings ingestJobSettings = new IngestJobSettings(IngestFileFiltersTest.class.getCanonicalName(), IngestType.FILES_ONLY, templates, extensionFilter);
            try {
                List<IngestModuleError> errs = IngestJobRunner.runIngestJob(openCase.getDataSources(), ingestJobSettings);
                //Ingest fails because Carving wants unallocated space
                assertEquals(1, errs.size());
                assertEquals("PhotoRec Carver", errs.get(0).getModuleDisplayName());
            } catch (InterruptedException ex) {
                Exceptions.printStackTrace(ex);
                Assert.fail(ex);
            }        
        } catch (NoCurrentCaseException | TskCoreException ex) {
            Exceptions.printStackTrace(ex);
            Assert.fail(ex);
        }
    }

    public void testEmbeddedModule() {
        Path casePath = Paths.get(System.getProperty("java.io.tmpdir"), "testEmbeddedModule");
        createCase(casePath);
        LocalFilesDSProcessor dataSourceProcessor = new LocalFilesDSProcessor();
        addDataSourceToCase(dataSourceProcessor, ZIPFILE_PATH);
        
        //Build the filter to find jpg files
        HashMap<String, Rule> rules = new HashMap<>();
        //Extension condition for jpg files
        rules.put("rule1", new Rule("FindJpgExtention", new ExtensionCondition("jpg"), new MetaTypeCondition(MetaTypeCondition.Type.FILES), null, null, null, null));
        //Extension condition for zip files, because we want test jpg extension filter for extracted files from a zip file 
        rules.put("rule2", new Rule("ZipExtention", new ExtensionCondition("zip"), new MetaTypeCondition(MetaTypeCondition.Type.FILES), null, null, null, null));
        FilesSet embeddedFilter = new FilesSet("Filter", "Filter to files with .jpg extension.", false, false, rules);
                  
        try {
            Case openCase = Case.getOpenCase();
            ArrayList<IngestModuleTemplate> templates =  new ArrayList<>();
            templates.add(getIngestModuleTemplate(new FileTypeIdModuleFactory()));
            templates.add(getIngestModuleTemplate(new EmbeddedFileExtractorModuleFactory()));
            runIngestJob(openCase.getDataSources(), templates, embeddedFilter);
            FileManager fileManager = Case.getOpenCase().getServices().getFileManager();
            //get all .jpg files in zip file
            List<AbstractFile> results = fileManager.findFiles("%%");
            assertEquals(39, results.size());
            int numTypeJpgFiles = 0;
            for (AbstractFile file : results) {
                if (file.getNameExtension().equalsIgnoreCase("jpg") || file.getNameExtension().equalsIgnoreCase("zip")) {
                    String errMsg = String.format("File %s (objId=%d) unexpectedly blocked by the file filter.", file.getName(), file.getId());
                    assertTrue(errMsg, file.getMIMEType() != null && !file.getMIMEType().isEmpty());
                    numTypeJpgFiles++;
                } else if (file.isDir() && (file.getType() == TSK_DB_FILES_TYPE_ENUM.DERIVED || file.getType() == TSK_DB_FILES_TYPE_ENUM.LOCAL)) {
                    String errMsg = String.format("File %s (objId=%d) unexpectedly blocked by the file filter.", file.getName(), file.getId());
                    assertTrue(errMsg, file.getMIMEType() != null && !file.getMIMEType().isEmpty());
                } else {
                    String errMsg = String.format("File %s (objId=%d) unexpectedly passed by the file filter.", file.getName(), file.getId());
                    assertTrue(errMsg, file.getMIMEType() == null);
                }
            }
            //Make sure 10 jpg files and 1 zip file have been typed
            assertEquals(11, numTypeJpgFiles);
        } catch (NoCurrentCaseException | TskCoreException ex) {
            Exceptions.printStackTrace(ex);
            Assert.fail(ex);
        }
    }

    private void runIngestJob(List<Content> datasources, ArrayList<IngestModuleTemplate> templates, FilesSet filter) {
        IngestJobSettings ingestJobSettings = new IngestJobSettings(IngestFileFiltersTest.class.getCanonicalName(), IngestType.FILES_ONLY, templates, filter);
        try {
            List<IngestModuleError> errs = IngestJobRunner.runIngestJob(datasources, ingestJobSettings);
            for (IngestModuleError err : errs) {
                System.out.println(String.format("Error: %s: %s.", err.getModuleDisplayName(), err.toString()));
            }
            assertEquals(0, errs.size());
        } catch (InterruptedException ex) {
            Exceptions.printStackTrace(ex);
            Assert.fail(ex);
        }        
    }
        
    private IngestModuleTemplate getIngestModuleTemplate(IngestModuleFactoryAdapter factory) {
        IngestModuleIngestJobSettings settings = factory.getDefaultIngestJobSettings();
        IngestModuleTemplate template = new IngestModuleTemplate(factory, settings);
        template.setEnabled(true);
        return template;
    }

    private void createCase(Path casePath) {
        // Delete the test directory, if it exists
        if (casePath.toFile().exists()) {
            try {
                FileUtils.deleteDirectory(casePath.toFile());
            } catch (IOException ex) {
                Exceptions.printStackTrace(ex);
                Assert.fail(ex);
            }
        }
        assertFalse("Unable to delete existing test directory", casePath.toFile().exists());
 
        // Create the test directory
        casePath.toFile().mkdirs();
        assertTrue("Unable to create test directory", casePath.toFile().exists());

        try {
            Case.createAsCurrentCase(Case.CaseType.SINGLE_USER_CASE, casePath.toString(), new CaseDetails("IngestFiltersTest"));
        } catch (CaseActionException ex) {
            Exceptions.printStackTrace(ex);
            Assert.fail(ex);
        }        
        assertTrue(casePath.toFile().exists());
    }
        
    private void addDataSourceToCase(AutoIngestDataSourceProcessor dataSourceProcessor, Path dataSourcePath) {
        try {
            ProcessorCallback callBack = DataSourceProcessorRunner.runDataSourceProcessor(dataSourceProcessor, dataSourcePath);
            List<Content> dataSourceContent = callBack.getDataSourceContent();
            assertEquals(1, dataSourceContent.size());
            List<String> errorMessages = callBack.getErrorMessages();
            assertEquals(0, errorMessages.size());
        } catch (AutoIngestDataSourceProcessor.AutoIngestDataSourceProcessorException | InterruptedException ex) {
            Exceptions.printStackTrace(ex);
            Assert.fail(ex);
            
        }        
    }
}

{'fraction_non_alphanumeric': 0.11137204951637941, 'fraction_numerical': 0.0028459976913585163, 'mean_word_length': 2.7584710898346922, 'pattern_counts': {'":': 0, '<': 87, '<?xml version=': 0, '>': 87, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1796977', 'n_tokens_mistral': 13177, 'n_tokens_neox': 12927, 'n_words': 3017}