<commit_before>/**
 * (c) 2014 StreamSets, Inc. All rights reserved. May not
 * be copied, modified, or distributed in whole or part without
 * written consent of StreamSets, Inc.
 */
package com.streamsets.pipeline.snapshotstore.impl;

import com.google.common.collect.ImmutableList;
import com.streamsets.pipeline.main.RuntimeInfo;
import com.streamsets.pipeline.api.Field;
import com.streamsets.pipeline.api.Record;
import com.streamsets.pipeline.record.RecordImpl;
import com.streamsets.pipeline.runner.ErrorSink;
import com.streamsets.pipeline.runner.StageOutput;
import com.streamsets.pipeline.snapshotstore.SnapshotInfo;
import com.streamsets.pipeline.snapshotstore.SnapshotStatus;
import org.apache.commons.io.FileUtils;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.mockito.Mockito;

import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

public class TestFileSnapshotStore {

  private static final String TEST_STRING = "TestSnapshotPersister";
  private static final String MIME = "application/octet-stream";
  private static final String PIPELINE_NAME = "myPipeline";
  private static final String PIPELINE_REV = "0";
  private static final String SNAPSHOT_NAME1 = "snapshot1";
  private static final String SNAPSHOT_NAME2 = "snapshot2";

  private FileSnapshotStore snapshotStore = null;

  @BeforeClass
  public static void beforeClass() {
    System.setProperty(RuntimeInfo.DATA_DIR, "./target/var");
  }

  @AfterClass
  public static void afterClass() {
    System.getProperties().remove(RuntimeInfo.DATA_DIR);
  }

  @Before
  public void setUp() throws IOException {
    File f = new File(System.getProperty(RuntimeInfo.DATA_DIR));
    FileUtils.deleteDirectory(f);
    RuntimeInfo info = new RuntimeInfo(ImmutableList.of(getClass().getClassLoader()));
    snapshotStore = new FileSnapshotStore(info);
    snapshotStore.setInProgress(PIPELINE_NAME, PIPELINE_REV, SNAPSHOT_NAME1, true);
  }

  @After
  public void tearDown() {

  }

  @Test
  public void testStoreAndRetrieveSnapshot() {
    Assert.assertTrue(snapshotStore.retrieveSnapshot(PIPELINE_NAME, PIPELINE_REV, SNAPSHOT_NAME1).isEmpty());
    List<StageOutput> snapshot = createSnapshotData();
    snapshotStore.storeSnapshot(PIPELINE_NAME, PIPELINE_REV, SNAPSHOT_NAME1, snapshot);

    InputStream in = snapshotStore.getSnapshot(PIPELINE_NAME, PIPELINE_REV, SNAPSHOT_NAME1);
    Assert.assertNotNull(in);

    //TODO: Retrieve snapshot and compare contents once de-serializer is ready
    //Assert.assertEquals(2, snapshotStore.retrieveSnapshot(PIPELINE_NAME).size());
    //StageOutput actualS1 = snapshotStore.retrieveSnapshot().get(0);
    //Assert.assertEquals(2);
  }

  @Test
  public void testGetSnapshotStatus() {
    SnapshotStatus snapshotStatus = snapshotStore.getSnapshotStatus(PIPELINE_NAME, PIPELINE_REV, SNAPSHOT_NAME1);
    Assert.assertNotNull(snapshotStatus);
    Assert.assertEquals(false, snapshotStatus.isExists());
    Assert.assertEquals(true, snapshotStatus.isSnapshotInProgress());

    //create snapshot
    List<StageOutput> snapshot = createSnapshotData();
    snapshotStore.storeSnapshot(PIPELINE_NAME, PIPELINE_REV, SNAPSHOT_NAME1, snapshot);

    snapshotStatus = snapshotStore.getSnapshotStatus(PIPELINE_NAME, PIPELINE_REV, SNAPSHOT_NAME1);
    Assert.assertNotNull(snapshotStatus);
    Assert.assertEquals(true, snapshotStatus.isExists());
    Assert.assertEquals(false, snapshotStatus.isSnapshotInProgress());
  }

  @Test
  public void testGetSnapshotsInfo() {
    List<SnapshotInfo> snapshotsInfo = snapshotStore.getSnapshotsInfo(PIPELINE_NAME, PIPELINE_REV);
    Assert.assertEquals(1, snapshotsInfo.size());

    //First one is in progress snapshot info
    Assert.assertEquals(PIPELINE_NAME, snapshotsInfo.get(0).getPipelineName());
    Assert.assertEquals(SNAPSHOT_NAME1, snapshotsInfo.get(0).getSnapshotName());
    Assert.assertNull(snapshotsInfo.get(0).getCaptured()); //Captured is set to null when capturing sanpshot is in progress

    //Store the first snapshot
    List<StageOutput> snapshot = createSnapshotData();
    snapshotStore.storeSnapshot(PIPELINE_NAME, PIPELINE_REV, SNAPSHOT_NAME1, snapshot);

    snapshotsInfo = snapshotStore.getSnapshotsInfo(PIPELINE_NAME, PIPELINE_REV);
    Assert.assertEquals(1, snapshotsInfo.size());
    Assert.assertEquals(PIPELINE_NAME, snapshotsInfo.get(0).getPipelineName());
    Assert.assertEquals(SNAPSHOT_NAME1, snapshotsInfo.get(0).getSnapshotName());
    Assert.assertNotNull(snapshotsInfo.get(0).getCaptured());

    //Capture second snapshot
    snapshotStore.setInProgress(PIPELINE_NAME, PIPELINE_REV, SNAPSHOT_NAME2, true);
    snapshotsInfo = snapshotStore.getSnapshotsInfo(PIPELINE_NAME, PIPELINE_REV);
    Assert.assertEquals(2, snapshotsInfo.size());
    Assert.assertEquals(PIPELINE_NAME, snapshotsInfo.get(1).getPipelineName());
    Assert.assertEquals(SNAPSHOT_NAME2, snapshotsInfo.get(1).getSnapshotName());
    Assert.assertNull(snapshotsInfo.get(1).getCaptured());


    snapshotStore.storeSnapshot(PIPELINE_NAME, PIPELINE_REV, SNAPSHOT_NAME2, snapshot);
    snapshotsInfo = snapshotStore.getSnapshotsInfo(PIPELINE_NAME, PIPELINE_REV);
    Assert.assertEquals(2, snapshotsInfo.size());
    Assert.assertEquals(PIPELINE_NAME, snapshotsInfo.get(1).getPipelineName());
    Assert.assertEquals(SNAPSHOT_NAME2, snapshotsInfo.get(1).getSnapshotName());
    Assert.assertNotNull(snapshotsInfo.get(1).getCaptured());
  }

  @Test
  public void testDeleteSnapshot() {
    //create snapshot
    List<StageOutput> snapshot = createSnapshotData();
    snapshotStore.storeSnapshot(PIPELINE_NAME, PIPELINE_REV, SNAPSHOT_NAME1, snapshot);

    SnapshotStatus snapshotStatus = snapshotStore.getSnapshotStatus(PIPELINE_NAME, PIPELINE_REV, SNAPSHOT_NAME1);
    Assert.assertNotNull(snapshotStatus);
    Assert.assertEquals(true, snapshotStatus.isExists());
    Assert.assertEquals(false, snapshotStatus.isSnapshotInProgress());

    //delete
    snapshotStore.deleteSnapshot(PIPELINE_NAME, PIPELINE_REV, SNAPSHOT_NAME1);
<commit_msg>Fixed test case, my assumption of return snapshot info order was wrong since it changes depending on OS<commit_after>/**
 * (c) 2014 StreamSets, Inc. All rights reserved. May not
 * be copied, modified, or distributed in whole or part without
 * written consent of StreamSets, Inc.
 */
package com.streamsets.pipeline.snapshotstore.impl;

import com.google.common.collect.ImmutableList;
import com.streamsets.pipeline.main.RuntimeInfo;
import com.streamsets.pipeline.api.Field;
import com.streamsets.pipeline.api.Record;
import com.streamsets.pipeline.record.RecordImpl;
import com.streamsets.pipeline.runner.ErrorSink;
import com.streamsets.pipeline.runner.StageOutput;
import com.streamsets.pipeline.snapshotstore.SnapshotInfo;
import com.streamsets.pipeline.snapshotstore.SnapshotStatus;
import org.apache.commons.io.FileUtils;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.mockito.Mockito;

import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

public class TestFileSnapshotStore {

  private static final String TEST_STRING = "TestSnapshotPersister";
  private static final String MIME = "application/octet-stream";
  private static final String PIPELINE_NAME = "myPipeline";
  private static final String PIPELINE_REV = "0";
  private static final String SNAPSHOT_NAME1 = "snapshot1";
  private static final String SNAPSHOT_NAME2 = "snapshot2";

  private FileSnapshotStore snapshotStore = null;

  @BeforeClass
  public static void beforeClass() {
    System.setProperty(RuntimeInfo.DATA_DIR, "./target/var");
  }

  @AfterClass
  public static void afterClass() {
    System.getProperties().remove(RuntimeInfo.DATA_DIR);
  }

  @Before
  public void setUp() throws IOException {
    File f = new File(System.getProperty(RuntimeInfo.DATA_DIR));
    FileUtils.deleteDirectory(f);
    RuntimeInfo info = new RuntimeInfo(ImmutableList.of(getClass().getClassLoader()));
    snapshotStore = new FileSnapshotStore(info);
    snapshotStore.setInProgress(PIPELINE_NAME, PIPELINE_REV, SNAPSHOT_NAME1, true);
  }

  @After
  public void tearDown() {

  }

  @Test
  public void testStoreAndRetrieveSnapshot() {
    Assert.assertTrue(snapshotStore.retrieveSnapshot(PIPELINE_NAME, PIPELINE_REV, SNAPSHOT_NAME1).isEmpty());
    List<StageOutput> snapshot = createSnapshotData();
    snapshotStore.storeSnapshot(PIPELINE_NAME, PIPELINE_REV, SNAPSHOT_NAME1, snapshot);

    InputStream in = snapshotStore.getSnapshot(PIPELINE_NAME, PIPELINE_REV, SNAPSHOT_NAME1);
    Assert.assertNotNull(in);

    //TODO: Retrieve snapshot and compare contents once de-serializer is ready
    //Assert.assertEquals(2, snapshotStore.retrieveSnapshot(PIPELINE_NAME).size());
    //StageOutput actualS1 = snapshotStore.retrieveSnapshot().get(0);
    //Assert.assertEquals(2);
  }

  @Test
  public void testGetSnapshotStatus() {
    SnapshotStatus snapshotStatus = snapshotStore.getSnapshotStatus(PIPELINE_NAME, PIPELINE_REV, SNAPSHOT_NAME1);
    Assert.assertNotNull(snapshotStatus);
    Assert.assertEquals(false, snapshotStatus.isExists());
    Assert.assertEquals(true, snapshotStatus.isSnapshotInProgress());

    //create snapshot
    List<StageOutput> snapshot = createSnapshotData();
    snapshotStore.storeSnapshot(PIPELINE_NAME, PIPELINE_REV, SNAPSHOT_NAME1, snapshot);

    snapshotStatus = snapshotStore.getSnapshotStatus(PIPELINE_NAME, PIPELINE_REV, SNAPSHOT_NAME1);
    Assert.assertNotNull(snapshotStatus);
    Assert.assertEquals(true, snapshotStatus.isExists());
    Assert.assertEquals(false, snapshotStatus.isSnapshotInProgress());
  }

  @Test
  public void testGetSnapshotsInfo() {
    List<SnapshotInfo> snapshotsInfo = snapshotStore.getSnapshotsInfo(PIPELINE_NAME, PIPELINE_REV);
    Assert.assertEquals(1, snapshotsInfo.size());

    //First one is in progress snapshot info
    Assert.assertEquals(PIPELINE_NAME, snapshotsInfo.get(0).getPipelineName());
    Assert.assertEquals(SNAPSHOT_NAME1, snapshotsInfo.get(0).getSnapshotName());
    Assert.assertNull(snapshotsInfo.get(0).getCaptured()); //Captured is set to null when capturing sanpshot is in progress

    //Store the first snapshot
    List<StageOutput> snapshot = createSnapshotData();
    snapshotStore.storeSnapshot(PIPELINE_NAME, PIPELINE_REV, SNAPSHOT_NAME1, snapshot);

    snapshotsInfo = snapshotStore.getSnapshotsInfo(PIPELINE_NAME, PIPELINE_REV);
    Assert.assertEquals(1, snapshotsInfo.size());
    Assert.assertEquals(PIPELINE_NAME, snapshotsInfo.get(0).getPipelineName());
    Assert.assertEquals(SNAPSHOT_NAME1, snapshotsInfo.get(0).getSnapshotName());
    Assert.assertNotNull(snapshotsInfo.get(0).getCaptured());

    //Capture second snapshot
    snapshotStore.setInProgress(PIPELINE_NAME, PIPELINE_REV, SNAPSHOT_NAME2, true);
    snapshotsInfo = snapshotStore.getSnapshotsInfo(PIPELINE_NAME, PIPELINE_REV);
    Assert.assertEquals(2, snapshotsInfo.size());

    SnapshotInfo secondSnapshotInfo = null;
    for(SnapshotInfo snapshotInfo: snapshotsInfo) {
      if(snapshotInfo.getSnapshotName().equals(SNAPSHOT_NAME2)) {
        secondSnapshotInfo = snapshotInfo;
      }
    }
    Assert.assertNotNull(secondSnapshotInfo);
    Assert.assertEquals(PIPELINE_NAME, secondSnapshotInfo.getPipelineName());
    Assert.assertEquals(SNAPSHOT_NAME2, secondSnapshotInfo.getSnapshotName());
    Assert.assertNull(secondSnapshotInfo.getCaptured());


    snapshotStore.storeSnapshot(PIPELINE_NAME, PIPELINE_REV, SNAPSHOT_NAME2, snapshot);
    snapshotsInfo = snapshotStore.getSnapshotsInfo(PIPELINE_NAME, PIPELINE_REV);
    Assert.assertEquals(2, snapshotsInfo.size());

    secondSnapshotInfo = null;
    for(SnapshotInfo snapshotInfo: snapshotsInfo) {
      if(snapshotInfo.getSnapshotName().equals(SNAPSHOT_NAME2)) {
        secondSnapshotInfo = snapshotInfo;
      }
    }
    Assert.assertNotNull(secondSnapshotInfo);
    Assert.assertEquals(PIPELINE_NAME, secondSnapshotInfo.getPipelineName());
    Assert.assertEquals(SNAPSHOT_NAME2, secondSnapshotInfo.getSnapshotName());
    Assert.assertNotNull(secondSnapshotInfo.getCaptured());
  }

  @Test
  public void testDeleteSnapshot() {
    //create snapshot
    List<StageOutput> snapshot = createSnapshotData();
    snapshotStore.storeSnapshot(PIPELINE_NAME, PIPELINE_REV, SNAPSHOT_NAME1, snapshot);

    SnapshotStatus snapshotStatus = snapshotStore.getSnapshotStatus(PIPELINE_NAME, PIPELINE_REV, SNAPSHOT_NAME1);
    Assert.assertNotNull(snapshotStatus);
    Assert.assertEquals(true, snapshotStatus.isExists());
    Assert.assertEquals(false, snapshotStatus.isSnapshotInProgress());

    //delete
    snapshotStore.deleteSnapshot(PIPELINE_NAME, PIPELINE_REV, SNAPSHOT_NAME1);

{'fraction_non_alphanumeric': 0.10123982657640526, 'fraction_numerical': 0.006693542252985472, 'mean_word_length': 5.657215189873417, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1570814', 'n_tokens_mistral': 3916, 'n_tokens_neox': 4004, 'n_words': 714}