<commit_before>/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 * 
 *   http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.uima.cas.impl;

import static org.apache.uima.cas.impl.SerDesTest6.TypeSystems.EqTwoTypes;
import static org.apache.uima.cas.impl.SerDesTest6.TypeSystems.OneType;
import static org.apache.uima.cas.impl.SerDesTest6.TypeSystems.OneTypeSubsetFeatures;
import static org.apache.uima.cas.impl.SerDesTest6.TypeSystems.TwoTypes;
import static org.apache.uima.cas.impl.SerDesTest6.TypeSystems.TwoTypesNoFeatures;
import static org.apache.uima.cas.impl.SerDesTest6.TypeSystems.TwoTypesSubsetFeatures;
import static org.apache.uima.cas.impl.SerDesTest6.Types.Akof1;
import static org.apache.uima.cas.impl.SerDesTest6.Types.Akof2;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Random;

import org.apache.uima.cas.ArrayFS;
import org.apache.uima.cas.BooleanArrayFS;
import org.apache.uima.cas.ByteArrayFS;
import org.apache.uima.cas.CAS;
import org.apache.uima.cas.CASRuntimeException;
import org.apache.uima.cas.DoubleArrayFS;
import org.apache.uima.cas.FSIterator;
import org.apache.uima.cas.Feature;
import org.apache.uima.cas.FeatureStructure;
import org.apache.uima.cas.FloatArrayFS;
import org.apache.uima.cas.IntArrayFS;
import org.apache.uima.cas.LongArrayFS;
import org.apache.uima.cas.ShortArrayFS;
import org.apache.uima.cas.StringArrayFS;
import org.apache.uima.cas.Type;
import org.apache.uima.cas.TypeSystem;
import org.apache.uima.cas.admin.FSIndexRepositoryMgr;
import org.apache.uima.cas.admin.TypeSystemMgr;
import org.apache.uima.cas.impl.BinaryCasSerDes6.ReuseInfo;
import org.apache.uima.cas.test.AnnotatorInitializer;
import org.apache.uima.cas.test.CASInitializer;
import org.apache.uima.jcas.cas.TOP;
import org.apache.uima.resource.ResourceInitializationException;
import org.apache.uima.resource.metadata.TypeSystemDescription;
import org.apache.uima.util.CasCreationUtils;
import org.apache.uima.util.impl.SerializationMeasures;

import junit.framework.TestCase;

/**
 * Serializer and Deserializer testing
 * 
 * 
 */
public class SerDesTest6 extends SerDesTstCommon {

  /**
   * TwoType, EqTwoTypes, TwoTypesSubsetFeatures, TwoTypesNoFeatures have Akof1 and Akof2
   *   
   *
   */
  enum TypeSystems {
    TwoTypes,    // two types, Akof1 Akof2, with all features, one type system
    EqTwoTypes,  // two typesystems, made up of a copy of TwoTypes
    OneType,     // one type, all features
    TwoTypesSubsetFeatures, // two types, but has only a subset of the features
    OneTypeSubsetFeatures,  // one type, but has a subset of features
    TwoTypesNoFeatures,     // two types, no features
  }

  /** the two types Akof1 and Akof2 */
  enum Types {
    Akof1, Akof2,
  }

  private final String           testDocText = "test document text";
  private CASImpl                remoteCas;

  private List<FeatureStructure> lfs;

  private boolean                doPlain     = false;
  // private TypeSystemMgr tsmSrc, tsmTgt;
  private TTypeSystem            mSrc;
  private CASImpl                casSrc;
  private TTypeSystem[]          alternateTTypeSystems;

  public class CASTestSetup implements AnnotatorInitializer {

    public TTypeSystem   m;
    public TypeSystemMgr tsm;
    final TypeSystems    kind; // enum TwoTypes, EqTwoTypes, OneType, TwoTypesSubsetFeatures, OneTypeSubsetFeatures, TwoTypesNoFeatures

    public CASTestSetup(TypeSystems kind) {
      this.kind = kind;
    }

    // TwoTypes, EqTwoTypes, OneType, TwoTypesSubsetFeatures,
    // OneTypeSubsetFeatures, NoFeatures,
    public void initTypeSystem(TypeSystemMgr tsm) {
      if (kind == EqTwoTypes) {
        throw new RuntimeException();
      }

      this.tsm = tsm;
      m = new TTypeSystem(tsm, kind);
      addBuiltins();

      m.addType(Akof1.name(), "Top");
      if (kind != TwoTypesNoFeatures) {
        for (String fn : featureNameRoots) {
          if (kind != OneTypeSubsetFeatures ||
              random.nextInt(3) < 2) { 
            m.add(Akof1, fn);
          }
        }
      }

      switch (kind) {
      case TwoTypes:
      case TwoTypesSubsetFeatures:
      case TwoTypesNoFeatures:
        m.addType(Akof2.name(), "Top");
        if (kind != TwoTypesNoFeatures) {
          for (String fn : featureNameRoots) {
            if (kind != TwoTypesSubsetFeatures ||
                random.nextInt(3) < 2) { 
              m.add(Akof2, fn);
            }
          }
        }
        break;
      default: // skip the other cases
      } // end of switch
    }

    void addBuiltins() {
      // Add new types and features.
      m.addType(tsm.getTopType(), "Top");

      m.addType(tsm.getType(CAS.TYPE_NAME_INTEGER_ARRAY), "Aint");
      m.addType(tsm.getType(CAS.TYPE_NAME_FS_ARRAY), "Afs");
      m.addType(tsm.getType(CAS.TYPE_NAME_FLOAT_ARRAY), "Afloat");
      m.addType(tsm.getType(CAS.TYPE_NAME_DOUBLE_ARRAY), "Adouble");
      m.addType(tsm.getType(CAS.TYPE_NAME_LONG_ARRAY), "Along");
      m.addType(tsm.getType(CAS.TYPE_NAME_SHORT_ARRAY), "Ashort");
      m.addType(tsm.getType(CAS.TYPE_NAME_BYTE_ARRAY), "Abyte");
      m.addType(tsm.getType(CAS.TYPE_NAME_BOOLEAN_ARRAY), "Aboolean");
      m.addType(tsm.getType(CAS.TYPE_NAME_STRING_ARRAY), "Astring");

      m.addType(tsm.getType(CAS.TYPE_NAME_INTEGER), "Int");
      m.addType(tsm.getType(CAS.TYPE_NAME_FLOAT), "Float");
      m.addType(tsm.getType(CAS.TYPE_NAME_DOUBLE), "Double");
      m.addType(tsm.getType(CAS.TYPE_NAME_LONG), "Long");
      m.addType(tsm.getType(CAS.TYPE_NAME_SHORT), "Short");
      m.addType(tsm.getType(CAS.TYPE_NAME_BYTE), "Byte");
      m.addType(tsm.getType(CAS.TYPE_NAME_BOOLEAN), "Boolean");
      m.addType(tsm.getType(CAS.TYPE_NAME_STRING), "String");
      m.addType(tsm.getType(CAS.TYPE_NAME_TOP), "Fs");
    }

    public void initIndexes(FSIndexRepositoryMgr irm, TypeSystem ts) {
    }
  }
  
  /** list of all feature value kinds e.g. Int, Fs, Float, Afloat, Astring */
  static final List<String> featureNameRoots = Arrays.asList(new String[] { 
    "Int", "Fs", "Float", "Double", "Long", "Short", "Byte", "Boolean", "String", 
          "Aint", "Afs", "Afloat", "Adouble", "Along", "Ashort", "Abyte", "Aboolean", "Astring" });
  
  /** have 2 instances of this, one per type system */
  static class TTypeSystem {
    final TypeSystems     kind; // enum TwoTypes, EqTwoTypes, OneType, TwoTypesSubsetFeatures, OneTypeSubsetFeatures, TwoTypesNoFeatures,
    TypeSystemMgr tsm;
    /** table, by two types, then sparse table, each slot not empty is a feature indexed by corresponding featureNameRoots */
    Feature[][] featureTable = new Feature[Types.values().length][featureNameRoots.size()];
    /** short-name to types map: e.g. Aint -> array of int  */
    Map<String, Type>     mapString2Type      = new HashMap<String, Type>();
    public TypeSystemImpl ts;
    public CASImpl cas;  // the Cas setup as part of initialization                                                                    // the
    private boolean isUpdateAfterCommit = false; // only needed for v3, ignored for v2

    public TTypeSystem(TypeSystemMgr tsm, TypeSystems kind) {
      this.tsm = tsm;
      this.kind = kind;
      this.ts = (TypeSystemImpl) tsm;
    }

    /** add existing type by short name to the map, e.g. Aint -> array of int */
    void addType(Type type, String shortName) {
      mapString2Type.put(shortName, type);
    }

    /** add new type with super type */
    void addType(String type, String superType) {
      addType(tsm.addType(type, getType(superType)), type);
    }

    /** get type by short name */
    Type getType(String shortName) {
      return mapString2Type.get(shortName);
    }

    /** get type by enum Akof1, Akof2 */
    Type getType(Types type) {
      return getType(type.name());
    }

    /** add feature to type, feat = one of feature value kinds e.g. Int, Fs, Float, Afloat, Astring */
    void add(Type type, String featNameRoot) {
      String typeName = type.getShortName();
      int i2 = featureNameRoots.indexOf(featNameRoot);
      featureTable[Types.valueOf(typeName).ordinal()][i2] = tsm.addFeature(typeName + featNameRoot,
          type, mapString2Type.get(featNameRoot));
    }
    
    /** add feature to type, arg1 is enum, feat = one of feature value kinds e.g. Int, Fs, Float, Afloat, Astring */
    void add(Types typeKind, String featNameRoot) {
      add(getType(typeKind.name()), featNameRoot);
    }

    /** get feature for type by typeEnum + one of feature value kinds: Int, Fs, Float, Afloat, Astring */
    Feature getFeature(Types typeKind, String featNameRoot) {
      return featureTable[typeKind.ordinal()][featureNameRoots.indexOf(featNameRoot)];
    }

    /** get feature for type of FS, + one of feature value kinds: Int, Fs, Float, Afloat, Astring */
    Feature getFeature(FeatureStructure fs, String featNameRoot) {
      Type t = fs.getType();
      return getFeature(Types.valueOf(t.getShortName()), featNameRoot);
    }

    void updateAfterCommit() {
      ts = cas.getTypeSystemImpl();
      tsm = ts;
      for (String typename : mapString2Type.keySet()) {
        TypeImpl ti = ts.getType(typename);
        mapString2Type.put(typename, ti);
      }

      for (Types typeKind : Types.values()) {
        Type ti = tsm.getType(typeKind.name());
        if (ti != null) {
          Feature[] features = featureTable[typeKind.ordinal()];
          for (int i = 0; i < features.length; i++) {
            features[i] = ti.getFeatureByBaseName(ti.getShortName() + featureNameRoots.get(i));
          }
        }
      }
      isUpdateAfterCommit = true;
    }
  }

  public TTypeSystem setupTTypeSystem(TypeSystems kind) {
    if (kind == EqTwoTypes) {
      TTypeSystem m = new TTypeSystem(mSrc.tsm, kind);
      m.cas = mSrc.cas;
      // m.ts = mSrc.cas.getTypeSystemImpl();
      return mSrc;
    }
    CASTestSetup setup = new CASTestSetup(kind);
    CASImpl cas = (CASImpl) CASInitializer.initCas(setup, null);
    TTypeSystem m = setup.m;
    m.cas = cas;
    m.updateAfterCommit();
    return m;
  }

  public void setUp() {
    mSrc = setupTTypeSystem(TwoTypes);
    casSrc = mSrc.cas;
    final TypeSystems[] tss = TypeSystems.values();
    final int nbrAltTs = tss.length;
    alternateTTypeSystems = new TTypeSystem[nbrAltTs];
    for (int i = 0; i < nbrAltTs; i++) {
      alternateTTypeSystems[i] = setupTTypeSystem(tss[i]);
    }
    lfs = new ArrayList<FeatureStructure>();
  }

  public void tearDown() {
    remoteCas = null;
    lfs = null;
    mSrc = null;
    casSrc = null;
    alternateTTypeSystems = null;
    // don't reset random
  }

  // void setupTgtTs(String kind) {
  // CASTestSetup setupTgt = new CASTestSetup(kind);
  // casTgt = (CASImpl) CASInitializer.initCas(setupTgt);
  // mTgt = setupTgt.m;
  // mTgt.ts = casTgt.getTypeSystemImpl();
  // }

  TTypeSystem getTT(TypeSystems kind) {
    return alternateTTypeSystems[kind.ordinal()];
  }

  CASImpl setupCas(TTypeSystem m) {
    return createCAS(m.ts);
  }

  CASImpl createCAS(TypeSystemImpl ts) {
    try {
      return (CASImpl) CasCreationUtils.createCas(ts, null, null, null, null);
    } catch (ResourceInitializationException e) {
      throw new RuntimeException(e);
    }
  }

  public void testDocText() {
    try {
      CAS cas = CasCreationUtils.createCas((TypeSystemDescription) null, null, null);
      cas.setDocumentLanguage("latin");
      cas.setDocumentText("test");

      ByteArrayOutputStream baos = new ByteArrayOutputStream(1024);

      Serialization.serializeWithCompression(cas, baos, cas.getTypeSystem());

      CAS cas2 = CasCreationUtils.createCas((TypeSystemDescription) null, null, null);
      ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
      Serialization.deserializeCAS(cas2, bais);

      assertEquals("latin", cas2.getDocumentLanguage());
      assertEquals("test", cas2.getDocumentText());
    } catch (Exception e) {
      throw new RuntimeException(e);
    }
  }

  public void testDocumentText() {
    // serdesSimple(getTT(EqTwoTypes));
    remoteCas = setupCas(getTT(EqTwoTypes));
    casSrc.reset();
    casSrc.setDocumentText(testDocText);
    loadCas(casSrc, mSrc);
    verify(remoteCas, "DocumentText");
    assertEquals(remoteCas.getDocumentText(), testDocText);

    // test case where serialization is done without type filtering,
    // and deserialization is done with filtering
    remoteCas.reset();
    verifyDeserFilter(remoteCas, "WithDeserFilterDocumentText");
    assertEquals(remoteCas.getDocumentText(), testDocText);
  }

  /**
   * Make one of each kind of artifact, including arrays
   * serialize to byte stream, deserialize into new cas, compare
   */

  public void testAllKinds() {
    if (doPlain) {
      serdesSimple(getTT(EqTwoTypes), "EqTwoTypes");
    } else {
      int i = 0;
      for (TTypeSystem m : alternateTTypeSystems) {
       
        switch (m.kind) {
        // note: case statements *not* grouped in order to faclitate debugging
        case OneTypeSubsetFeatures:
          serdesSimple(m, "OneTypeSubsetFeatures" + m.kind.toString());
          break;
        case TwoTypesSubsetFeatures:
          serdesSimple(m, "TwoTypesSubsetFeatures" + m.kind.toString());
          break;
        case TwoTypes:
          i++;
          serdesSimple(m, "OtherAllKinds" + m.kind.toString() + Integer.toString(i));
          break;
        case EqTwoTypes:
        case OneType:
        case TwoTypesNoFeatures:
          serdesSimple(m, "OtherAllKinds" + m.kind.toString());
          break;
        }
      }
    }
  }

  // Test chains going through filtered type
  // Repeat below with OneType, and TwoTypes with filtered slot == fsRef

  // T1 fsArray ref -> T2 -> T1 (new) (not indexed)
  // T1 ref -> T2 -> T1 (new) (not indexed)
  // T1 fsArray ref -> T2 -> T1 (new) (indexed)
  // T1 ref -> T2 -> T1 (new) (indexed)

  public void testRefThroughFilteredType() {
    reftft(OneType, 0);
    for (int i = 0; i < 10; i++) {
      reftft(TwoTypesSubsetFeatures, i);
    }
    reftft(TwoTypesNoFeatures, 0);
  }

  private void reftft(TypeSystems tskind, int i) {
    reftft(tskind, true, i);
    reftft(tskind, false, i);
  }

  /**
   * Inner part of test of refs through filtered type
   * 
   * @param tskind -
   * @param indexed -
   */
  private void reftft(TypeSystems tskind, boolean indexed, int i) {
    lfs.clear();

    TTypeSystem m = getTT(tskind);
    remoteCas = setupCas(m);
    // casSrc.reset();
    makeFeaturesForAkof(casSrc, mSrc, Akof1);

    FeatureStructure otherTsFs = casSrc.createFS(mSrc.getType(Akof2));
    FeatureStructure fsOrig = lfs.get(0);
    fsOrig.setFeatureValue(mSrc.getFeature(fsOrig, "Fs"), otherTsFs);

    FeatureStructure ts1Fs = casSrc.createFS(mSrc.getType(Akof1));
    otherTsFs.setFeatureValue(mSrc.getFeature(otherTsFs, "Fs"), ts1Fs);

    if (indexed) {
      casSrc.addFsToIndexes(ts1Fs);
    }

    verify(remoteCas, "refThroughFilteredType" 
       + (indexed ? "Indexed" : "NotIndexed") 
       + tskind.toString() + Integer.toString(i));
    
  }

  // broken out special instances of random tests
  public void testDeltaWithStringArrayMod() {
    // casSrc -> remoteCas,remoteCas updated, serialized back to srcCas
    for (int i = 0; i < 10; i++) {
      TTypeSystem m = getTT(EqTwoTypes);
      remoteCas = setupCas(m);
      loadCas(casSrc, mSrc);
      ReuseInfo[] ri = serializeDeserialize(casSrc, remoteCas, null, null);
      MarkerImpl marker = (MarkerImpl) remoteCas.createMarker();
      lfs = getIndexedFSs(remoteCas, m);
      FeatureStructure fs = lfs.get(10);
      StringArrayFS sa = (StringArrayFS) maybeGetFeatureKind(fs, m, "Astring");
      if (sa == null) { // could happen because features are randomly omitted
        System.out.println("    Astring feature omitted, retrying");
      } else if (sa.size() == 0) {
        System.out.println("    Astring feature array has 0 length, retrying");
      } else {
        sa.set(0, "change2");
        verifyDelta(marker, ri);
        break;
      }
      // setRandom();
      setUp();
      long seed = random.nextLong();
      random.setSeed(seed);
      System.out.println(" testDelta w/ String array mod random = " + seed + ", i = " + i);
    }
  }

  public void testDeltaWithDblArrayMod() {
    for (int i = 0; i < 10; i++) {
      TTypeSystem m = getTT(EqTwoTypes);
      remoteCas = setupCas(m);
      loadCas(casSrc, mSrc);
      ReuseInfo[] ri = serializeDeserialize(casSrc, remoteCas, null, null);
      MarkerImpl marker = (MarkerImpl) remoteCas.createMarker();
      lfs = getIndexedFSs(remoteCas, m);
      FeatureStructure fs = lfs.get(10); /* has double array length 2 */
      DoubleArrayFS d = (DoubleArrayFS) maybeGetFeatureKind(fs, m, "Adouble");
      if (d == null) { // could happen because features are randomly omitted
        System.out.println("    Adouble feature omitted, retrying");
      } else if (d.size() == 0) {
        System.out.println("    Adouble feature array has 0 length, retrying");
      } else {
        d.set(0, 12.34D);
        verifyDelta(marker, ri);
        break;
      }
      // setRandom();
      setUp();
      long seed = random.nextLong();
      random.setSeed(seed);
      System.out.println(" testDelta w/ dbl array mod random = " + seed + ", i = " + i);
    }
  }

  public void testDeltaWithByteArrayMod() {
    for (int i = 0; i < 10; i++) {
      TTypeSystem m = getTT(EqTwoTypes);
      remoteCas = setupCas(m);
      loadCas(casSrc, mSrc);
      ReuseInfo[] ri = serializeDeserialize(casSrc, remoteCas, null, null);
      MarkerImpl marker = (MarkerImpl) remoteCas.createMarker();

      lfs = getIndexedFSs(remoteCas, m); // gets FSs below the line

      FeatureStructure fs = lfs.get(10);
      ByteArrayFS sfs = (ByteArrayFS) maybeGetFeatureKind(fs, m, "Abyte");
      if (sfs == null) { // could happen because features are randomly omitted
        System.out.println("    Abyte feature omitted, retrying");
      } else if (sfs.size() == 0) {
        System.out.println("    Abyte feature array has 0 length, retrying");
      } else {
        sfs.set(0, (byte) 21);
        verifyDelta(marker, ri);
        break;
      }
      // setRandom(); // retry with different random number
      setUp();
      System.out.println("  testDelta w byte array mod retrying, i = " + i);
    }
  }

  public void testDeltaWithStrArrayMod() {
    TTypeSystem m = getTT(EqTwoTypes);
    remoteCas = setupCas(m);
    loadCas(casSrc, mSrc);
    ReuseInfo[] ri = serializeDeserialize(casSrc, remoteCas, null, null);
    MarkerImpl marker = (MarkerImpl) remoteCas.createMarker();

    lfs = getIndexedFSs(remoteCas, m);

    FeatureStructure fs = lfs.get(10);
    StringArrayFS sfs = (StringArrayFS) maybeGetFeatureKind(fs, m, "Astring");
    if (sfs != null) {
      sfs.set(0, "change");
    }

    verifyDelta(marker, ri);
  }

  private void serdesSimple(TTypeSystem m, String kind) {
    remoteCas = setupCas(m);
    casSrc.reset();
    loadCas(casSrc, mSrc);
    verify(remoteCas, kind);

    // test case where serialization is done without type filtering,
    // and deserialization is done with filtering
    remoteCas.reset();
    verifyDeserFilter(remoteCas, "WithDeserFilter" + kind);
  }

  /**
   * 1) create a base cas with some data
   * 2) serialize it out and then back into a remoteCas
   *    This is needed to get the proper ordering in the remote cas of the 
   *    feature structures - they will be ordered by their fs addr, but omitting
   *    "skipped" FSs (not in the index or reachable, and not in the target ts)
   * 2) create the mark in the remoteCAS: cas.createMarker()
   * 3) add more cas data to the remoteCas
   * 4) serialize with marker
   * 5) deserialize back into base cas
   * 6) check resulting base cas = remote cas
   * 
   * 
   */
  public void testDelta() {
    if (doPlain) {
      serdesDelta(getTT(EqTwoTypes));
    } else {
      for (TTypeSystem m : alternateTTypeSystems) {
        switch (m.kind) {
        case TwoTypesSubsetFeatures:
          serdesDelta(m);
          break;
        case OneTypeSubsetFeatures:
          serdesDelta(m);
          break;
        case TwoTypes:
        case EqTwoTypes:
        case OneType:
        case TwoTypesNoFeatures:
          serdesDelta(m);
          break;
        }
      }
    }
  }

  private void serdesDelta(TTypeSystem m) {
    remoteCas = setupCas(m); // create empty new CAS with specified type system from m.ts
//    casSrc.reset();
    loadCas(casSrc, mSrc); // load up the src cas using mSrc spec
    // src -> serialize -> deserialize -> rmt
    ReuseInfo[] ri = serializeDeserialize(casSrc, remoteCas, null, null); 

    MarkerImpl marker = (MarkerImpl) remoteCas.createMarker();
    loadCas(remoteCas, m); // load some changes into remote
    // rmt -> serialize(full ts) -> deserialize(2 ts) -> src, then compare src & rmt
    verifyDelta(marker, ri); 
  }

  public void testDeltaWithRefsBelow() {
    lfs.clear();
    TTypeSystem m = getTT(EqTwoTypes);
    remoteCas = setupCas(m);
    loadCas(casSrc, mSrc);
    ReuseInfo ri[] = serializeDeserialize(casSrc, remoteCas, null, null);
    MarkerImpl marker = (MarkerImpl) remoteCas.createMarker();

    lfs = getIndexedFSs(remoteCas, m);
    FeatureStructure fs = remoteCas.createFS(m.getType(Akof1));
    maybeSetFeature(fs, m, lfs.get(0));
    ArrayFS fsafs = remoteCas.createArrayFS(4);
    fsafs.set(1, lfs.get(1));
    fsafs.set(2, lfs.get(2));
    fsafs.set(3, lfs.get(3));
    maybeSetFeatureKind(fs, m, "Afs", fsafs);

    verifyDelta(marker, ri);
  }

  public void testDeltaWithMods() {
    lfs.clear();
    TTypeSystem m = getTT(EqTwoTypes);
    remoteCas = setupCas(m);
    loadCas(casSrc, mSrc);
    ReuseInfo ri[] = serializeDeserialize(casSrc, remoteCas, null, null);
    MarkerImpl marker = (MarkerImpl) remoteCas.createMarker();

    lfs = getIndexedFSs(remoteCas, m); // get list of all "Akof1" FS
    FeatureStructure fs = remoteCas.createFS(m.getType(Akof1));
    // set the lfs.get(0) featurestructure's feature "Fs" to the new fs
    maybeSetFeatureKind(lfs.get(0), m, "Fs", fs); 

    verifyDelta(marker, ri);
  }

  /**
   * Variations to cover:
   *   all kinds of slots
   *   multiple sets of values
   *     test diffs
   * multiple orders (try reverse and random order)
   * 
   * Driver for random values
   *   pick among random and "interesting" edge case values
   * 
   * 
   */
  public void testDeltaWithAllMods() {

    for (int i = 0; i < 100; i++) {
      checkDeltaWithAllMods();
      tearDown();
      setUp();
    }
  }

  public void checkDeltaWithAllMods() {
    makeRandomFss(casSrc, mSrc, Akof1, 7);
    TTypeSystem m = getTT(EqTwoTypes);
    remoteCas = setupCas(m);
    loadCas(casSrc, mSrc);
    ReuseInfo ri[] = serializeDeserialize(casSrc, remoteCas, null, null);
    MarkerImpl marker = (MarkerImpl) remoteCas.createMarker();

    lfs = getIndexedFSs(remoteCas, m);

    makeRandomFss(remoteCas, m, Akof1, 8);

    int i = 0;
    for (FeatureStructure fs : lfs) {
      if (((i++) % 2) == 0) {
        maybeSetFeature(fs, m, lfs.get(random.nextInt(lfs.size())));
      }
    }

    makeRandomUpdatesBelowMark(remoteCas, m, Akof1);

    verifyDelta(marker, ri);

  }

  public void testDeltaWithIndexMods() {
    TTypeSystem m = getTT(EqTwoTypes);
    remoteCas = setupCas(m);
    loadCas(casSrc, mSrc);
    ReuseInfo ri[] = serializeDeserialize(casSrc, remoteCas, null, null);
    MarkerImpl marker = (MarkerImpl) remoteCas.createMarker();

    lfs = new ArrayList<FeatureStructure>();
    loadCas(remoteCas, m);

    List<FeatureStructure> lfs2 = getIndexedFSs(remoteCas, m);

    remoteCas.getIndexRepository().removeFS(lfs2.get(0));
    remoteCas.getIndexRepository().removeFS(lfs2.get(1));
    remoteCas.getIndexRepository().addFS(lfs2.get(1));  // should appear as reindexed

    remoteCas.getIndexRepository().removeFS(lfs.get(0));
    remoteCas.getIndexRepository().removeFS(lfs.get(1));
    remoteCas.getIndexRepository().addFS(lfs.get(1));

    verifyDelta(marker, ri);
  }

  public void testWithOtherSerializer() {
    doPlain = true;
    testDeltaWithMods();
    tearDown(); setUp();
    testDeltaWithRefsBelow();
    tearDown(); setUp();
    // testDeltaWithAllMods();
    tearDown(); setUp();
    testAllKinds();
    tearDown(); setUp();
    testArrayAux();
  }

  private void runCaptureSet() {
    //  Java 8 style
//    setupRunTeardown(this::testDocText);
//    setupRunTeardown(this::testDocumentText);
//    setupRunTeardown(this::testAllKinds);
//    setupRunTeardown(this::testRefThroughFilteredType);
//    setupRunTeardown(this::testDeltaWithStringArrayMod);
//    setupRunTeardown(this::testDeltaWithDblArrayMod);
//    setupRunTeardown(this::testDeltaWithByteArrayMod);
//    setupRunTeardown(this::testDeltaWithStrArrayMod);
//    setupRunTeardown(this::testDelta);
//    setupRunTeardown(this::testDeltaWithRefsBelow);
//    setupRunTeardown(this::testDeltaWithMods);
//    setupRunTeardown(this::testDeltaWithIndexMods);
//    setupRunTeardown(this::testArrayAux);
    
    // Java 7 style
    setupRunTeardown(new Runnable() {public void run() {testDocText();}});     
    setupRunTeardown(new Runnable() {public void run() {testDocumentText();}});
    setupRunTeardown(new Runnable() {public void run() {testAllKinds();}});
    setupRunTeardown(new Runnable() {public void run() {testRefThroughFilteredType();}});
    setupRunTeardown(new Runnable() {public void run() {testDeltaWithStringArrayMod();}});
    setupRunTeardown(new Runnable() {public void run() {testDeltaWithDblArrayMod();}});
    setupRunTeardown(new Runnable() {public void run() {testDeltaWithByteArrayMod();}});
    setupRunTeardown(new Runnable() {public void run() {testDeltaWithStrArrayMod();}});
    setupRunTeardown(new Runnable() {public void run() {testDelta();}});
    setupRunTeardown(new Runnable() {public void run() {testDeltaWithRefsBelow();}});
    setupRunTeardown(new Runnable() {public void run() {testDeltaWithMods();}});
    setupRunTeardown(new Runnable() {public void run() {testDeltaWithIndexMods();}});
    setupRunTeardown(new Runnable() {public void run() {testArrayAux();}});

  }

  public void captureGenerated() {
    capture = true;
    initWriteSavedInts();
    runCaptureSet();
    try {
      savedIntsOutStream.close();
    } catch (IOException e) {
      throw new RuntimeException(e);
    }
    capture = false;<commit_msg>add a test looper to catch infrequent bugs.  Normally disabled.<commit_after>/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 * 
 *   http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.uima.cas.impl;

import static org.apache.uima.cas.impl.SerDesTest6.TypeSystems.EqTwoTypes;
import static org.apache.uima.cas.impl.SerDesTest6.TypeSystems.OneType;
import static org.apache.uima.cas.impl.SerDesTest6.TypeSystems.OneTypeSubsetFeatures;
import static org.apache.uima.cas.impl.SerDesTest6.TypeSystems.TwoTypes;
import static org.apache.uima.cas.impl.SerDesTest6.TypeSystems.TwoTypesNoFeatures;
import static org.apache.uima.cas.impl.SerDesTest6.TypeSystems.TwoTypesSubsetFeatures;
import static org.apache.uima.cas.impl.SerDesTest6.Types.Akof1;
import static org.apache.uima.cas.impl.SerDesTest6.Types.Akof2;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Random;

import org.apache.uima.cas.ArrayFS;
import org.apache.uima.cas.BooleanArrayFS;
import org.apache.uima.cas.ByteArrayFS;
import org.apache.uima.cas.CAS;
import org.apache.uima.cas.CASRuntimeException;
import org.apache.uima.cas.DoubleArrayFS;
import org.apache.uima.cas.FSIterator;
import org.apache.uima.cas.Feature;
import org.apache.uima.cas.FeatureStructure;
import org.apache.uima.cas.FloatArrayFS;
import org.apache.uima.cas.IntArrayFS;
import org.apache.uima.cas.LongArrayFS;
import org.apache.uima.cas.ShortArrayFS;
import org.apache.uima.cas.StringArrayFS;
import org.apache.uima.cas.Type;
import org.apache.uima.cas.TypeSystem;
import org.apache.uima.cas.admin.FSIndexRepositoryMgr;
import org.apache.uima.cas.admin.TypeSystemMgr;
import org.apache.uima.cas.impl.BinaryCasSerDes6.ReuseInfo;
import org.apache.uima.cas.test.AnnotatorInitializer;
import org.apache.uima.cas.test.CASInitializer;
import org.apache.uima.jcas.cas.TOP;
import org.apache.uima.resource.ResourceInitializationException;
import org.apache.uima.resource.metadata.TypeSystemDescription;
import org.apache.uima.util.CasCreationUtils;
import org.apache.uima.util.impl.SerializationMeasures;

import junit.framework.TestCase;

/**
 * Serializer and Deserializer testing
 * 
 * 
 */
public class SerDesTest6 extends SerDesTstCommon {

  /**
   * TwoType, EqTwoTypes, TwoTypesSubsetFeatures, TwoTypesNoFeatures have Akof1 and Akof2
   *   
   *
   */
  enum TypeSystems {
    TwoTypes,    // two types, Akof1 Akof2, with all features, one type system
    EqTwoTypes,  // two typesystems, made up of a copy of TwoTypes
    OneType,     // one type, all features
    TwoTypesSubsetFeatures, // two types, but has only a subset of the features
    OneTypeSubsetFeatures,  // one type, but has a subset of features
    TwoTypesNoFeatures,     // two types, no features
  }

  /** the two types Akof1 and Akof2 */
  enum Types {
    Akof1, Akof2,
  }

  private final String           testDocText = "test document text";
  private CASImpl                remoteCas;

  private List<FeatureStructure> lfs;

  private boolean                doPlain     = false;
  // private TypeSystemMgr tsmSrc, tsmTgt;
  private TTypeSystem            mSrc;
  private CASImpl                casSrc;
  private TTypeSystem[]          alternateTTypeSystems;

  public class CASTestSetup implements AnnotatorInitializer {

    public TTypeSystem   m;
    public TypeSystemMgr tsm;
    final TypeSystems    kind; // enum TwoTypes, EqTwoTypes, OneType, TwoTypesSubsetFeatures, OneTypeSubsetFeatures, TwoTypesNoFeatures

    public CASTestSetup(TypeSystems kind) {
      this.kind = kind;
    }

    // TwoTypes, EqTwoTypes, OneType, TwoTypesSubsetFeatures,
    // OneTypeSubsetFeatures, NoFeatures,
    public void initTypeSystem(TypeSystemMgr tsm) {
      if (kind == EqTwoTypes) {
        throw new RuntimeException();
      }

      this.tsm = tsm;
      m = new TTypeSystem(tsm, kind);
      addBuiltins();

      m.addType(Akof1.name(), "Top");
      if (kind != TwoTypesNoFeatures) {
        for (String fn : featureNameRoots) {
          if (kind != OneTypeSubsetFeatures ||
              random.nextInt(3) < 2) { 
            m.add(Akof1, fn);
          }
        }
      }

      switch (kind) {
      case TwoTypes:
      case TwoTypesSubsetFeatures:
      case TwoTypesNoFeatures:
        m.addType(Akof2.name(), "Top");
        if (kind != TwoTypesNoFeatures) {
          for (String fn : featureNameRoots) {
            if (kind != TwoTypesSubsetFeatures ||
                random.nextInt(3) < 2) { 
              m.add(Akof2, fn);
            }
          }
        }
        break;
      default: // skip the other cases
      } // end of switch
    }

    void addBuiltins() {
      // Add new types and features.
      m.addType(tsm.getTopType(), "Top");

      m.addType(tsm.getType(CAS.TYPE_NAME_INTEGER_ARRAY), "Aint");
      m.addType(tsm.getType(CAS.TYPE_NAME_FS_ARRAY), "Afs");
      m.addType(tsm.getType(CAS.TYPE_NAME_FLOAT_ARRAY), "Afloat");
      m.addType(tsm.getType(CAS.TYPE_NAME_DOUBLE_ARRAY), "Adouble");
      m.addType(tsm.getType(CAS.TYPE_NAME_LONG_ARRAY), "Along");
      m.addType(tsm.getType(CAS.TYPE_NAME_SHORT_ARRAY), "Ashort");
      m.addType(tsm.getType(CAS.TYPE_NAME_BYTE_ARRAY), "Abyte");
      m.addType(tsm.getType(CAS.TYPE_NAME_BOOLEAN_ARRAY), "Aboolean");
      m.addType(tsm.getType(CAS.TYPE_NAME_STRING_ARRAY), "Astring");

      m.addType(tsm.getType(CAS.TYPE_NAME_INTEGER), "Int");
      m.addType(tsm.getType(CAS.TYPE_NAME_FLOAT), "Float");
      m.addType(tsm.getType(CAS.TYPE_NAME_DOUBLE), "Double");
      m.addType(tsm.getType(CAS.TYPE_NAME_LONG), "Long");
      m.addType(tsm.getType(CAS.TYPE_NAME_SHORT), "Short");
      m.addType(tsm.getType(CAS.TYPE_NAME_BYTE), "Byte");
      m.addType(tsm.getType(CAS.TYPE_NAME_BOOLEAN), "Boolean");
      m.addType(tsm.getType(CAS.TYPE_NAME_STRING), "String");
      m.addType(tsm.getType(CAS.TYPE_NAME_TOP), "Fs");
    }

    public void initIndexes(FSIndexRepositoryMgr irm, TypeSystem ts) {
    }
  }
  
  /** list of all feature value kinds e.g. Int, Fs, Float, Afloat, Astring */
  static final List<String> featureNameRoots = Arrays.asList(new String[] { 
    "Int", "Fs", "Float", "Double", "Long", "Short", "Byte", "Boolean", "String", 
          "Aint", "Afs", "Afloat", "Adouble", "Along", "Ashort", "Abyte", "Aboolean", "Astring" });
  
  /** have 2 instances of this, one per type system */
  static class TTypeSystem {
    final TypeSystems     kind; // enum TwoTypes, EqTwoTypes, OneType, TwoTypesSubsetFeatures, OneTypeSubsetFeatures, TwoTypesNoFeatures,
    TypeSystemMgr tsm;
    /** table, by two types, then sparse table, each slot not empty is a feature indexed by corresponding featureNameRoots */
    Feature[][] featureTable = new Feature[Types.values().length][featureNameRoots.size()];
    /** short-name to types map: e.g. Aint -> array of int  */
    Map<String, Type>     mapString2Type      = new HashMap<String, Type>();
    public TypeSystemImpl ts;
    public CASImpl cas;  // the Cas setup as part of initialization                                                                    // the
    private boolean isUpdateAfterCommit = false; // only needed for v3, ignored for v2

    public TTypeSystem(TypeSystemMgr tsm, TypeSystems kind) {
      this.tsm = tsm;
      this.kind = kind;
      this.ts = (TypeSystemImpl) tsm;
    }

    /** add existing type by short name to the map, e.g. Aint -> array of int */
    void addType(Type type, String shortName) {
      mapString2Type.put(shortName, type);
    }

    /** add new type with super type */
    void addType(String type, String superType) {
      addType(tsm.addType(type, getType(superType)), type);
    }

    /** get type by short name */
    Type getType(String shortName) {
      return mapString2Type.get(shortName);
    }

    /** get type by enum Akof1, Akof2 */
    Type getType(Types type) {
      return getType(type.name());
    }

    /** add feature to type, feat = one of feature value kinds e.g. Int, Fs, Float, Afloat, Astring */
    void add(Type type, String featNameRoot) {
      String typeName = type.getShortName();
      int i2 = featureNameRoots.indexOf(featNameRoot);
      featureTable[Types.valueOf(typeName).ordinal()][i2] = tsm.addFeature(typeName + featNameRoot,
          type, mapString2Type.get(featNameRoot));
    }
    
    /** add feature to type, arg1 is enum, feat = one of feature value kinds e.g. Int, Fs, Float, Afloat, Astring */
    void add(Types typeKind, String featNameRoot) {
      add(getType(typeKind.name()), featNameRoot);
    }

    /** get feature for type by typeEnum + one of feature value kinds: Int, Fs, Float, Afloat, Astring */
    Feature getFeature(Types typeKind, String featNameRoot) {
      return featureTable[typeKind.ordinal()][featureNameRoots.indexOf(featNameRoot)];
    }

    /** get feature for type of FS, + one of feature value kinds: Int, Fs, Float, Afloat, Astring */
    Feature getFeature(FeatureStructure fs, String featNameRoot) {
      Type t = fs.getType();
      return getFeature(Types.valueOf(t.getShortName()), featNameRoot);
    }

    void updateAfterCommit() {
      ts = cas.getTypeSystemImpl();
      tsm = ts;
      for (String typename : mapString2Type.keySet()) {
        TypeImpl ti = ts.getType(typename);
        mapString2Type.put(typename, ti);
      }

      for (Types typeKind : Types.values()) {
        Type ti = tsm.getType(typeKind.name());
        if (ti != null) {
          Feature[] features = featureTable[typeKind.ordinal()];
          for (int i = 0; i < features.length; i++) {
            features[i] = ti.getFeatureByBaseName(ti.getShortName() + featureNameRoots.get(i));
          }
        }
      }
      isUpdateAfterCommit = true;
    }
  }

  public TTypeSystem setupTTypeSystem(TypeSystems kind) {
    if (kind == EqTwoTypes) {
      TTypeSystem m = new TTypeSystem(mSrc.tsm, kind);
      m.cas = mSrc.cas;
      // m.ts = mSrc.cas.getTypeSystemImpl();
      return mSrc;
    }
    CASTestSetup setup = new CASTestSetup(kind);
    CASImpl cas = (CASImpl) CASInitializer.initCas(setup, null);
    TTypeSystem m = setup.m;
    m.cas = cas;
    m.updateAfterCommit();
    return m;
  }

  public void setUp() {
    mSrc = setupTTypeSystem(TwoTypes);
    casSrc = mSrc.cas;
    final TypeSystems[] tss = TypeSystems.values();
    final int nbrAltTs = tss.length;
    alternateTTypeSystems = new TTypeSystem[nbrAltTs];
    for (int i = 0; i < nbrAltTs; i++) {
      alternateTTypeSystems[i] = setupTTypeSystem(tss[i]);
    }
    lfs = new ArrayList<FeatureStructure>();
  }

  public void tearDown() {
    remoteCas = null;
    lfs = null;
    mSrc = null;
    casSrc = null;
    alternateTTypeSystems = null;
    // don't reset random
  }

  // void setupTgtTs(String kind) {
  // CASTestSetup setupTgt = new CASTestSetup(kind);
  // casTgt = (CASImpl) CASInitializer.initCas(setupTgt);
  // mTgt = setupTgt.m;
  // mTgt.ts = casTgt.getTypeSystemImpl();
  // }

  TTypeSystem getTT(TypeSystems kind) {
    return alternateTTypeSystems[kind.ordinal()];
  }

  CASImpl setupCas(TTypeSystem m) {
    return createCAS(m.ts);
  }

  CASImpl createCAS(TypeSystemImpl ts) {
    try {
      return (CASImpl) CasCreationUtils.createCas(ts, null, null, null, null);
    } catch (ResourceInitializationException e) {
      throw new RuntimeException(e);
    }
  }

  public void testDocText() {
    try {
      CAS cas = CasCreationUtils.createCas((TypeSystemDescription) null, null, null);
      cas.setDocumentLanguage("latin");
      cas.setDocumentText("test");

      ByteArrayOutputStream baos = new ByteArrayOutputStream(1024);

      Serialization.serializeWithCompression(cas, baos, cas.getTypeSystem());

      CAS cas2 = CasCreationUtils.createCas((TypeSystemDescription) null, null, null);
      ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
      Serialization.deserializeCAS(cas2, bais);

      assertEquals("latin", cas2.getDocumentLanguage());
      assertEquals("test", cas2.getDocumentText());
    } catch (Exception e) {
      throw new RuntimeException(e);
    }
  }

  public void testDocumentText() {
    // serdesSimple(getTT(EqTwoTypes));
    remoteCas = setupCas(getTT(EqTwoTypes));
    casSrc.reset();
    casSrc.setDocumentText(testDocText);
    loadCas(casSrc, mSrc);
    verify(remoteCas, "DocumentText");
    assertEquals(remoteCas.getDocumentText(), testDocText);

    // test case where serialization is done without type filtering,
    // and deserialization is done with filtering
    remoteCas.reset();
    verifyDeserFilter(remoteCas, "WithDeserFilterDocumentText");
    assertEquals(remoteCas.getDocumentText(), testDocText);
  }

  /**
   * Make one of each kind of artifact, including arrays
   * serialize to byte stream, deserialize into new cas, compare
   */

  public void testAllKinds() {
    if (doPlain) {
      serdesSimple(getTT(EqTwoTypes), "EqTwoTypes");
    } else {
      int i = 0;
      for (TTypeSystem m : alternateTTypeSystems) {
       
        switch (m.kind) {
        // note: case statements *not* grouped in order to faclitate debugging
        case OneTypeSubsetFeatures:
          serdesSimple(m, "OneTypeSubsetFeatures" + m.kind.toString());
          break;
        case TwoTypesSubsetFeatures:
          serdesSimple(m, "TwoTypesSubsetFeatures" + m.kind.toString());
          break;
        case TwoTypes:
          i++;
          serdesSimple(m, "OtherAllKinds" + m.kind.toString() + Integer.toString(i));
          break;
        case EqTwoTypes:
        case OneType:
        case TwoTypesNoFeatures:
          serdesSimple(m, "OtherAllKinds" + m.kind.toString());
          break;
        }
      }
    }
  }

  // Test chains going through filtered type
  // Repeat below with OneType, and TwoTypes with filtered slot == fsRef

  // T1 fsArray ref -> T2 -> T1 (new) (not indexed)
  // T1 ref -> T2 -> T1 (new) (not indexed)
  // T1 fsArray ref -> T2 -> T1 (new) (indexed)
  // T1 ref -> T2 -> T1 (new) (indexed)

  public void testRefThroughFilteredType() {
    reftft(OneType, 0);
    for (int i = 0; i < 10; i++) {
      reftft(TwoTypesSubsetFeatures, i);
    }
    reftft(TwoTypesNoFeatures, 0);
  }

  private void reftft(TypeSystems tskind, int i) {
    reftft(tskind, true, i);
    reftft(tskind, false, i);
  }

  /**
   * Inner part of test of refs through filtered type
   * 
   * @param tskind -
   * @param indexed -
   */
  private void reftft(TypeSystems tskind, boolean indexed, int i) {
    lfs.clear();

    TTypeSystem m = getTT(tskind);
    remoteCas = setupCas(m);
    // casSrc.reset();
    makeFeaturesForAkof(casSrc, mSrc, Akof1);

    FeatureStructure otherTsFs = casSrc.createFS(mSrc.getType(Akof2));
    FeatureStructure fsOrig = lfs.get(0);
    fsOrig.setFeatureValue(mSrc.getFeature(fsOrig, "Fs"), otherTsFs);

    FeatureStructure ts1Fs = casSrc.createFS(mSrc.getType(Akof1));
    otherTsFs.setFeatureValue(mSrc.getFeature(otherTsFs, "Fs"), ts1Fs);

    if (indexed) {
      casSrc.addFsToIndexes(ts1Fs);
    }

    verify(remoteCas, "refThroughFilteredType" 
       + (indexed ? "Indexed" : "NotIndexed") 
       + tskind.toString() + Integer.toString(i));
    
  }

  // broken out special instances of random tests
  public void testDeltaWithStringArrayMod() {
    // casSrc -> remoteCas,remoteCas updated, serialized back to srcCas
    for (int i = 0; i < 10; i++) {
      TTypeSystem m = getTT(EqTwoTypes);
      remoteCas = setupCas(m);
      loadCas(casSrc, mSrc);
      ReuseInfo[] ri = serializeDeserialize(casSrc, remoteCas, null, null);
      MarkerImpl marker = (MarkerImpl) remoteCas.createMarker();
      lfs = getIndexedFSs(remoteCas, m);
      FeatureStructure fs = lfs.get(10);
      StringArrayFS sa = (StringArrayFS) maybeGetFeatureKind(fs, m, "Astring");
      if (sa == null) { // could happen because features are randomly omitted
        System.out.println("    Astring feature omitted, retrying");
      } else if (sa.size() == 0) {
        System.out.println("    Astring feature array has 0 length, retrying");
      } else {
        sa.set(0, "change2");
        verifyDelta(marker, ri);
        break;
      }
      // setRandom();
      setUp();
      long seed = random.nextLong();
      random.setSeed(seed);
      System.out.println(" testDelta w/ String array mod random = " + seed + ", i = " + i);
    }
  }

  public void testDeltaWithDblArrayMod() {
    for (int i = 0; i < 10; i++) {
      TTypeSystem m = getTT(EqTwoTypes);
      remoteCas = setupCas(m);
      loadCas(casSrc, mSrc);
      ReuseInfo[] ri = serializeDeserialize(casSrc, remoteCas, null, null);
      MarkerImpl marker = (MarkerImpl) remoteCas.createMarker();
      lfs = getIndexedFSs(remoteCas, m);
      FeatureStructure fs = lfs.get(10); /* has double array length 2 */
      DoubleArrayFS d = (DoubleArrayFS) maybeGetFeatureKind(fs, m, "Adouble");
      if (d == null) { // could happen because features are randomly omitted
        System.out.println("    Adouble feature omitted, retrying");
      } else if (d.size() == 0) {
        System.out.println("    Adouble feature array has 0 length, retrying");
      } else {
        d.set(0, 12.34D);
        verifyDelta(marker, ri);
        break;
      }
      // setRandom();
      setUp();
      long seed = random.nextLong();
      random.setSeed(seed);
      System.out.println(" testDelta w/ dbl array mod random = " + seed + ", i = " + i);
    }
  }

  public void testDeltaWithByteArrayMod() {
    for (int i = 0; i < 10; i++) {
      TTypeSystem m = getTT(EqTwoTypes);
      remoteCas = setupCas(m);
      loadCas(casSrc, mSrc);
      ReuseInfo[] ri = serializeDeserialize(casSrc, remoteCas, null, null);
      MarkerImpl marker = (MarkerImpl) remoteCas.createMarker();

      lfs = getIndexedFSs(remoteCas, m); // gets FSs below the line

      FeatureStructure fs = lfs.get(10);
      ByteArrayFS sfs = (ByteArrayFS) maybeGetFeatureKind(fs, m, "Abyte");
      if (sfs == null) { // could happen because features are randomly omitted
        System.out.println("    Abyte feature omitted, retrying");
      } else if (sfs.size() == 0) {
        System.out.println("    Abyte feature array has 0 length, retrying");
      } else {
        sfs.set(0, (byte) 21);
        verifyDelta(marker, ri);
        break;
      }
      // setRandom(); // retry with different random number
      setUp();
      System.out.println("  testDelta w byte array mod retrying, i = " + i);
    }
  }

  public void testDeltaWithStrArrayMod() {
    TTypeSystem m = getTT(EqTwoTypes);
    remoteCas = setupCas(m);
    loadCas(casSrc, mSrc);
    ReuseInfo[] ri = serializeDeserialize(casSrc, remoteCas, null, null);
    MarkerImpl marker = (MarkerImpl) remoteCas.createMarker();

    lfs = getIndexedFSs(remoteCas, m);

    FeatureStructure fs = lfs.get(10);
    StringArrayFS sfs = (StringArrayFS) maybeGetFeatureKind(fs, m, "Astring");
    if (sfs != null) {
      sfs.set(0, "change");
    }

    verifyDelta(marker, ri);
  }

  private void serdesSimple(TTypeSystem m, String kind) {
    remoteCas = setupCas(m);
    casSrc.reset();
    loadCas(casSrc, mSrc);
    verify(remoteCas, kind);

    // test case where serialization is done without type filtering,
    // and deserialization is done with filtering
    remoteCas.reset();
    verifyDeserFilter(remoteCas, "WithDeserFilter" + kind);
  }

  /**
   * 1) create a base cas with some data
   * 2) serialize it out and then back into a remoteCas
   *    This is needed to get the proper ordering in the remote cas of the 
   *    feature structures - they will be ordered by their fs addr, but omitting
   *    "skipped" FSs (not in the index or reachable, and not in the target ts)
   * 2) create the mark in the remoteCAS: cas.createMarker()
   * 3) add more cas data to the remoteCas
   * 4) serialize with marker
   * 5) deserialize back into base cas
   * 6) check resulting base cas = remote cas
   * 
   * 
   */
  public void testDelta() {
    if (doPlain) {
      serdesDelta(getTT(EqTwoTypes));
    } else {
      for (TTypeSystem m : alternateTTypeSystems) {
        switch (m.kind) {
        case TwoTypesSubsetFeatures:
          serdesDelta(m);
          break;
        case OneTypeSubsetFeatures:
          serdesDelta(m);
          break;
        case TwoTypes:
        case EqTwoTypes:
        case OneType:
        case TwoTypesNoFeatures:
          serdesDelta(m);
          break;
        }
      }
    }
  }

  private void serdesDelta(TTypeSystem m) {
    remoteCas = setupCas(m); // create empty new CAS with specified type system from m.ts
//    casSrc.reset();
    loadCas(casSrc, mSrc); // load up the src cas using mSrc spec
    // src -> serialize -> deserialize -> rmt
    ReuseInfo[] ri = serializeDeserialize(casSrc, remoteCas, null, null); 

    MarkerImpl marker = (MarkerImpl) remoteCas.createMarker();
    loadCas(remoteCas, m); // load some changes into remote
    // rmt -> serialize(full ts) -> deserialize(2 ts) -> src, then compare src & rmt
    verifyDelta(marker, ri); 
  }

  public void testDeltaWithRefsBelow() {
    lfs.clear();
    TTypeSystem m = getTT(EqTwoTypes);
    remoteCas = setupCas(m);
    loadCas(casSrc, mSrc);
    ReuseInfo ri[] = serializeDeserialize(casSrc, remoteCas, null, null);
    MarkerImpl marker = (MarkerImpl) remoteCas.createMarker();

    lfs = getIndexedFSs(remoteCas, m);
    FeatureStructure fs = remoteCas.createFS(m.getType(Akof1));
    maybeSetFeature(fs, m, lfs.get(0));
    ArrayFS fsafs = remoteCas.createArrayFS(4);
    fsafs.set(1, lfs.get(1));
    fsafs.set(2, lfs.get(2));
    fsafs.set(3, lfs.get(3));
    maybeSetFeatureKind(fs, m, "Afs", fsafs);

    verifyDelta(marker, ri);
  }

  public void testDeltaWithMods() {
    lfs.clear();
    TTypeSystem m = getTT(EqTwoTypes);
    remoteCas = setupCas(m);
    loadCas(casSrc, mSrc);
    ReuseInfo ri[] = serializeDeserialize(casSrc, remoteCas, null, null);
    MarkerImpl marker = (MarkerImpl) remoteCas.createMarker();

    lfs = getIndexedFSs(remoteCas, m); // get list of all "Akof1" FS
    FeatureStructure fs = remoteCas.createFS(m.getType(Akof1));
    // set the lfs.get(0) featurestructure's feature "Fs" to the new fs
    maybeSetFeatureKind(lfs.get(0), m, "Fs", fs); 

    verifyDelta(marker, ri);
  }

  /**
   * Variations to cover:
   *   all kinds of slots
   *   multiple sets of values
   *     test diffs
   * multiple orders (try reverse and random order)
   * 
   * Driver for random values
   *   pick among random and "interesting" edge case values
   * 
   * 
   */
  public void testDeltaWithAllMods() {

    for (int i = 0; i < 100; i++) {
      checkDeltaWithAllMods();
      tearDown();
      setUp();
    }
  }

  public void checkDeltaWithAllMods() {
    makeRandomFss(casSrc, mSrc, Akof1, 7);
    TTypeSystem m = getTT(EqTwoTypes);
    remoteCas = setupCas(m);
    loadCas(casSrc, mSrc);
    ReuseInfo ri[] = serializeDeserialize(casSrc, remoteCas, null, null);
    MarkerImpl marker = (MarkerImpl) remoteCas.createMarker();

    lfs = getIndexedFSs(remoteCas, m);

    makeRandomFss(remoteCas, m, Akof1, 8);

    int i = 0;
    for (FeatureStructure fs : lfs) {
      if (((i++) % 2) == 0) {
        maybeSetFeature(fs, m, lfs.get(random.nextInt(lfs.size())));
      }
    }

    makeRandomUpdatesBelowMark(remoteCas, m, Akof1);

    verifyDelta(marker, ri);

  }

  public void testDeltaWithIndexMods() {
    TTypeSystem m = getTT(EqTwoTypes);
    remoteCas = setupCas(m);
    loadCas(casSrc, mSrc);
    ReuseInfo ri[] = serializeDeserialize(casSrc, remoteCas, null, null);
    MarkerImpl marker = (MarkerImpl) remoteCas.createMarker();

    lfs = new ArrayList<FeatureStructure>();
    loadCas(remoteCas, m);

    List<FeatureStructure> lfs2 = getIndexedFSs(remoteCas, m);

    remoteCas.getIndexRepository().removeFS(lfs2.get(0));
    remoteCas.getIndexRepository().removeFS(lfs2.get(1));
    remoteCas.getIndexRepository().addFS(lfs2.get(1));  // should appear as reindexed

    remoteCas.getIndexRepository().removeFS(lfs.get(0));
    remoteCas.getIndexRepository().removeFS(lfs.get(1));
    remoteCas.getIndexRepository().addFS(lfs.get(1));

    verifyDelta(marker, ri);
  }

  public void testWithOtherSerializer() {
    doPlain = true;
    testDeltaWithMods();
    tearDown(); setUp();
    testDeltaWithRefsBelow();
    tearDown(); setUp();
    // testDeltaWithAllMods();
    tearDown(); setUp();
    testAllKinds();
    tearDown(); setUp();
    testArrayAux();
  }

  private void runCaptureSet() {
    //  Java 8 style
//    setupRunTeardown(this::testDocText);
//    setupRunTeardown(this::testDocumentText);
//    setupRunTeardown(this::testAllKinds);
//    setupRunTeardown(this::testRefThroughFilteredType);
//    setupRunTeardown(this::testDeltaWithStringArrayMod);
//    setupRunTeardown(this::testDeltaWithDblArrayMod);
//    setupRunTeardown(this::testDeltaWithByteArrayMod);
//    setupRunTeardown(this::testDeltaWithStrArrayMod);
//    setupRunTeardown(this::testDelta);
//    setupRunTeardown(this::testDeltaWithRefsBelow);
//    setupRunTeardown(this::testDeltaWithMods);
//    setupRunTeardown(this::testDeltaWithIndexMods);
//    setupRunTeardown(this::testArrayAux);
    
    // Java 7 style
    setupRunTeardown(new Runnable() {public void run() {testDocText();}});     
    setupRunTeardown(new Runnable() {public void run() {testDocumentText();}});
    setupRunTeardown(new Runnable() {public void run() {testAllKinds();}});
    setupRunTeardown(new Runnable() {public void run() {testRefThroughFilteredType();}});
    setupRunTeardown(new Runnable() {public void run() {testDeltaWithStringArrayMod();}});
    setupRunTeardown(new Runnable() {public void run() {testDeltaWithDblArrayMod();}});
    setupRunTeardown(new Runnable() {public void run() {testDeltaWithByteArrayMod();}});
    setupRunTeardown(new Runnable() {public void run() {testDeltaWithStrArrayMod();}});
    setupRunTeardown(new Runnable() {public void run() {testDelta();}});
    setupRunTeardown(new Runnable() {public void run() {testDeltaWithRefsBelow();}});
    setupRunTeardown(new Runnable() {public void run() {testDeltaWithMods();}});
    setupRunTeardown(new Runnable() {public void run() {testDeltaWithIndexMods();}});
    setupRunTeardown(new Runnable() {public void run() {testArrayAux();}});

  }

  // change tst to test to run this as a looper to catch infrequent errors
  public void tstLoop() {
    Random seeder = new Random();
    for (int i = 0; i < 800; i++) {
      long seed = seeder.nextLong();
//          -670766016644278339L;  20, 33 IntegerArray miscompare
//      -5294919707375572185L;   -28, 33  IntegerArray miscompare
//      7169526779687013172L;   // ref miscompare
      System.out.println("i: " + i + "  seed: " + seed);
      random.setSeed(seed);
      runCaptureSet();
    }
  }
  
  public void captureGenerated() {
    capture = true;
    initWriteSavedInts();
    runCaptureSet();
    try {
      savedIntsOutStream.close();
    } catch (IOException e) {
      throw new RuntimeException(e);
    }
    capture = false;
{'fraction_non_alphanumeric': 0.12076743301266374, 'fraction_numerical': 0.007065601391379966, 'mean_word_length': 3.113727828290356, 'pattern_counts': {'":': 0, '<': 36, '<?xml version=': 0, '>': 51, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '375309', 'n_tokens_mistral': 18318, 'n_tokens_neox': 17561, 'n_words': 4288}