<commit_before>/*
 * Copyright 2015-16, Yahoo! Inc.
 * Licensed under the terms of the Apache License 2.0. See LICENSE file at the project root for terms.
 */

package com.yahoo.sketches.theta;

import static com.yahoo.sketches.QuickSelect.selectExcludingZeros;
import static com.yahoo.sketches.theta.CompactSketch.compactCache;
import static com.yahoo.sketches.theta.CompactSketch.createCompactSketch;
import static com.yahoo.sketches.theta.PreambleUtil.COMPACT_FLAG_MASK;
import static com.yahoo.sketches.theta.PreambleUtil.FAMILY_BYTE;
import static com.yahoo.sketches.theta.PreambleUtil.FLAGS_BYTE;
import static com.yahoo.sketches.theta.PreambleUtil.LG_ARR_LONGS_BYTE;
import static com.yahoo.sketches.theta.PreambleUtil.MAX_THETA_LONG_AS_DOUBLE;
import static com.yahoo.sketches.theta.PreambleUtil.ORDERED_FLAG_MASK;
import static com.yahoo.sketches.theta.PreambleUtil.PREAMBLE_LONGS_BYTE;
import static com.yahoo.sketches.theta.PreambleUtil.RETAINED_ENTRIES_INT;
import static com.yahoo.sketches.theta.PreambleUtil.SEED_HASH_SHORT;
import static com.yahoo.sketches.theta.PreambleUtil.SER_VER_BYTE;
import static com.yahoo.sketches.theta.PreambleUtil.THETA_LONG;
import static com.yahoo.sketches.theta.PreambleUtil.UNION_THETA_LONG;
import static java.lang.Math.min;

import com.yahoo.sketches.Family;
import com.yahoo.sketches.HashOperations;
import com.yahoo.sketches.ResizeFactor;
import com.yahoo.sketches.SketchesArgumentException;
import com.yahoo.sketches.Util;
import com.yahoo.sketches.memory.Memory;
import com.yahoo.sketches.memory.NativeMemory;


/**
 * Shared code for the HeapUnion and DirectUnion implementations.
 * 
 * @author Lee Rhodes
 * @author <NAME>
 */
final class UnionImpl extends SetOperation implements Union {
  private UpdateSketch gadget_;
  private long unionThetaLong_;
  private short seedHash_;
  private Memory unionMem_;
  
  
  private UnionImpl(UpdateSketch gadget, long seed) {
    gadget_ = gadget;
    unionThetaLong_ = gadget.getThetaLong();
    seedHash_ = computeSeedHash(seed);
  }
  
  /**
   * Construct a new Union SetOperation on the java heap. 
   * Called by SetOperation.Builder.
   * 
   * @param lgNomLongs <a href="{@docRoot}/resources/dictionary.html#lgNomLogs">See lgNomLongs</a>
   * @param seed <a href="{@docRoot}/resources/dictionary.html#seed">See seed</a>
   * @param p <a href="{@docRoot}/resources/dictionary.html#p">See Sampling Probability, <i>p</i></a>
   * @param rf <a href="{@docRoot}/resources/dictionary.html#resizeFactor">See Resize Factor</a>
   * @return instance of this sketch
   */
  static UnionImpl initNewHeapInstance(int lgNomLongs, long seed, float p, ResizeFactor rf) {
    UpdateSketch gadget = HeapQuickSelectSketch.getInstance(lgNomLongs, seed, p, rf, true);
    UnionImpl unionImpl = new UnionImpl(gadget, seed);
    unionImpl.unionMem_ = null;
    return unionImpl;
  }
  
  /**
   * Heapify a Union from a Memory object containing data. 
   * Called by SetOperation.Builder.
   * @param srcMem The source Memory object.
   * <a href="{@docRoot}/resources/dictionary.html#mem">See Memory</a>
   * @param seed <a href="{@docRoot}/resources/dictionary.html#seed">See seed</a>
   * @return this class
   */
  static UnionImpl heapifyInstance(Memory srcMem, long seed) {
    Family.UNION.checkFamilyID(srcMem.getByte(FAMILY_BYTE));
    UpdateSketch gadget = HeapQuickSelectSketch.getInstance(srcMem, seed);
    UnionImpl unionImpl = new UnionImpl(gadget, seed);
    unionImpl.unionMem_ = null;
    return unionImpl;
  }
  
  /**
   * Construct a new Direct Union in the off-heap destination Memory. 
   * Called by SetOperation.Builder.
   * 
   * @param lgNomLongs <a href="{@docRoot}/resources/dictionary.html#lgNomLogs">See lgNomLongs</a>.
   * @param seed <a href="{@docRoot}/resources/dictionary.html#seed">See seed</a>
   * @param p <a href="{@docRoot}/resources/dictionary.html#p">See Sampling Probability, <i>p</i></a>
   * @param rf <a href="{@docRoot}/resources/dictionary.html#resizeFactor">See Resize Factor</a>
   * @param dstMem the given Memory object destination. It will be cleared prior to use.
   * @return this class
   */
  static UnionImpl 
      initNewDirectInstance(int lgNomLongs, long seed, float p, ResizeFactor rf, Memory dstMem) {
    UpdateSketch gadget = 
        DirectQuickSelectSketch.getInstance(lgNomLongs, seed, p, rf, dstMem, true);
    UnionImpl unionImpl = new UnionImpl(gadget, seed);
    unionImpl.unionMem_ = dstMem;
    return unionImpl;
  }
  
  /**
   * Wrap a Union object around a Union Memory object containing data. 
   * @param srcMem The source Memory object.
   * <a href="{@docRoot}/resources/dictionary.html#mem">See Memory</a>
   * @param seed <a href="{@docRoot}/resources/dictionary.html#seed">See seed</a>
   * @return this class
   */
  static UnionImpl wrapInstance(Memory srcMem, long seed) {
    Family.UNION.checkFamilyID(srcMem.getByte(FAMILY_BYTE));
    UpdateSketch gadget = DirectQuickSelectSketch.getInstance(srcMem, seed);
    UnionImpl unionImpl = new UnionImpl(gadget, seed);
    unionImpl.unionMem_ = srcMem;
    return unionImpl;
  }
  
  @Override
  public CompactSketch getResult(boolean dstOrdered, Memory dstMem) {
    int gadgetCurCount = gadget_.getRetainedEntries(true);
    long gadgetThetaLong = gadget_.getThetaLong();
    int arrLongs = 1 << gadget_.getLgArrLongs();
    int k = 1 << gadget_.getLgNomLongs();
    
    long[] gadgetCache = gadget_.getCache(); //if direct a copy, otherwise a reference
    long gNewThetaLong = gadgetThetaLong;
    
    if (gadgetCurCount > k) {
      if (!gadget_.isDirect()) {
        gadgetCache = new long[arrLongs];
        System.arraycopy(gadget_.getCache(), 0, gadgetCache, 0, arrLongs);
      }
      gNewThetaLong = selectExcludingZeros(gadgetCache, gadgetCurCount, k + 1);//messes the cache_ 
    }
    
    long thetaLongR = min(gNewThetaLong, unionThetaLong_);
    int curCountR = (thetaLongR < gadget_.getThetaLong()) 
        ? HashOperations.count(gadgetCache, thetaLongR)
        : gadgetCurCount;
    long[] compactCacheR = compactCache(gadgetCache, curCountR, thetaLongR, dstOrdered);
    boolean emptyR = (gadget_.isEmpty() 
        && (gadget_.getP() >= thetaLongR / MAX_THETA_LONG_AS_DOUBLE) && (curCountR == 0));
    
    return createCompactSketch(compactCacheR, emptyR, seedHash_, curCountR, thetaLongR, 
        dstOrdered, dstMem);
  }
  <commit_msg>add space around code<commit_after>/*
 * Copyright 2015-16, Yahoo! Inc.
 * Licensed under the terms of the Apache License 2.0. See LICENSE file at the project root for terms.
 */

package com.yahoo.sketches.theta;

import static com.yahoo.sketches.QuickSelect.selectExcludingZeros;
import static com.yahoo.sketches.theta.CompactSketch.compactCache;
import static com.yahoo.sketches.theta.CompactSketch.createCompactSketch;
import static com.yahoo.sketches.theta.PreambleUtil.COMPACT_FLAG_MASK;
import static com.yahoo.sketches.theta.PreambleUtil.FAMILY_BYTE;
import static com.yahoo.sketches.theta.PreambleUtil.FLAGS_BYTE;
import static com.yahoo.sketches.theta.PreambleUtil.LG_ARR_LONGS_BYTE;
import static com.yahoo.sketches.theta.PreambleUtil.MAX_THETA_LONG_AS_DOUBLE;
import static com.yahoo.sketches.theta.PreambleUtil.ORDERED_FLAG_MASK;
import static com.yahoo.sketches.theta.PreambleUtil.PREAMBLE_LONGS_BYTE;
import static com.yahoo.sketches.theta.PreambleUtil.RETAINED_ENTRIES_INT;
import static com.yahoo.sketches.theta.PreambleUtil.SEED_HASH_SHORT;
import static com.yahoo.sketches.theta.PreambleUtil.SER_VER_BYTE;
import static com.yahoo.sketches.theta.PreambleUtil.THETA_LONG;
import static com.yahoo.sketches.theta.PreambleUtil.UNION_THETA_LONG;
import static java.lang.Math.min;

import com.yahoo.sketches.Family;
import com.yahoo.sketches.HashOperations;
import com.yahoo.sketches.ResizeFactor;
import com.yahoo.sketches.SketchesArgumentException;
import com.yahoo.sketches.Util;
import com.yahoo.sketches.memory.Memory;
import com.yahoo.sketches.memory.NativeMemory;


/**
 * Shared code for the HeapUnion and DirectUnion implementations.
 * 
 * @author <NAME>
 * @author <NAME>
 */
final class UnionImpl extends SetOperation implements Union {
  private UpdateSketch gadget_;
  private long unionThetaLong_;
  private short seedHash_;
  private Memory unionMem_;
  
  
  private UnionImpl(UpdateSketch gadget, long seed) {
    gadget_ = gadget;
    unionThetaLong_ = gadget.getThetaLong();
    seedHash_ = computeSeedHash(seed);
  }
  
  /**
   * Construct a new Union SetOperation on the java heap. 
   * Called by SetOperation.Builder.
   * 
   * @param lgNomLongs <a href="{@docRoot}/resources/dictionary.html#lgNomLogs">See lgNomLongs</a>
   * @param seed <a href="{@docRoot}/resources/dictionary.html#seed">See seed</a>
   * @param p <a href="{@docRoot}/resources/dictionary.html#p">See Sampling Probability, <i>p</i></a>
   * @param rf <a href="{@docRoot}/resources/dictionary.html#resizeFactor">See Resize Factor</a>
   * @return instance of this sketch
   */
  static UnionImpl initNewHeapInstance(int lgNomLongs, long seed, float p, ResizeFactor rf) {
    UpdateSketch gadget = HeapQuickSelectSketch.getInstance(lgNomLongs, seed, p, rf, true);
    UnionImpl unionImpl = new UnionImpl(gadget, seed);
    unionImpl.unionMem_ = null;
    return unionImpl;
  }
  
  /**
   * Heapify a Union from a Memory object containing data. 
   * Called by SetOperation.Builder.
   * @param srcMem The source Memory object.
   * <a href="{@docRoot}/resources/dictionary.html#mem">See Memory</a>
   * @param seed <a href="{@docRoot}/resources/dictionary.html#seed">See seed</a>
   * @return this class
   */
  static UnionImpl heapifyInstance(Memory srcMem, long seed) {
    Family.UNION.checkFamilyID(srcMem.getByte(FAMILY_BYTE));
    UpdateSketch gadget = HeapQuickSelectSketch.getInstance(srcMem, seed);
    UnionImpl unionImpl = new UnionImpl(gadget, seed);
    unionImpl.unionMem_ = null;
    return unionImpl;
  }
  
  /**
   * Construct a new Direct Union in the off-heap destination Memory. 
   * Called by SetOperation.Builder.
   * 
   * @param lgNomLongs <a href="{@docRoot}/resources/dictionary.html#lgNomLogs">See lgNomLongs</a>.
   * @param seed <a href="{@docRoot}/resources/dictionary.html#seed">See seed</a>
   * @param p <a href="{@docRoot}/resources/dictionary.html#p">See Sampling Probability, <i>p</i></a>
   * @param rf <a href="{@docRoot}/resources/dictionary.html#resizeFactor">See Resize Factor</a>
   * @param dstMem the given Memory object destination. It will be cleared prior to use.
   * @return this class
   */
  static UnionImpl 
      initNewDirectInstance(int lgNomLongs, long seed, float p, ResizeFactor rf, Memory dstMem) {
    UpdateSketch gadget = 
        DirectQuickSelectSketch.getInstance(lgNomLongs, seed, p, rf, dstMem, true);
    UnionImpl unionImpl = new UnionImpl(gadget, seed);
    unionImpl.unionMem_ = dstMem;
    return unionImpl;
  }
  
  /**
   * Wrap a Union object around a Union Memory object containing data. 
   * @param srcMem The source Memory object.
   * <a href="{@docRoot}/resources/dictionary.html#mem">See Memory</a>
   * @param seed <a href="{@docRoot}/resources/dictionary.html#seed">See seed</a>
   * @return this class
   */
  static UnionImpl wrapInstance(Memory srcMem, long seed) {
    Family.UNION.checkFamilyID(srcMem.getByte(FAMILY_BYTE));
    UpdateSketch gadget = DirectQuickSelectSketch.getInstance(srcMem, seed);
    UnionImpl unionImpl = new UnionImpl(gadget, seed);
    unionImpl.unionMem_ = srcMem;
    return unionImpl;
  }
  
  @Override
  public CompactSketch getResult(boolean dstOrdered, Memory dstMem) {
    int gadgetCurCount = gadget_.getRetainedEntries(true);
    long gadgetThetaLong = gadget_.getThetaLong();
    int arrLongs = 1 << gadget_.getLgArrLongs();
    int k = 1 << gadget_.getLgNomLongs();
    long[] gadgetCache = gadget_.getCache(); //if direct a copy, otherwise a reference
    long gNewThetaLong = gadgetThetaLong;
    
    if (gadgetCurCount > k) {
      if (!gadget_.isDirect()) {
        gadgetCache = new long[arrLongs];
        System.arraycopy(gadget_.getCache(), 0, gadgetCache, 0, arrLongs);
      }
      gNewThetaLong = selectExcludingZeros(gadgetCache, gadgetCurCount, k + 1);//messes the cache_ 
    }
    
    long thetaLongR = min(gNewThetaLong, unionThetaLong_);
    
    int curCountR = (thetaLongR < gadget_.getThetaLong()) 
        ? HashOperations.count(gadgetCache, thetaLongR)
        : gadgetCurCount;
    
    long[] compactCacheR = compactCache(gadgetCache, curCountR, thetaLongR, dstOrdered);
    
    boolean emptyR = (gadget_.isEmpty() 
        && (gadget_.getP() >= thetaLongR / MAX_THETA_LONG_AS_DOUBLE) && (curCountR == 0));
    
    return createCompactSketch(compactCacheR, emptyR, seedHash_, curCountR, thetaLongR, 
        dstOrdered, dstMem);
  }
  
{'fraction_non_alphanumeric': 0.1093041438623925, 'fraction_numerical': 0.00218921032056294, 'mean_word_length': 4.278992983904251, 'pattern_counts': {'":': 0, '<': 72, '<?xml version=': 0, '>': 66, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3867059', 'n_tokens_mistral': 4350, 'n_tokens_neox': 4170, 'n_words': 926}