<commit_before>//
//   Copyright 2018  <NAME>.A.S.
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.
//

package io.warp10.script.functions;

import java.util.HashMap;
import java.util.Map;

import org.apache.thrift.TDeserializer;
import org.apache.thrift.TException;
import org.apache.thrift.protocol.TCompactProtocol;

import com.google.common.base.Charsets;

import io.warp10.continuum.gts.GTSDecoder;
import io.warp10.continuum.gts.GTSEncoder;
import io.warp10.continuum.gts.GTSHelper;
import io.warp10.continuum.gts.GTSWrapperHelper;
import io.warp10.continuum.gts.GeoTimeSerie;
import io.warp10.continuum.store.thrift.data.GTSWrapper;
import io.warp10.crypto.OrderPreservingBase64;
import io.warp10.script.NamedWarpScriptFunction;
import io.warp10.script.WarpScriptException;
import io.warp10.script.WarpScriptStack;
import io.warp10.script.WarpScriptStackFunction;

/**
 * Converts an encoder into a map of gts, one per type
 */
public class TOGTS extends NamedWarpScriptFunction implements WarpScriptStackFunction {
  
  public TOGTS(String name) {
    super(name);
  }  

  @Override
  public Object apply(WarpScriptStack stack) throws WarpScriptException {
    Object top = stack.pop();
    
    if (!(top instanceof String) && !(top instanceof byte[]) && !(top instanceof GTSEncoder)) {
      throw new WarpScriptException(getName() + " operates on a string, byte array or encoder.");
    }
    
    Map<String,GeoTimeSerie> series = new HashMap<String,GeoTimeSerie>();
    
    GTSDecoder decoder;
    
    if (top instanceof GTSEncoder) {
      decoder = ((GTSEncoder) top).getUnsafeDecoder(false);
    } else {
      try {
        byte[] bytes = top instanceof String ? OrderPreservingBase64.decode(top.toString().getBytes(Charsets.US_ASCII)) : (byte[]) top;
        
        TDeserializer deser = new TDeserializer(new TCompactProtocol.Factory());
        
        GTSWrapper wrapper = new GTSWrapper();
        
        deser.deserialize(wrapper, bytes);

        decoder = GTSWrapperHelper.fromGTSWrapperToGTSDecoder(wrapper);        
      } catch (TException te) {
        throw new WarpScriptException(getName() + " failed to unwrap encoder.", te);
      }            
    }

    GeoTimeSerie gts;
    
    while(decoder.next()) {
      // We are populating a GTS, so we do not call getBinaryValue
      Object value = decoder.getValue();
      
      String type = "DOUBLE";
      
      if (value instanceof String) {
        type = "STRING";
      } else if (value instanceof Boolean) {
        type = "BOOLEAN";
      } else if (value instanceof Long) {
        type = "LONG";
      } else {
        type = "DOUBLE";
      }
      
      gts = series.get(type);<commit_msg>Added support for binary values<commit_after>//
//   Copyright 2018  SenX S.A.S.
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.
//

package io.warp10.script.functions;

import java.math.BigDecimal;
import java.util.HashMap;
import java.util.Map;

import org.apache.thrift.TDeserializer;
import org.apache.thrift.TException;
import org.apache.thrift.protocol.TCompactProtocol;

import com.google.common.base.Charsets;

import io.warp10.continuum.gts.GTSDecoder;
import io.warp10.continuum.gts.GTSEncoder;
import io.warp10.continuum.gts.GTSHelper;
import io.warp10.continuum.gts.GTSWrapperHelper;
import io.warp10.continuum.gts.GeoTimeSerie;
import io.warp10.continuum.store.thrift.data.GTSWrapper;
import io.warp10.crypto.OrderPreservingBase64;
import io.warp10.script.NamedWarpScriptFunction;
import io.warp10.script.WarpScriptException;
import io.warp10.script.WarpScriptStack;
import io.warp10.script.WarpScriptStackFunction;

/**
 * Converts an encoder into a map of gts, one per type
 */
public class TOGTS extends NamedWarpScriptFunction implements WarpScriptStackFunction {
  
  public TOGTS(String name) {
    super(name);
  }  

  @Override
  public Object apply(WarpScriptStack stack) throws WarpScriptException {
    Object top = stack.pop();
    
    if (!(top instanceof String) && !(top instanceof byte[]) && !(top instanceof GTSEncoder)) {
      throw new WarpScriptException(getName() + " operates on a string, byte array or encoder.");
    }
    
    Map<String,GeoTimeSerie> series = new HashMap<String,GeoTimeSerie>();
    
    GTSDecoder decoder;
    
    if (top instanceof GTSEncoder) {
      decoder = ((GTSEncoder) top).getUnsafeDecoder(false);
    } else {
      try {
        byte[] bytes = top instanceof String ? OrderPreservingBase64.decode(top.toString().getBytes(Charsets.US_ASCII)) : (byte[]) top;
        
        TDeserializer deser = new TDeserializer(new TCompactProtocol.Factory());
        
        GTSWrapper wrapper = new GTSWrapper();
        
        deser.deserialize(wrapper, bytes);

        decoder = GTSWrapperHelper.fromGTSWrapperToGTSDecoder(wrapper);        
      } catch (TException te) {
        throw new WarpScriptException(getName() + " failed to unwrap encoder.", te);
      }            
    }

    GeoTimeSerie gts;
    
    while(decoder.next()) {
      Object value = decoder.getBinaryValue();
      
      String type = "DOUBLE";
      
      if (value instanceof String) {
        type = "STRING";
      } else if (value instanceof Boolean) {
        type = "BOOLEAN";
      } else if (value instanceof Long) {
        type = "LONG";
      } else if (value instanceof Double || value instanceof BigDecimal) { 
        type = "DOUBLE";
      } else if (value instanceof byte[]) {
        type = "BINARY";
      }
      
      gts = series.get(type);
      
{'fraction_non_alphanumeric': 0.10297029702970296, 'fraction_numerical': 0.010967250571210967, 'mean_word_length': 3.038130381303813, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2401509', 'n_tokens_mistral': 2019, 'n_tokens_neox': 1983, 'n_words': 535}