<commit_before>//
//   Copyright 2016  <NAME>
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.
//

package io.warp10.script;


import com.google.common.collect.Lists;
import org.boon.json.JsonFactory;
import org.boon.json.ObjectMapper;

import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;

/**
 * Export the FULL functions list slitted into these categories
 * Function
 * operator (comparaison / arithemtic / logical )
 * Constants
 * frameworks (bucketize / map / reduce / apply )
 * stack structures <% %> <S S> [] {}
 */
public class ExportFunctions {

  private final static String OP_ARITMETIC = "operator.artimetic";
  private final static String OP_LOGICAL = "operator.logical";
  private final static String OP_COMPARAISON = "operator.comparison";
  private final static String OP_BITWISE = "operator.bitwise";

  private final static String FCT_MATH = "functions.math";
  private final static String FCT_TIMEUNIT = "functions.timeunit";
  private final static String FCT_TRIGO = "functions.trigonometry";
  private final static String FCT_DATE = "functions.date";
  private final static String FCT_STRING = "functions.string";
  private final static String FCT_LIST = "functions.list";
  private final static String FCT_STACK = "functions.stack";
  private final static String FCT_GTS = "functions.gts";
  private final static String FCT_LOGIC_STRUCTURE = "functions.logicStructures";
  private final static String FCT_PLATFORM = "functions.platform";
  private final static String FCT_OUTLIER = "functions.outlier";
  private final static String FCT_BUCKETIZED  = "functions.bucketized";
  private final static String FCT_GEO  = "functions.geo";
  private final static String FCT_TYPE_CONVERSION  = "functions.typeConversion";
  private final static String FCT_MISC  = "functions.misc";

  private final static String SINGLE_VALUE_MAPPER = "mapper.singleValue";
  private final static String SLIDING_WINDOW_MAPPER = "mapper.slidingWindow";
  private final static String GEO_MAPPER = "mapper.geo";
  private final static String CUSTOM_MAPPER = "mapper.custom";

  private final static String DEFAULT_REDUCER = "reducer.default";
  private final static String CUSTOM_REDUCER = "reducer.custom";

  private final static String DEFAULT_FILTER = "filter.default";
  private final static String CUSTOM_FILTER = "filter.custom";

  private final static String DEFAULT_BUCKETIZER = "bucketizer.default";
  private final static String CUSTOM_BUCKETIZER = "bucketizer.custom";

  private final static String DEFAULT_OP = "op.default";

  // WARP SCRIPT FRAMEWORKS
  private final static String FMK_MAP = "MAP";
  private final static String FMK_REDUCE = "REDUCE";
  private final static String FMK_FILTER = "FILTER";
  private final static String FMK_APPLY = "APPLY";
  private final static String FMK_BUCKETIZE = "BUCKETIZE";

  public static void main(String[] args) throws Exception {
    List<String> functionsFullList = new ArrayList<>();
    try {
      // THIS is a very bad thing, used only for extraction purpose

      // Extract java functions
      Field field = JavaLibrary.class.getDeclaredField("functions");
      field.setAccessible(true);
      Map<String,Object> functions  = (Map<String,Object>) field.get(null);
      functionsFullList.addAll(functions.keySet());

      // Extract WarpScript functions
      Field warpScriptField = WarpScriptLib.class.getDeclaredField("functions");
      warpScriptField.setAccessible(true);
      functions  = (Map<String,Object>) warpScriptField.get(null);
      functionsFullList.addAll(functions.keySet());
    } catch (Exception exp) {
      exp.printStackTrace();
    }

    // -------------------------------------------------------------<commit_msg>Interrupt extraction if functions list is not readable<commit_after>//
//   Copyright 2016  Cityzen Data
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.
//

package io.warp10.script;


import com.google.common.collect.Lists;
import org.boon.json.JsonFactory;
import org.boon.json.ObjectMapper;

import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;

/**
 * Export the FULL functions list slitted into these categories
 * Function
 * operator (comparaison / arithemtic / logical )
 * Constants
 * frameworks (bucketize / map / reduce / apply )
 * stack structures <% %> <S S> [] {}
 */
public class ExportFunctions {

  private final static String OP_ARITMETIC = "operator.artimetic";
  private final static String OP_LOGICAL = "operator.logical";
  private final static String OP_COMPARAISON = "operator.comparison";
  private final static String OP_BITWISE = "operator.bitwise";

  private final static String FCT_MATH = "functions.math";
  private final static String FCT_TIMEUNIT = "functions.timeunit";
  private final static String FCT_TRIGO = "functions.trigonometry";
  private final static String FCT_DATE = "functions.date";
  private final static String FCT_STRING = "functions.string";
  private final static String FCT_LIST = "functions.list";
  private final static String FCT_STACK = "functions.stack";
  private final static String FCT_GTS = "functions.gts";
  private final static String FCT_LOGIC_STRUCTURE = "functions.logicStructures";
  private final static String FCT_PLATFORM = "functions.platform";
  private final static String FCT_OUTLIER = "functions.outlier";
  private final static String FCT_BUCKETIZED  = "functions.bucketized";
  private final static String FCT_GEO  = "functions.geo";
  private final static String FCT_TYPE_CONVERSION  = "functions.typeConversion";
  private final static String FCT_MISC  = "functions.misc";

  private final static String SINGLE_VALUE_MAPPER = "mapper.singleValue";
  private final static String SLIDING_WINDOW_MAPPER = "mapper.slidingWindow";
  private final static String GEO_MAPPER = "mapper.geo";
  private final static String CUSTOM_MAPPER = "mapper.custom";

  private final static String DEFAULT_REDUCER = "reducer.default";
  private final static String CUSTOM_REDUCER = "reducer.custom";

  private final static String DEFAULT_FILTER = "filter.default";
  private final static String CUSTOM_FILTER = "filter.custom";

  private final static String DEFAULT_BUCKETIZER = "bucketizer.default";
  private final static String CUSTOM_BUCKETIZER = "bucketizer.custom";

  private final static String DEFAULT_OP = "op.default";

  // WARP SCRIPT FRAMEWORKS
  private final static String FMK_MAP = "MAP";
  private final static String FMK_REDUCE = "REDUCE";
  private final static String FMK_FILTER = "FILTER";
  private final static String FMK_APPLY = "APPLY";
  private final static String FMK_BUCKETIZE = "BUCKETIZE";

  public static void main(String[] args) throws Exception {
    List<String> functionsFullList = new ArrayList<>();
    try {
      // THIS is a very bad thing, used only for extraction purpose

      // Extract java functions
      Field field = JavaLibrary.class.getDeclaredField("functions");
      field.setAccessible(true);
      Map<String,Object> functions  = (Map<String,Object>) field.get(null);
      functionsFullList.addAll(functions.keySet());

      // Extract WarpScript functions
      Field warpScriptField = WarpScriptLib.class.getDeclaredField("functions");
      warpScriptField.setAccessible(true);
      functions  = (Map<String,Object>) warpScriptField.get(null);
      functionsFullList.addAll(functions.keySet());
    } catch (Exception exp) {
      exp.printStackTrace();
      throw exp;
    }

    // -------------------------------------------------------------
{'fraction_non_alphanumeric': 0.10303514376996806, 'fraction_numerical': 0.0022820629849383844, 'mean_word_length': 3.910364145658263, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '825723', 'n_tokens_mistral': 2492, 'n_tokens_neox': 2425, 'n_words': 794}