<commit_before>package roart.etl;

import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import roart.common.config.MyMyConfig;
import roart.common.constants.Constants;
import roart.common.util.ArraysUtil;

public class ValueETL {

    protected static Logger log = LoggerFactory.getLogger(ValueETL.class);

    public static void zeroPrice(Map<String, Double[][]> aListMap, int category) {
        if (category == Constants.PRICECOLUMN || category == Constants.INDEXVALUECOLUMN) {
            for (Entry<String, Double[][]> entry : aListMap.entrySet()) {
                Double[][] value = entry.getValue();
                for(int i = 0; i < value[0].length; i++) {
                    if (value[0][i] != null && value[0][i] == 0) {
                        log.info("Value 0 for {}", entry.getKey());
                        value[0][i] = null;
                        value[1][i] = null;
                        value[2][i] = null;
                    }
                }
            }
        }
    }

    public static Map<String, Double[][]> getBase100D(Map<String, Double[][]> aListMap) {
        Map<String, Double[][]> aMap = new HashMap<>();
        for (Entry<String, Double[][]> entry : aListMap.entrySet()) {
            Double[][] value = entry.getValue();
            if (value != null) {
                Double[][] newValue = new Double[value.length][];
                for (int i = value.length - 1; i >= 0; i--) {
                    newValue[i] = ArraysUtil.getPercentizedPriceIndex(value[i], value[0]);
                }
                aMap.put(entry.getKey(), newValue);
            }
        }
        return aMap;
    }

    public static Map<String, double[][]> getBase100(Map<String, double[][]> aListMap) {
        Map<String, double[][]> aMap = new HashMap<>();
        for (Entry<String, double[][]> entry : aListMap.entrySet()) {
            double[][] value = entry.getValue();
            if (value != null) {
                double[][] newValue = new double[value.length][];
                double first = value.length > 0 && value[0].length > 0 ? value[0][0] : 0;
                for (int i = 0; i < value.length; i++) {
                    newValue[i] = ArraysUtil.getPercentizedPriceIndex(value[i], first);
                }
                aMap.put(entry.getKey(), newValue);
            }
        }
        return aMap;
    }

    public static Map<String, Double[][]> getReverseArrSparseFillHolesArr(MyMyConfig conf, Map<String, Double[][]> listMap) {
        Map<String, Double[][]> retMap = new HashMap<>();
        for (Entry<String, Double[][]> entry : listMap.entrySet()) {
            Double[][] array = entry.getValue();
            Double[][] newArray = new Double[array.length][];
            if ("F00000ZHEV".equals(entry.getKey())) {
                int jj = 0;<commit_msg>Check for first null for new ETL for pipeline (I210).<commit_after>package roart.etl;

import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import roart.common.config.MyMyConfig;
import roart.common.constants.Constants;
import roart.common.util.ArraysUtil;

public class ValueETL {

    protected static Logger log = LoggerFactory.getLogger(ValueETL.class);

    public static void zeroPrice(Map<String, Double[][]> aListMap, int category) {
        if (category == Constants.PRICECOLUMN || category == Constants.INDEXVALUECOLUMN) {
            for (Entry<String, Double[][]> entry : aListMap.entrySet()) {
                Double[][] value = entry.getValue();
                for(int i = 0; i < value[0].length; i++) {
                    if (value[0][i] != null && value[0][i] == 0) {
                        log.info("Value 0 for {}", entry.getKey());
                        value[0][i] = null;
                        value[1][i] = null;
                        value[2][i] = null;
                    }
                }
            }
        }
    }

    public static Map<String, Double[][]> getBase100D(Map<String, Double[][]> aListMap) {
        Map<String, Double[][]> aMap = new HashMap<>();
        for (Entry<String, Double[][]> entry : aListMap.entrySet()) {
            Double[][] value = entry.getValue();
            if (value != null) {
                Double[][] newValue = new Double[value.length][];
                for (int i = value.length - 1; i >= 0; i--) {
                    newValue[i] = ArraysUtil.getPercentizedPriceIndex(value[i], value[0]);
                }
                aMap.put(entry.getKey(), newValue);
            }
        }
        return aMap;
    }

    public static Map<String, double[][]> getBase100(Map<String, double[][]> aListMap) {
        Map<String, double[][]> aMap = new HashMap<>();
        for (Entry<String, double[][]> entry : aListMap.entrySet()) {
            double[][] value = entry.getValue();
            if (value != null) {
                double[][] newValue = new double[value.length][];
                double first = value.length > 0 && value[0].length > 0 ? value[0][0] : 0;
                if (first == 0) {
                    continue;
                }
                for (int i = 0; i < value.length; i++) {
                    newValue[i] = ArraysUtil.getPercentizedPriceIndex(value[i], first);
                }
                aMap.put(entry.getKey(), newValue);
            }
        }
        return aMap;
    }

    public static Map<String, Double[][]> getReverseArrSparseFillHolesArr(MyMyConfig conf, Map<String, Double[][]> listMap) {
        Map<String, Double[][]> retMap = new HashMap<>();
        for (Entry<String, Double[][]> entry : listMap.entrySet()) {
            Double[][] array = entry.getValue();
            Double[][] newArray = new Double[array.length][];
            if ("F00000ZHEV".equals(entry.getKey())) {
                int jj = 0;
{'fraction_non_alphanumeric': 0.15379426644182126, 'fraction_numerical': 0.011804384485666104, 'mean_word_length': 1.9536852589641434, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '816866', 'n_tokens_mistral': 1792, 'n_tokens_neox': 1722, 'n_words': 392}