<commit_before>package roart.aggregate;

import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.math3.util.Pair;

import com.fasterxml.jackson.databind.ObjectMapper;

import roart.config.MyConfig;
import roart.config.MyMyConfig;
import roart.indicator.IndicatorUtils;
import roart.model.MetaItem;
import roart.model.StockItem;
import roart.pipeline.Pipeline;
import roart.pipeline.PipelineConstants;
import roart.util.ArraysUtil;
import roart.util.Constants;
import roart.util.MarketData;
import roart.util.MetaDao;
import roart.util.StockDao;
import roart.util.StockUtil;

public class ExtraReader extends Pipeline {

    Map<Pair, List<StockItem>> pairStockMap;
    Map<Pair, Map<Date, StockItem>> pairDateMap;
    Map<Pair, String> pairCatMap;
    Map<Pair, Double[][]> pairListMap;
    Map<Pair, List<Date>> pairDateListMap;
    Map<Pair, double[][]> pairTruncListMap;
    
    public ExtraReader(MyMyConfig conf, int category) throws Exception {
        super(conf, category);
        readData(conf, null, category);        
        
    }
    private void readData(MyMyConfig conf, Map<String, MarketData> marketdatamap, int category) throws Exception {
        pairListMap = new HashMap<>();
        pairDateMap = new HashMap<>();
        pairCatMap = new HashMap<>();
        pairStockMap = new HashMap<>();
        pairDateListMap = new HashMap<>();
        pairTruncListMap = new HashMap<>();
        
        String str0 = conf.getAggregatorsIndicatorExtras();
        //String str = "{ [ 'market' : 'cboevol', 'id' : 'VIX'], [ 'market' : 'tradcomm' , 'id' : 'CL1:COM' ], [ 'market' : 'tradcomm', 'id' : 'XAUUSD:CUR' ] ]   }";
        String str = "[ { \"market\" : \"cboevol\", \"id\" : \"VIX\", \"category\" : \"Index\"}, { \"market\" : \"tradcomm\" , \"id\" : \"CL1:COM\" , \"category\" : \"Price\" }, { \"market\" : \"tradcomm\", \"id\" : \"XAUUSD:CUR\" , \"category\" : \"Price\" } ]";
        //str = "";
        str = str0;
        if (str == null || str.isEmpty()) {
            return;
        }
        ObjectMapper mapper = new ObjectMapper();
        List<LinkedHashMap> list = (List<LinkedHashMap>) mapper.readValue(str, List.class);
        List<Pair> pairs = new ArrayList<>();
        for (LinkedHashMap mi : list) {
            System.out.println("mi"+mi+" " + mi.toString());
            Pair pair = new Pair(mi.get("market"), mi.get("id"));
            String cat = (String) mi.get("category");
            pairs.add(pair);
            pairCatMap.put(pair, cat);
            //pairs.add(new Pair(mi.getMarket(), mi.getId()));
        }

        SimpleDateFormat dt = new SimpleDateFormat(Constants.MYDATEFORMAT);
        String dateme = dt.format(conf.getdate());
        for (Pair pair : pairs) {
            String market = (String) pair.getFirst();
            List<StockItem> stocks = null;
            try {
                stocks = StockItem.getAll(market, conf);
            } catch (Exception e) {
                log.error(Constants.EXCEPTION, e);
            }
            log.info("stocks " + stocks.size());
            if (stocks == null) {
                continue;
            }
            List<StockItem> stocksId = new ArrayList<>();
            for (StockItem stock : stocks) {
                String id = (String) pair.getSecond();
                if (stock.getId().equals(id)) {
                    stocksId.add(stock);
                }
            }
            stocks = null;

            Map<String, List<StockItem>> stockdatemap = StockUtil.splitDate(stocksId);

            List<StockItem> mystocks = new ArrayList<>();
            Map<Date, StockItem> mymap = new HashMap<>();
            String id = (String) pair.getSecond();
            for (StockItem stock : stocksId) {
                mystocks.add(stock);
                mymap.put(stock.getDate(), stock);
            }
            List<StockItem> datedstocklists[] = StockUtil.getDatedstocklists(stockdatemap, conf.getdate(), conf.getDays(), conf.getTableIntervalDays());
            pairStockMap.put(pair, mystocks);
            pairDateMap.put(pair, mymap);<commit_msg>Sonar.<commit_after>package roart.aggregate;

import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.math3.util.Pair;

import com.fasterxml.jackson.databind.ObjectMapper;

import roart.config.MyConfig;
import roart.config.MyMyConfig;
import roart.indicator.IndicatorUtils;
import roart.model.MetaItem;
import roart.model.StockItem;
import roart.pipeline.Pipeline;
import roart.pipeline.PipelineConstants;
import roart.util.ArraysUtil;
import roart.util.Constants;
import roart.util.MarketData;
import roart.util.MetaDao;
import roart.util.StockDao;
import roart.util.StockUtil;

public class ExtraReader extends Pipeline {

    Map<Pair, List<StockItem>> pairStockMap;
    Map<Pair, Map<Date, StockItem>> pairDateMap;
    Map<Pair, String> pairCatMap;
    Map<Pair, Double[][]> pairListMap;
    Map<Pair, List<Date>> pairDateListMap;
    Map<Pair, double[][]> pairTruncListMap;
    
    public ExtraReader(MyMyConfig conf, int category) throws Exception {
        super(conf, category);
        readData(conf, null, category);        
        
    }
    private void readData(MyMyConfig conf, Map<String, MarketData> marketdatamap, int category) throws Exception {
        pairListMap = new HashMap<>();
        pairDateMap = new HashMap<>();
        pairCatMap = new HashMap<>();
        pairStockMap = new HashMap<>();
        pairDateListMap = new HashMap<>();
        pairTruncListMap = new HashMap<>();
        
        String str0 = conf.getAggregatorsIndicatorExtras();
        //String str = "{ [ 'market' : 'cboevol', 'id' : 'VIX'], [ 'market' : 'tradcomm' , 'id' : 'CL1:COM' ], [ 'market' : 'tradcomm', 'id' : 'XAUUSD:CUR' ] ]   }";
        String str = "[ { \"market\" : \"cboevol\", \"id\" : \"VIX\", \"category\" : \"Index\"}, { \"market\" : \"tradcomm\" , \"id\" : \"CL1:COM\" , \"category\" : \"Price\" }, { \"market\" : \"tradcomm\", \"id\" : \"XAUUSD:CUR\" , \"category\" : \"Price\" } ]";
        //str = "";
        str = str0;
        if (str == null || str.isEmpty()) {
            return;
        }
        ObjectMapper mapper = new ObjectMapper();
        List<LinkedHashMap> list = (List<LinkedHashMap>) mapper.readValue(str, List.class);
        List<Pair> pairs = new ArrayList<>();
        for (LinkedHashMap mi : list) {
            System.out.println("mi"+mi+" " + mi.toString());
            Pair pair = new Pair(mi.get("market"), mi.get("id"));
            String cat = (String) mi.get("category");
            pairs.add(pair);
            pairCatMap.put(pair, cat);
            //pairs.add(new Pair(mi.getMarket(), mi.getId()));
        }

        SimpleDateFormat dt = new SimpleDateFormat(Constants.MYDATEFORMAT);
        String dateme = dt.format(conf.getdate());
        for (Pair pair : pairs) {
            String market = (String) pair.getFirst();
            List<StockItem> stocks = null;
            try {
                stocks = StockItem.getAll(market, conf);
                log.info("stocks " + stocks.size());
            } catch (Exception e) {
                log.error(Constants.EXCEPTION, e);
            }
            if (stocks == null) {
                continue;
            }
            List<StockItem> stocksId = new ArrayList<>();
            for (StockItem stock : stocks) {
                String id = (String) pair.getSecond();
                if (stock.getId().equals(id)) {
                    stocksId.add(stock);
                }
            }
            stocks = null;

            Map<String, List<StockItem>> stockdatemap = StockUtil.splitDate(stocksId);

            List<StockItem> mystocks = new ArrayList<>();
            Map<Date, StockItem> mymap = new HashMap<>();
            String id = (String) pair.getSecond();
            for (StockItem stock : stocksId) {
                mystocks.add(stock);
                mymap.put(stock.getDate(), stock);
            }
            List<StockItem> datedstocklists[] = StockUtil.getDatedstocklists(stockdatemap, conf.getdate(), conf.getDays(), conf.getTableIntervalDays());
            pairStockMap.put(pair, mystocks);
            pairDateMap.put(pair, mymap);
{'fraction_non_alphanumeric': 0.14743515850144093, 'fraction_numerical': 0.00138328530259366, 'mean_word_length': 2.4346793349168645, 'pattern_counts': {'":': 0, '<': 63, '<?xml version=': 0, '>': 63, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '973709', 'n_tokens_mistral': 2602, 'n_tokens_neox': 2553, 'n_words': 559}