<commit_before>package roart.db;

import roart.config.ConfigConstantMaps;
import roart.config.MyMyConfig;
import roart.config.MyXMLConfig;
import roart.indicator.Indicator;
import roart.indicator.IndicatorMACD;
import roart.model.MetaItem;
import roart.model.StockItem;
import roart.util.ArraysUtil;
import roart.util.Constants;
import roart.util.SparkUtil;
import scala.Tuple2;
import scala.collection.JavaConversions;
import scala.collection.mutable.WrappedArray;

import java.util.Map;
import java.util.Properties;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import java.io.StringReader;

import org.apache.commons.lang.ArrayUtils;
import org.apache.spark.SparkConf;
import org.apache.spark.SparkContext;
import org.apache.spark.sql.SQLContext;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.SparkSession;
import org.apache.spark.sql.types.DataTypes;
import org.apache.spark.sql.types.StructField;
import org.apache.spark.sql.types.StructType;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.RowFactory;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.PairFunction;
import org.apache.spark.sql.types.ArrayType;
import org.apache.spark.sql.types.StringType;
import org.apache.spark.sql.types.DoubleType;
import org.apache.spark.ml.Model;
import org.apache.spark.ml.PipelineModel;
import org.apache.spark.ml.classification.LogisticRegression;
import org.apache.spark.ml.classification.MultilayerPerceptronClassificationModel;
import org.apache.spark.ml.classification.MultilayerPerceptronClassifier;
import org.apache.spark.ml.evaluation.MulticlassClassificationEvaluator;
import org.apache.spark.ml.linalg.DenseVector;

import com.google.common.collect.ConcurrentHashMultiset;
import com.google.common.collect.Multiset;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class DbSpark {

	private static Logger log = LoggerFactory.getLogger(DbSpark.class);

	private static SparkSession spark;
	private static Properties prop;

    //private static Model model;
	public DbSpark(MyMyConfig conf) {
	    
		try {
			String sparkmaster = conf.getDbSparkMaster();
			//sparkmaster = MyPropertyConfig.instance().sparkMaster;
			spark = SparkUtil.createSparkSession(sparkmaster, "Stockstat DB");
			prop = new java.util.Properties();
			prop.setProperty("driver", "org.postgresql.Driver");
			System.out.println("spark conf fin");
		} catch (Exception e) {
			log.error(Constants.EXCEPTION, e);
		}

	}

	public static List<StockItem> getAll(String market) throws Exception {
		long time0 = System.currentTimeMillis();
		List<StockItem> retList = new ArrayList<>();
		Dataset<Row> allstocks = spark.read().jdbc("jdbc:postgresql://localhost:5432/stockstat?user=stockstat&password=password", "stock", prop);
		Dataset<Row> allstocksMarket = allstocks.filter(allstocks.col("marketid").equalTo(market));
		for (Row row : allstocksMarket.collectAsList()) {
			String dbid = row.getAs("dbid");
			String marketid = row.getAs("marketid");
			String id = row.getAs("id");
			String name = row.getAs("name");
			Date date = row.getAs("date");
			Double indexvalue = row.getAs("indexvalue");
            Double indexvaluelow = row.getAs("indexvaluelow");
            Double indexvaluehigh = row.getAs("indexvaluehigh");
			Double price = row.getAs("price");
            Double pricelow = row.getAs("pricelow");
            Double pricehigh = row.getAs("pricehigh");
            Integer volume = row.getAs("volume");
			String currency = row.getAs("currency");
			Double period1 = row.getAs("period1");
			Double period2 = row.getAs("period2");
			Double period3 = row.getAs("period3");
			Double period4 = row.getAs("period4");
			Double period5 = row.getAs("period5");
			Double period6 = row.getAs("period6");
            Double period7 = row.getAs("period7");
            Double period8 = row.getAs("period8");
            Double period9 = row.getAs("period9");
			retList.add(new StockItem(dbid, marketid, id, name, date, indexvalue, indexvaluelow, indexvaluehigh, price, pricelow, pricehigh, volume, currency, period1, period2, period3, period4, period5, period6, period7, period8, period9));			
		}
		log.info("time0 " + (System.currentTimeMillis() - time0));
		{
		    //allstocks.select("date").distinct().sort("date").show();<commit_msg>Volume as long.<commit_after>package roart.db;

import roart.config.ConfigConstantMaps;
import roart.config.MyMyConfig;
import roart.config.MyXMLConfig;
import roart.indicator.Indicator;
import roart.indicator.IndicatorMACD;
import roart.model.MetaItem;
import roart.model.StockItem;
import roart.util.ArraysUtil;
import roart.util.Constants;
import roart.util.SparkUtil;
import scala.Tuple2;
import scala.collection.JavaConversions;
import scala.collection.mutable.WrappedArray;

import java.util.Map;
import java.util.Properties;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import java.io.StringReader;

import org.apache.commons.lang.ArrayUtils;
import org.apache.spark.SparkConf;
import org.apache.spark.SparkContext;
import org.apache.spark.sql.SQLContext;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.SparkSession;
import org.apache.spark.sql.types.DataTypes;
import org.apache.spark.sql.types.StructField;
import org.apache.spark.sql.types.StructType;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.RowFactory;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.PairFunction;
import org.apache.spark.sql.types.ArrayType;
import org.apache.spark.sql.types.StringType;
import org.apache.spark.sql.types.DoubleType;
import org.apache.spark.ml.Model;
import org.apache.spark.ml.PipelineModel;
import org.apache.spark.ml.classification.LogisticRegression;
import org.apache.spark.ml.classification.MultilayerPerceptronClassificationModel;
import org.apache.spark.ml.classification.MultilayerPerceptronClassifier;
import org.apache.spark.ml.evaluation.MulticlassClassificationEvaluator;
import org.apache.spark.ml.linalg.DenseVector;

import com.google.common.collect.ConcurrentHashMultiset;
import com.google.common.collect.Multiset;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class DbSpark {

	private static Logger log = LoggerFactory.getLogger(DbSpark.class);

	private static SparkSession spark;
	private static Properties prop;

    //private static Model model;
	public DbSpark(MyMyConfig conf) {
	    
		try {
			String sparkmaster = conf.getDbSparkMaster();
			//sparkmaster = MyPropertyConfig.instance().sparkMaster;
			spark = SparkUtil.createSparkSession(sparkmaster, "Stockstat DB");
			prop = new java.util.Properties();
			prop.setProperty("driver", "org.postgresql.Driver");
			System.out.println("spark conf fin");
		} catch (Exception e) {
			log.error(Constants.EXCEPTION, e);
		}

	}

	public static List<StockItem> getAll(String market) throws Exception {
		long time0 = System.currentTimeMillis();
		List<StockItem> retList = new ArrayList<>();
		Dataset<Row> allstocks = spark.read().jdbc("jdbc:postgresql://localhost:5432/stockstat?user=stockstat&password=password", "stock", prop);
		Dataset<Row> allstocksMarket = allstocks.filter(allstocks.col("marketid").equalTo(market));
		for (Row row : allstocksMarket.collectAsList()) {
			String dbid = row.getAs("dbid");
			String marketid = row.getAs("marketid");
			String id = row.getAs("id");
			String name = row.getAs("name");
			Date date = row.getAs("date");
			Double indexvalue = row.getAs("indexvalue");
            Double indexvaluelow = row.getAs("indexvaluelow");
            Double indexvaluehigh = row.getAs("indexvaluehigh");
			Double price = row.getAs("price");
            Double pricelow = row.getAs("pricelow");
            Double pricehigh = row.getAs("pricehigh");
            Long volume = row.getAs("volume");
			String currency = row.getAs("currency");
			Double period1 = row.getAs("period1");
			Double period2 = row.getAs("period2");
			Double period3 = row.getAs("period3");
			Double period4 = row.getAs("period4");
			Double period5 = row.getAs("period5");
			Double period6 = row.getAs("period6");
            Double period7 = row.getAs("period7");
            Double period8 = row.getAs("period8");
            Double period9 = row.getAs("period9");
			retList.add(new StockItem(dbid, marketid, id, name, date, indexvalue, indexvaluelow, indexvaluehigh, price, pricelow, pricehigh, volume, currency, period1, period2, period3, period4, period5, period6, period7, period8, period9));			
		}
		log.info("time0 " + (System.currentTimeMillis() - time0));
		{
		    //allstocks.select("date").distinct().sort("date").show();
{'fraction_non_alphanumeric': 0.1283416528008874, 'fraction_numerical': 0.008208541320022186, 'mean_word_length': 4.885117493472585, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4139322', 'n_tokens_mistral': 3057, 'n_tokens_neox': 2842, 'n_words': 539}