<commit_before>/*
 *  Copyright (c) 2015, WSO2 Inc. (http://www.wso2.org) All Rights Reserved.
 *
 *  WSO2 Inc. licenses this file to you under the Apache License,
 *  Version 2.0 (the "License"); you may not use this file except
 *  in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *  http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing,
 *  software distributed under the License is distributed on an
 *  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 *  KIND, either express or implied.  See the License for the
 *  specific language governing permissions and limitations
 *  under the License.
 *
 */
package org.wso2.carbon.analytics.spark.core.internal;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.deploy.master.Master;
import org.apache.spark.deploy.worker.Worker;
import org.apache.spark.sql.DataFrame;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.SQLContext;
import org.wso2.carbon.analytics.dataservice.AnalyticsDataService;
import org.wso2.carbon.analytics.dataservice.AnalyticsServiceHolder;
import org.wso2.carbon.analytics.dataservice.clustering.AnalyticsClusterException;
import org.wso2.carbon.analytics.dataservice.clustering.AnalyticsClusterManager;
import org.wso2.carbon.analytics.dataservice.clustering.GroupEventListener;
import org.wso2.carbon.analytics.datasource.commons.Record;
import org.wso2.carbon.analytics.datasource.commons.exception.AnalyticsException;
import org.wso2.carbon.analytics.datasource.commons.exception.AnalyticsTableNotAvailableException;
import org.wso2.carbon.analytics.datasource.core.util.GenericUtils;
import org.wso2.carbon.analytics.spark.core.AnalyticsExecutionCall;
import org.wso2.carbon.analytics.spark.core.exception.AnalyticsExecutionException;
import org.wso2.carbon.analytics.spark.core.util.AnalyticsConstants;
import org.wso2.carbon.analytics.spark.core.util.AnalyticsQueryResult;
import org.wso2.carbon.analytics.spark.core.util.AnalyticsRelation;
import org.wso2.carbon.utils.CarbonUtils;
import scala.None$;
import scala.Option;

import java.io.File;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.lang.reflect.Method;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.nio.file.attribute.PosixFilePermission;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

/**
 * This class represents the analytics query execution context.
 */
public class SparkAnalyticsExecutor implements GroupEventListener {

    private static final int P2_BASE_PORT = 8090;

    private static final int P1_BASE_PORT = 4501;

    private static final String MASTER_PORT_GROUP_PROP = "MASTER_PORT";

    private static final String MASTER_HOST_GROUP_PROP = "MASTER_HOST";

    private static final int BASE_WEBUI_PORT = 8081;

    private static final int BASE_MASTER_PORT = 7077;

    private static final String CLUSTER_GROUP_NAME = "CARBON_ANALYTICS_EXECUTION";

    private static final String LOCAL_MASTER_URL = "local";

    private static final String CARBON_ANALYTICS_SPARK_APP_NAME = "CarbonAnalytics";

    private static final Log log = LogFactory.getLog(SparkAnalyticsExecutor.class);
    
    private SparkConf sparkConf;

    private JavaSparkContext javaSparkCtx;

    private SQLContext sqlCtx;

    private String myHost;
    
    private int portOffset;
    
    private int workerCount = 1;
    
    private Object workerActorSystem;
    private Object masterActorSystem;

    public SparkAnalyticsExecutor(String myHost, int portOffset) throws AnalyticsClusterException {
        this.myHost = myHost;
        this.portOffset = portOffset;
        AnalyticsClusterManager acm = AnalyticsServiceHolder.getAnalyticsClusterManager();
        if (acm.isClusteringEnabled()) {
            this.initSparkDataListener();
            acm.joinGroup(CLUSTER_GROUP_NAME, this);
        } else {
            this.sparkConf = new SparkConf();
            this.initClient(LOCAL_MASTER_URL);
        }
    }
    
    private void initClient(String masterUrl) {
        this.sparkConf.setMaster(masterUrl).setAppName(CARBON_ANALYTICS_SPARK_APP_NAME);
        this.sparkConf.set("spark.scheduler.mode", "FAIR");
        this.sparkConf.set("spark.deploy.recoveryMode", "FILESYSTEM");
        this.sparkConf.set("spark.deploy.recoveryDirectory", "/home/niranda/wso2/bamm5/recovery/");

        this.javaSparkCtx = new JavaSparkContext(this.sparkConf);
        this.sqlCtx = new SQLContext(this.javaSparkCtx);
    }

    private void startMaster(String host, int port, int webUIport) {
        this.masterActorSystem = Master.startSystemAndActor(host, port, webUIport, this.sparkConf)._1();
    }

    private void startWorker(String workerHost, String masterHost, int masterPort, int p1, int p2) {
        this.sparkConf = new SparkConf();
        this.sparkConf.setMaster("spark://" + masterHost + ":" + masterPort)
                .setAppName(CARBON_ANALYTICS_SPARK_APP_NAME);
        this.sparkConf.set("spark.scheduler.mode", "FAIR");

        this.workerActorSystem = Worker.startSystemAndActor(workerHost, p1, p2, 2, 1000000, new String[]{"spark://" + masterHost + ":" + masterPort},
                                                            null, (Option) None$.MODULE$, sparkConf)._1();
          }

    private void initSparkDataListener() {
        this.validateSparkScriptPathPermission();
        ExecutorService executor = Executors.newFixedThreadPool(1);
        SparkDataListener listener = new SparkDataListener();
        executor.execute(listener);
    }

    private void validateSparkScriptPathPermission() {
        Set<PosixFilePermission> perms = new HashSet<>();
        //add owners permission
        perms.add(PosixFilePermission.OWNER_READ);<commit_msg>adding kryo serializer to sparkconf<commit_after>/*
 *  Copyright (c) 2015, WSO2 Inc. (http://www.wso2.org) All Rights Reserved.
 *
 *  WSO2 Inc. licenses this file to you under the Apache License,
 *  Version 2.0 (the "License"); you may not use this file except
 *  in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *  http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing,
 *  software distributed under the License is distributed on an
 *  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 *  KIND, either express or implied.  See the License for the
 *  specific language governing permissions and limitations
 *  under the License.
 *
 */
package org.wso2.carbon.analytics.spark.core.internal;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.deploy.master.Master;
import org.apache.spark.deploy.worker.Worker;
import org.apache.spark.sql.DataFrame;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.SQLContext;
import org.wso2.carbon.analytics.dataservice.AnalyticsDataService;
import org.wso2.carbon.analytics.dataservice.AnalyticsServiceHolder;
import org.wso2.carbon.analytics.dataservice.clustering.AnalyticsClusterException;
import org.wso2.carbon.analytics.dataservice.clustering.AnalyticsClusterManager;
import org.wso2.carbon.analytics.dataservice.clustering.GroupEventListener;
import org.wso2.carbon.analytics.datasource.commons.Record;
import org.wso2.carbon.analytics.datasource.commons.exception.AnalyticsException;
import org.wso2.carbon.analytics.datasource.commons.exception.AnalyticsTableNotAvailableException;
import org.wso2.carbon.analytics.datasource.core.util.GenericUtils;
import org.wso2.carbon.analytics.spark.core.AnalyticsExecutionCall;
import org.wso2.carbon.analytics.spark.core.exception.AnalyticsExecutionException;
import org.wso2.carbon.analytics.spark.core.util.AnalyticsConstants;
import org.wso2.carbon.analytics.spark.core.util.AnalyticsQueryResult;
import org.wso2.carbon.analytics.spark.core.util.AnalyticsRelation;
import org.wso2.carbon.utils.CarbonUtils;
import scala.None$;
import scala.Option;

import java.io.File;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.lang.reflect.Method;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.nio.file.attribute.PosixFilePermission;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

/**
 * This class represents the analytics query execution context.
 */
public class SparkAnalyticsExecutor implements GroupEventListener {

    private static final int P2_BASE_PORT = 8090;

    private static final int P1_BASE_PORT = 4501;

    private static final String MASTER_PORT_GROUP_PROP = "MASTER_PORT";

    private static final String MASTER_HOST_GROUP_PROP = "MASTER_HOST";

    private static final int BASE_WEBUI_PORT = 8081;

    private static final int BASE_MASTER_PORT = 7077;

    private static final String CLUSTER_GROUP_NAME = "CARBON_ANALYTICS_EXECUTION";

    private static final String LOCAL_MASTER_URL = "local";

    private static final String CARBON_ANALYTICS_SPARK_APP_NAME = "CarbonAnalytics";

    private static final Log log = LogFactory.getLog(SparkAnalyticsExecutor.class);
    
    private SparkConf sparkConf;

    private JavaSparkContext javaSparkCtx;

    private SQLContext sqlCtx;

    private String myHost;
    
    private int portOffset;
    
    private int workerCount = 1;
    
    private Object workerActorSystem;
    private Object masterActorSystem;

    public SparkAnalyticsExecutor(String myHost, int portOffset) throws AnalyticsClusterException {
        this.myHost = myHost;
        this.portOffset = portOffset;
        AnalyticsClusterManager acm = AnalyticsServiceHolder.getAnalyticsClusterManager();
        if (acm.isClusteringEnabled()) {
            this.initSparkDataListener();
            acm.joinGroup(CLUSTER_GROUP_NAME, this);
        } else {
            this.sparkConf = new SparkConf();
            this.initClient(LOCAL_MASTER_URL);
        }
    }
    
    private void initClient(String masterUrl) {
        this.sparkConf.setMaster(masterUrl).setAppName(CARBON_ANALYTICS_SPARK_APP_NAME);
        this.sparkConf.set("spark.scheduler.mode", "FAIR");

        this.javaSparkCtx = new JavaSparkContext(this.sparkConf);
        this.sqlCtx = new SQLContext(this.javaSparkCtx);
    }

    private void startMaster(String host, int port, int webUIport) {
        this.masterActorSystem = Master.startSystemAndActor(host, port, webUIport, this.sparkConf)._1();
    }

    private void startWorker(String workerHost, String masterHost, int masterPort, int p1, int p2) {
        this.sparkConf = new SparkConf();
        this.sparkConf.setMaster("spark://" + masterHost + ":" + masterPort)
                .setAppName(CARBON_ANALYTICS_SPARK_APP_NAME);
        this.sparkConf.set("spark.scheduler.mode", "FAIR");
        this.sparkConf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer");

        this.workerActorSystem = Worker.startSystemAndActor(workerHost, p1, p2, 2, 1000000, new String[]{"spark://" + masterHost + ":" + masterPort},
                                                            null, (Option) None$.MODULE$, sparkConf)._1();
          }

    private void initSparkDataListener() {
        this.validateSparkScriptPathPermission();
        ExecutorService executor = Executors.newFixedThreadPool(1);
        SparkDataListener listener = new SparkDataListener();
        executor.execute(listener);
    }

    private void validateSparkScriptPathPermission() {
        Set<PosixFilePermission> perms = new HashSet<>();
        //add owners permission
        perms.add(PosixFilePermission.OWNER_READ);
{'fraction_non_alphanumeric': 0.09367088607594937, 'fraction_numerical': 0.010126582278481013, 'mean_word_length': 3.9699675324675323, 'pattern_counts': {'":': 4, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3963387', 'n_tokens_mistral': 3800, 'n_tokens_neox': 3569, 'n_words': 781}