<jupyter_start><jupyter_text>https://docs.databricks.com/user-guide/faq/xgboost.htmlxgboost<jupyter_code>%%bash 

pip install xgboost --pre

sudo apt-get update
sudo apt-get install -y maven
sudo git clone --recursive https://github.com/dmlc/xgboost

cd xgboost
sudo make -j4

export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-amd64/

cd jvm-packages
sudo mvn package
import ml.dmlc.xgboost4j.scala.{Booster, DMatrix}
import ml.dmlc.xgboost4j.scala.spark.{DataUtils, XGBoost}
import org.apache.spark.mllib.util.MLUtils

sc.parallelize(Seq("")).map(x => {
  import ml.dmlc.xgboost4j.scala.{Booster, DMatrix}
}).collect()

import org.apache.spark.ml.feature.LabeledPoint
import org.apache.spark.ml.linalg.DenseVector

val paramMap = List(
  "eta" -> 0.1f,
  "max_depth" -> 2,
  "objective" -> "binary:logistic").toMap
val trainRDD = sc.parallelize(Seq(
  LabeledPoint(1.0, new DenseVector(Array(2.0, 3.0, 4.0))), 
  LabeledPoint(0.0, new DenseVector(Array(5.0, 5.0, 5.0))), 
  LabeledPoint(1.0, new DenseVector(Array(2.0, 3.0, 4.0))), 
  LabeledPoint(0.0, new DenseVector(Array(5.0, 5.0, 5.0))), 
  LabeledPoint(1.0, new DenseVector(Array(2.0, 3.0, 4.0))), 
  LabeledPoint(0.0, new DenseVector(Array(5.0, 5.0, 5.0))), 
  LabeledPoint(1.0, new DenseVector(Array(2.0, 3.0, 4.0))), 
  LabeledPoint(0.0, new DenseVector(Array(5.0, 5.0, 5.0))), 
  LabeledPoint(1.0, new DenseVector(Array(2.0, 3.0, 4.0))), 
  LabeledPoint(0.0, new DenseVector(Array(5.0, 5.0, 5.0))), 
  LabeledPoint(1.0, new DenseVector(Array(2.0, 3.0, 4.0))), 
  LabeledPoint(1.0, new DenseVector(Array(2.0, 3.0, 4.0))), 
  LabeledPoint(0.0, new DenseVector(Array(5.0, 5.0, 5.0)))
), 4)
trainRDD.take(1)

//val xgboostModel = XGBoost.train()
/*  def train(
      trainingData: RDD[MLLabeledPoint], params: Map[String, Any], round: Int,
      nWorkers: Int, obj: ObjectiveTrait = null, eval: EvalTrait = null,
      useExternalMemory: Boolean = false, missing: Float = Float.NaN): XGBoostModel = {
    require(nWorkers > 0, "you must specify more than 0 workers")
    trainWithRDD(trainingData, params, round, nWorkers, obj, eval, useExternalMemory, missing)
  }
*/
val xgboostModel = XGBoost.train(trainRDD, paramMap, 1, 4, useExternalMemory=true)
xgboostModel

val xgboostPrediction = xgboostModel.predict(trainRDD.map{x => x.features})

xgboostPrediction.collect()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1668081494057725, 'fraction_numerical': 0.05178268251273345, 'mean_word_length': 4.532863849765258, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 11, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '327528', 'license': 'Apache-2.0', 'n_tokens_mistral': 1055, 'n_tokens_neox': 986, 'n_words': 183, 'path': 'jupyterhub/notebooks/zz_under_construction/zz_old/xgboost/xgboost.ipynb', 'repo_name': 'just4jc/pipeline', 'size': '2356'}