<jupyter_start><jupyter_text>Create CSV fileThis notebook show how to load an Array of triples stored as objectFile and save it again as a CSV file.  Dependencies<jupyter_code>import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}
import geotrellis.spark.io.hadoop._

import org.apache.spark.rdd.RDD
import org.apache.hadoop.io._
import org.apache.hadoop.io.{IOUtils, SequenceFile}<jupyter_output><empty_output><jupyter_text>Configuration<jupyter_code>var offline_dir_path = "hdfs:///user/pheno/spring-index/"
var geoTiff_dir = "BloomFinal"
var wssse_path :String = offline_dir_path + geoTiff_dir + "/wssse"
var wssse_csv_path :String = offline_dir_path + geoTiff_dir + "/wssse.csv"

var conf = sc.hadoopConfiguration
var fs = org.apache.hadoop.fs.FileSystem.get(conf)

if (fs.exists(new org.apache.hadoop.fs.Path(wssse_csv_path))) {
    println("The file " + wssse_csv_path + " already exists we will delete it!!!")
    try { fs.delete(new org.apache.hadoop.fs.Path(wssse_csv_path), true) } catch { case _ : Throwable => { } }
}<jupyter_output><empty_output><jupyter_text>Load Data<jupyter_code>var wssse_data :RDD[(Int, Int, Double)] = sc.emptyRDD

//from disk
if (fs.exists(new org.apache.hadoop.fs.Path(wssse_path))) {
    wssse_data = sc.objectFile(wssse_path)
    println(wssse_data.collect().toList)        
}<jupyter_output>List((12,35,4.3297733110180664E10), (2,35,1.3269493028574286E11), (3,35,7.612110282892206E10), (4,35,5.192563731031121E10), (5,35,3.86547975867681E10), (6,35,3.1070608358128193E10), (7,35,2.657061805793561E10), (8,35,2.307391282797956E10), (9,35,2.0876037759553226E10), (10,35,1.9153577999634872E10), (11,35,1.8028667788894394E10), (12,35,1.7020487136558575E10), (13,35,1.671272347748397E10), (14,35,1.5865979019165756E10), (15,35,1.5225546057630272E10))<jupyter_text>Save Data<jupyter_code>val wssse = wssse_data.repartition(1).sortBy(_._1).map{case (a,b,c) => Array(a,b,c).mkString(",")}
wssse.saveAsTextFile(wssse_csv_path)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14838709677419354, 'fraction_numerical': 0.16476426799007443, 'mean_word_length': 6.905882352941177, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '199707', 'license': 'Apache-2.0', 'n_tokens_mistral': 1039, 'n_tokens_neox': 844, 'n_words': 115, 'path': 'applications/notebooks/examples/scala/create_csv_file.ipynb', 'repo_name': 'phenology/infrastructure', 'size': '2015'}