<jupyter_start><jupyter_text>Group the pixels from all images  on pixel positions<jupyter_code>var grids_RDD: RDD[Array[Double]] = tiles_RDD.map(m => m.toArrayDouble())

var indexedValues: RDD[Array[(Int, Double)]] =  grids_RDD.map(m => m.zipWithIndex.map{case (e,v) => (v,e)})
indexedValues.partitions

var flattenValues : RDD[(Int, Double)] = indexedValues.flatMap(m => m).filter( m => !m._2.isNaN)

var groupedValues : RDD[(Int, Iterable[Double])] = flattenValues.groupByKey(1000)<jupyter_output><empty_output><jupyter_text>Calculate Average<jupyter_code>val averagedValues = groupedValues.mapValues(value =>
StatCounter(value).mean).map(m => (m._1.toLong, Math.rint(m._2)))<jupyter_output><empty_output><jupyter_text>Calculate Std<jupyter_code>val stdevduration = groupedValues.mapValues(value =>
StatCounter(value).stdev).map(m => (m._1.toLong, Math.rint(m._2)))<jupyter_output><empty_output><jupyter_text>Use Template<jupyter_code>var num_cols_rows :(Int, Int) = (0, 0)

var tile0 = (tiles_RDD.take(1))(0)
num_cols_rows = (tile0.cols,tile0.rows)

var filepath2 = "hdfs:///user/hadoop/evi_new_template.tif"
//EVI
// var filepath = "hdfs:///user/hadoop/evi_template.tif"
//Since it is a single GeoTiff, it will be a RDD with a tile.
var geoTiff : RDD[(ProjectedExtent, Tile)] = sc.hadoopGeoTiffRDD(filepath2)
// var proExtents_RDD : RDD[ProjectedExtent] = geoTiff.keys
var tile_my : RDD[Tile]= geoTiff.values


var proExtents_RDD : RDD[ProjectedExtent] = geoTiff.keys
var extents_withIndex = proExtents_RDD.zipWithIndex().map{case (e,v) => (v,e)}
var projected_extent = (extents_withIndex.filter(m => m._1 == 0).values.cache.collect())(0)


var grids_RDD = tile_my.map(m => m.toArrayDouble())
var bands_withIndex = grids_RDD.zipWithIndex().map { case (v, i) => (i, v) }
var grid0 : RDD[(Long, Double)] = bands_withIndex.filter(m => m._1 == 0).values.flatMap( m => m).zipWithIndex.map{case (v,i) => (i,v)}.cache<jupyter_output><empty_output><jupyter_text>Produce GeoTiff<jupyter_code>var result_2: RDD[(Long, (Double, Option[Double]))] = grid0.leftOuterJoin(averagedValues) //averagedValues, stdevduration
    // result_2.take(20).foreach(println)

    //TODO: 1. If value in original .tiff  , but None in seasonality ==> set NaN for the corresponding index
var result: RDD[(Long, Double)] = result_2.map { 
    case (index, (value_org, value_pheno)) => {if (value_pheno.isDefined)  (index,value_pheno.get)
                                               else (index, Double.NaN)  }
}


var season = DoubleArrayTile(result.sortBy(_._1).map(_._2).collect(),num_cols_rows._1, num_cols_rows._2)

var geoTif = new SinglebandGeoTiff(season, projected_extent.extent, projected_extent.crs, Tags.empty, GeoTiffOptions(compression.DeflateCompression))
var output = "/data/local/home/parrot/minio2/" + "average_1" + ".tif"
GeoTiffWriter.write(geoTif, output)<jupyter_output>[Stage 19:==================================================> (973 + 17) / 1000]
{'fraction_non_alphanumeric': 0.17040054310930075, 'fraction_numerical': 0.016293279022403257, 'mean_word_length': 5.682539682539683, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '99220', 'license': 'Apache-2.0', 'n_tokens_mistral': 1151, 'n_tokens_neox': 1096, 'n_words': 193, 'path': 'phenology_analysis.ipynb', 'repo_name': 'phenology/platform-for-pheno-products', 'size': '2946'}