<jupyter_start><jupyter_text>Table of Contents1&nbsp;&nbsp;aggregate and filter  aggregate and filter- https://stackoverflow.com/questions/59931770/sum-of-array-elements-depending-on-value-condition-pyspark<jupyter_code>import numpy as np
import pandas as pd

# pyspark
import pyspark
spark = pyspark.sql.SparkSession.builder.appName('app').getOrCreate()

# sql
from pyspark.sql.functions import col as _col
from pyspark.sql.functions import udf

# @udf("integer") def myfunc(x,y): return x - y
# stddev format_number date_format, dayofyear, when
from pyspark.sql import functions as F
from pyspark.sql.window import Window
from pyspark.sql.functions import (mean as _mean, min as _min,
                                   max as _max, avg as _avg,
                                   when as _when
                                  )

from pyspark.sql.types import (StructField,StringType,
                               IntegerType, FloatType,
                               DoubleType,StructType)

from pyspark import SparkConf, SparkContext, SQLContext

sc = spark.sparkContext
sqlContext = SQLContext(sc) 
sqc = sqlContext
# spark_df = sqlContext.createDataFrame(pandas_df)
from pyspark.sql.functions import expr
sdf = spark.createDataFrame([(1,[0.2, 2.1, 3., 4., 3., 0.5]),
                             (2,[7., 0.3, 0.3, 8., 2.,])],                                        ['id','column'])

sdf.toPandas()
(sdf
.withColumn("column<2",
  expr("""aggregate(filter(column, x -> x < 2), 0D,
  (x, acc) -> acc + x)"""))
 
.withColumn("column>2",
  expr("""aggregate(filter(column, x -> x > 2), 0D,
  (x, acc) -> acc + x)"""))
 
.withColumn("column=2",
  expr("""aggregate(filter(column, x -> x == 2), 0D,
  (x, acc) -> acc + x)"""))

).toPandas()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13810597519729426, 'fraction_numerical': 0.020293122886133032, 'mean_word_length': 2.652263374485597, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 13, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '562775', 'license': 'Apache-2.0', 'n_tokens_mistral': 623, 'n_tokens_neox': 585, 'n_words': 142, 'path': 'a01_PySpark/a04_Pyspark_Methods/aggregate_filter/aggregate_filter.ipynb', 'repo_name': 'mindis/Big_Data_Analysis', 'size': '1774'}