from pyspark.sql.functions import array, col, explode, struct, lit
df = sc.parallelize([(1, 0.0, 0.6), (1, 0.6, 0.7)]).toDF(['A', 'col_1',
    'col_2'])

def to_long(df, by):
    cols, dtypes = zip(*((c, t) for c, t in df.dtypes if c not in by))
assert len(set(dtypes)) == 1, 'All columns have to be of the same type'
kvs = explode(array([struct(lit(c).alias('key'), col(c).alias('val')) for c in
    cols])).alias('kvs')
return df.select(by + [kvs]).select(by + ['kvs.key', 'kvs.val'])
to_long(df, ['A'])
Transpose column to row with Spark