<jupyter_start><jupyter_text>PC Specification | Intel Core i7-7500U - 2.70GHz | 64bit| 8GB Ram<jupyter_code>from pyspark.sql import SparkSession
from pyspark.sql.types import *
from spark_privacy_preserver.mondrian_preserver import Preserver
import time
spark = SparkSession.builder.appName("SimpleApp").getOrCreate()
spark.conf.set("spark.sql.execution.arrow.enabled", "true")
logFile = "data/adult.all.txt"

# reading csv
df = spark.read.csv(logFile).toDF('age',
                                  'workclass',
                                  'fnlwgt',
                                  'education',
                                  'education-num',
                                  'marital-status',
                                  'occupation',
                                  'relationship',
                                  'race',
                                  'sex',
                                  'capital-gain',
                                  'capital-loss',
                                  'hours-per-week',
                                  'native-country',
                                  'income')
# variables
categorical = set((
    'workclass',
    'education',
    'marital-status',
    'occupation',
    'relationship',
    'sex',
    'native-country',
    'race',
    'income',
))
#K-Anonymity 

start = time.time()

feature_columns = ['age', 'occupation']
sensitive_column = 'income'

schema = StructType([
    StructField("age", StringType()),
    StructField("occupation", StringType()),
    StructField("income", StringType()),
    StructField("count", IntegerType())
])
k = 2

# anonymizing
dfn = Preserver.k_anonymize(
    df, k, feature_columns, sensitive_column, categorical, schema)
dfn.show()

end = time.time()

print((end - start)," seconds")
#L-Diversity

start = time.time()

feature_columns = ['age', 'occupation']
sensitive_column = 'income'

schema = StructType([
    StructField("age", StringType()),
    StructField("occupation", StringType()),
    StructField("income", StringType()),
    StructField("count", IntegerType())
])
k = 2
l = 2

# anonymizing
dfn = Preserver.l_diversity(
    df, k,l, feature_columns, sensitive_column, categorical, schema)
dfn.show()

end = time.time()

print((end - start)," seconds")
#T-Closeness

start = time.time()

feature_columns = ['age', 'occupation']
sensitive_column = 'income'

schema = StructType([
    StructField("age", StringType()),
    StructField("occupation", StringType()),
    StructField("income", StringType()),
    StructField("count", IntegerType())
])
k = 2
t = 0.2

# anonymizing
dfn = Preserver.t_closeness(
    df, k,t, feature_columns, sensitive_column, categorical, schema)
dfn.show()

end = time.time()

print((end - start)," seconds")
logFile = "data/calls.csv"
df = spark.read.csv(logFile,header= True)
categorical = set((
    'user', 
    'other', 
    'direction', 
    'timestamp'
))
#K-Anonymize without row suppresion

start = time.time()

feature_columns = ['user', 'other','direction']
sensitive_column = 'timestamp'

schema = StructType([
    StructField("user", StringType()),
    StructField("other", StringType()),
    StructField("direction", StringType()),
    StructField("duration", DoubleType()),
    StructField("timestamp", StringType()),
    # StructField("count", IntegerType())
])

k = 2

# anonymizing
dfn = Preserver.k_anonymize_w_user(
    df, k, feature_columns, sensitive_column, categorical, schema)
dfn.show()

end = time.time()

print((end - start)," seconds")
#L-diversty without row suppresion

start = time.time()

feature_columns = ['user', 'other','direction']
sensitive_column = 'timestamp'

schema = StructType([
    StructField("user", StringType()),
    StructField("other", StringType()),
    StructField("direction", StringType()),
    StructField("duration", DoubleType()),
    StructField("timestamp", StringType()),
    # StructField("count", IntegerType())
])

k = 2
l =2 

# anonymizing
dfn = Preserver.l_diversity_w_user(
    df, k,l, feature_columns, sensitive_column, categorical, schema)
dfn.show()

end = time.time()

print((end - start)," seconds")
#T-closeness without row suppresion

start = time.time()

feature_columns = ['user', 'other','direction']
sensitive_column = 'timestamp'

schema = StructType([
    StructField("user", StringType()),
    StructField("other", StringType()),
    StructField("direction", StringType()),
    StructField("duration", DoubleType()),
    StructField("timestamp", StringType()),
    # StructField("count", IntegerType())
])

k = 2
t = 0.2 

# anonymizing
dfn = Preserver.l_diversity_w_user(
    df, k,t, feature_columns, sensitive_column, categorical, schema)
dfn.show()

end = time.time()

print((end - start)," seconds")
#Single user anonymization

start = time.time()

feature_columns = ['user', 'other','direction']
sensitive_column = 'timestamp'

schema = StructType([
    StructField("user", StringType()),
    StructField("other", StringType()),
    StructField("direction", StringType()),
    StructField("duration", StringType()),
    StructField("timestamp", StringType()),
    # StructField("count", IntegerType())
])

user = '07641036117'
usercolumn_name = "user"
k = 3

# anonymizing
dfn = Preserver.anonymize_user(df, k, user, usercolumn_name, sensitive_column, categorical,schema)
dfn.show()

end = time.time()

print((end - start)," seconds")<jupyter_output>+-----------+--------------------+--------------------+--------+--------------------+
|       user|               other|           direction|duration|           timestamp|
+-----------+--------------------+--------------------+--------+--------------------+
|07610039694|         07434677419|            Incoming|     211|Wed Sep 15 19:17:...|
|07641036117|01666472054,07371...|Outgoing,Incoming...|    0-50|Mon Feb 11 07:18:...|
|07641036117|01666472054,07371...|Outgoing,Incoming...|    0-50|Mon Feb 11 07:45:...|
|07641036117|01666472054,07371...|Outgoing,Incoming...|    0-50|Mon Feb 11 08:04:...|
|07641036117|01666472054,07371...|Outgoing,Incoming...|    0-50|Mon Feb 11 08:05:...|
|07641036117|01666472054,07371...|Outgoing,Incoming...|    0-50|Mon Feb 11 08:06:...|
|07641036117|01666472054,07371...|Outgoing,Incoming...|    0-50|Mon Feb 11 08:06:...|
|07641036117|01666472054,07371...|Outgoing,Incoming...|    0-50|Thu Sep 09 19:35:...|
|07981267897|         07784425582|            Outgoing[...]
{'fraction_non_alphanumeric': 0.180553384398937, 'fraction_numerical': 0.0531499140221979, 'mean_word_length': 2.8542168674698796, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '595261', 'license': 'Apache-2.0', 'n_tokens_mistral': 2251, 'n_tokens_neox': 1966, 'n_words': 400, 'path': 'mondrian_preserver_loadtest.ipynb', 'repo_name': 'ThaminduR/spark-privacy-preserver', 'size': '6397'}