<jupyter_start><jupyter_text>Transformation from 5 to 4-level ATC codes<jupyter_code>//UDFs for transformation from 5-level to 4-level ATC codes
val take4levelsUDF = udf( (x:String)=>x.slice(0,5) )

val df = spark.sql("select drugcode from hospitalization_prescriptions")

val df_4levels = df.select(take4levelsUDF(col("drugcode")).as("4L")).groupBy("4L").count
df_4levels.show
df_4levels.count
df_4levels.createOrReplaceTempView("ATC_4L")<jupyter_output>+-----+------+
|   4L| count|
+-----+------+
|D06BX| 17413|
|N01BX|  2578|
|A02BX|  4026|
|QG03D|     7|
|V03AH|    22|
|N02AJ|999220|
|N04BA|241991|
|V04CL|    87|
|J01DH|   129|
|L02AA|     2|
|M05BB|  4682|
|G01AA|  4780|
|A06AD| 67895|
|A02BB|   780|
|G03DC|   218|
|L01XE| 10951|
|G04CA|458263|
|C07AA|163041|
|C09BB|  9282|
|A10AB| 85470|
+-----+------+
only showing top 20 rows<jupyter_text>___ Our columns sum to 61.9M prescriptions (correct)<jupyter_code>df_4levels.select(sum("count")).show
%%python
atc_4l = spark.sql("select * from ATC_4L where count>10000").toPandas()
print(atc_4l.shape)
%%python
from src.python.helpers import plotfig_magic

atc_4l_sorted = atc_4l.sort_values(by=["count"], ascending=False)
atc_4l_sorted["col_idx"] = np.arange(0,atc_4l.shape[0])
#print(atc_4l_sorted[:20])
atc_4l_sorted.plot(x="col_idx",y="count")
retval = plotfig_magic()<jupyter_output><empty_output><jupyter_text>Encoding schemes___  One-Hot encoding of the 4L ATC codes* We observe that taking 4 levels of the ATC hierarchy, and ignoring drugs that are prescribed less than 10000 times we get  _216_ unique 4-level drug codes* Let's try to onehot encode these into a vector format using SPARK<jupyter_code>import org.apache.spark.ml.feature.{OneHotEncoder, StringIndexer}
val df = spark.sql("select * from ATC_4L where count>10000").sort(asc("4L"))
df.count
//first we index the drugcodes
val indexer = new StringIndexer()
    .setInputCol("4L")
    .setOutputCol("4L_index")
    .fit(df)
val indexed = indexer.transform(df)

val encoder = new OneHotEncoder()
    .setInputCol("4L_index")
    .setOutputCol("4L_vec")

val encoded = encoder.transform(indexed)<jupyter_output><empty_output><jupyter_text>Test how the encoder works<jupyter_code>indexed.sort(asc("4L")).show
import org.apache.spark.sql.SaveMode
encoded.select("*")
    .repartition(1).write
    .mode(SaveMode.Overwrite)
    .json(
        "spark-warehouse/preprocessing/TEST1HOTENCODEDSPARK"     
         )
encoded.show(5,false)
encoded.select("4L", "4L_vec").createOrReplaceTempView("encoded")
%%python
df = spark.sql("select * from encoded").toPandas()
%python print(df[:10])
%python<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15157256536566882, 'fraction_numerical': 0.07161803713527852, 'mean_word_length': 5.39225181598063, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '561240', 'license': 'MIT', 'n_tokens_mistral': 1161, 'n_tokens_neox': 1039, 'n_words': 206, 'path': 'tsk_20190122_preprocessing_onehot_ATC_4level.ipynb', 'repo_name': 'uit-hdl/norpd_prescription_analyses', 'size': '2639'}