<jupyter_start><jupyter_text>Install TransmogrifAI library usinghttps://docs.transmogrif.ai/en/stable/installation/index.htmlhttps://docs.transmogrif.ai/en/stable/examples/Running-from-Jupyter-Notebook.htmlInstall both python and scala kernels for jupyter notebook<jupyter_code>%classpath add mvn com.salesforce.transmogrifai transmogrifai-core_2.11 0.7.0
%classpath add mvn org.apache.spark spark-mllib_2.11 2.4.5
import org.apache.spark.SparkConf
import org.apache.spark.sql.SparkSession
import org.apache.spark.SparkContext
import org.apache.spark.sql.functions.udf

import com.salesforce.op._
import com.salesforce.op.features._
import com.salesforce.op.features.types._
import com.salesforce.op.evaluators.Evaluators
import java.io._

import com.salesforce.op.OpWorkflow
import com.salesforce.op.evaluators.Evaluators
import com.salesforce.op.readers.DataReaders
// package com.salesforce.op.readers

import com.salesforce.op.test.PassengerSparkFixtureTest
import org.apache.avro.Schema
import org.apache.avro.generic.GenericRecord
import org.junit.runner.RunWith
import org.apache.avro.generic.GenericRecord
// import org.scalatest.FlatSpec
// import org.scalatest.junit.JUnitRunner
import scala.collection.JavaConverters._
import java.nio.file.Paths
import java.io.File
import org.apache.spark.sql.execution.datasources.csv.CSVSchemaUtils
import com.salesforce.op.utils.io.csv.{CSVInOut, CSVOptions, CSVToAvro}
import com.salesforce.op.readers.{CSVDefaults}
// val conf = new SparkConf().setMaster("local[*]").setAppName("TestPrediction")
val conf = new SparkConf().setMaster("local[*]").setAppName("TestPrediction").set("spark.testing.memory", "2147480000") //use this if memory limit not set


// conf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer") 
// conf.registerKryoClasses(Array(classOf[org.apache.avro.generic.GenericData$Record]))

conf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
conf.registerKryoClasses(Array(classOf[org.apache.avro.generic.GenericData.Record]))

implicit val spark = SparkSession.builder.config(conf).getOrCreate()
// Python Snippet. Switch to Python Kernel
import pandas as pd
testdf = pd.read_csv('../../Benchmark-Labeled-Data/data_test.csv')
test_metadata = pd.read_csv('../../RawCSV/Metadata/meta_data.csv')

# print(len(testdf),len(test_metadata))
test_merged = pd.merge(testdf,test_metadata,on='Record_id')
test_merged

prv_csv_name,csv_name = '',''
for index, row in test_merged.iterrows():
    if index%100==0:  print(index)
    col = row['Attribute_name']
    prv_csv_name = csv_name
    csv_name = '../../RawCSV/RawCSVFiles/' + row['name']
    
    print(csv_name)
    print(col)
    
    if prv_csv_name != csv_name:  df = pd.read_csv(csv_name,encoding='latin1')
    
    try: df_col = df[[col]]
    except:
        df = pd.read_csv(csv_name)
        try: df_col = df[[col]]
        except: df_col = pd.DataFrame(columns=[col])
        
    df_col.to_csv('column_level_csvs/'+ str(index) + '.csv',index=False)
    
def testDataDir: String = {
Some(new File("column_level_csvs/"))
  .collect{ case d if d.isDirectory => d.getPath}
  .getOrElse(Paths.get("column_level_csvs-sibling").relativize(Paths.get("column_level_csvs")).toString)
}
val fw = new FileWriter("TransmogrifAI_Predictions.csv", true)

for (i <- 0 until 1985) {
print(i)


val istr = i.toString
// var Newstr = '../../' + '/' + istr + ".csv"    
var Newstr = testDataDir + '/' + istr + ".csv"
// def passengerCsvWithHeaderPath: String = Paths.get(Newstr, "2.csv").toString

val finalPath = Newstr
val options: CSVOptions = CSVDefaults.CSVOptions
val csvData = new CSVInOut(options).readRDD(finalPath)
val headers: Seq[String] = Seq.empty
val hdrs = if (headers.nonEmpty) headers else csvData.first()
val hdrsSet = hdrs.toSet
val data = csvData.filter(_.exists(!hdrsSet.contains(_)))

val columnPrunning = spark.sessionState.conf.csvColumnPruning

val inferredSchema = CSVSchemaUtils.infer(data.map(_.toArray), hdrs, options, columnPrunning)

fw.write(inferredSchema.toString)
fw.write("\n")
    
println()
println(inferredSchema)
}

fw.close()

// Python Snippet. So, Again switch to Python Kernel
import pandas as pd
from sklearn.metrics import accuracy_score
from sklearn.metrics import confusion_matrix

inf_types = pd.read_csv('TransmogrifAI_Predictions.csv',header=None)
dict_label = {
    'IntegerType': 0,
    'DecimalType(4': 0,
    'DoubleType':0, 
    'LongType':0, 
    'DecimalType(3': 0,
    'StringType': 8,    
    'TimestampType': 2,
    'BooleanType': 8  
}

inf_types[1] = [dict_label[i] for i in inf_types[1]]
y_trans = inf_types[1].values.tolist()
print(len(y_trans))

df = pd.read_csv('../../Benchmark-Labeled-Data/data_test.csv')

dict_label_true = {
    'numeric': 0,
    'categorical': 1,
    'datetime': 2,
    'sentence': 3,
    'url': 4,
    'embedded-number': 5,
    'list': 6,
    'not-generalizable': 7,
    'context-specific': 8
}

y_true = df.y_act.values.tolist()
y_true = [dict_label_true[str(i)] for i in y_true]
                 
print(accuracy_score(y_true, y_trans))
print(confusion_matrix(y_true, y_trans))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14035429238855363, 'fraction_numerical': 0.010706638115631691, 'mean_word_length': 4.652365236523653, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '345127', 'license': 'Apache-2.0', 'n_tokens_mistral': 1905, 'n_tokens_neox': 1810, 'n_words': 320, 'path': 'MLFeatureTypeInference/Source-Code/Existing-Tools/TransmogrifAI.ipynb', 'repo_name': 'ahhuisg/ML-Data-Prep-Zoo', 'size': '5137'}