<jupyter_start><jupyter_text>Import<jupyter_code>// %install-swiftpm-flags -c release
// %install '.package(url: "https://github.com/JacopoMangiavacchi/SwiftCoreMLTools.git", from: "0.0.5")' SwiftCoreMLTools
// %install '.package(url: "https://github.com/dduan/Just.git", from: "0.8.0")' Just
import Foundation
import TensorFlow
// import SwiftCoreMLTools
// import Just<jupyter_output><empty_output><jupyter_text>Data DownloadBoston house prices dataset---------------------------**Data Set Characteristics:**      :Number of Instances: 506     :Number of Attributes: 13 numeric/categorical predictive. Median Value (attribute 14) is usually the target.    :Attribute Information (in order):        - CRIM     per capita crime rate by town        - ZN       proportion of residential land zoned for lots over 25,000 sq.ft.        - INDUS    proportion of non-retail business acres per town        - CHAS     Charles River dummy variable (= 1 if tract bounds river; 0 otherwise)        - NOX      nitric oxides concentration (parts per 10 million)        - RM       average number of rooms per dwelling        - AGE      proportion of owner-occupied units built prior to 1940        - DIS      weighted distances to five Boston employment centres        - RAD      index of accessibility to radial highways        - TAX      full-value property-tax rate per ten thousand dollars        - PTRATIO  pupil-teacher ratio by town        - B        1000(Bk - 0.63)^2 where Bk is the proportion of blacks by town        - LSTAT    % lower status of the population        - MEDV     Median value of owner-occupied homes in a thousand dollar    :Missing Attribute Values: None    :Creator: <NAME>. and <NAME>.This is a copy of UCI ML housing dataset.https://archive.ics.uci.edu/ml/machine-learning-databases/housing/<jupyter_code>// if let cts = Just.get(URL(string: "https://archive.ics.uci.edu/ml/machine-learning-databases/housing/housing.data")!).content {
//     try! cts.write(to: URL(fileURLWithPath:"../data/housing.csv"))
// }<jupyter_output><empty_output><jupyter_text>Data Ingestion<jupyter_code>let data = try String(contentsOfFile:"../data/housing.csv", encoding: String.Encoding.utf8)
let dataRecords: [[Float]] = data.split(separator: "\n").map{ String($0).split(separator: " ").compactMap{ Float(String($0)) } }

let numRecords = dataRecords.count
let numColumns = dataRecords[0].count

var index = Set<Int>()

while index.count < numRecords {
    index.insert(Int.random(in: 0..<numRecords))
}

let randomDataRecords = index.map{ dataRecords[$0] }

let dataFeatures = randomDataRecords.map{ Array($0[0..<numColumns-1]) }
let dataLabels = randomDataRecords.map{ Array($0[(numColumns-1)...]) }<jupyter_output><empty_output><jupyter_text>Data Transformation  Split Numerical Categorical Features<jupyter_code>let categoricalColumns = [3, 8]
let numericalColumns = [0, 1, 2, 4, 5, 6, 7, 9, 10, 11, 12]
let numCategoricalFeatures = categoricalColumns.count
let numNumericalFeatures = numericalColumns.count
let numLabels = 1

assert(numColumns == numCategoricalFeatures + numNumericalFeatures + 1)

// Get Categorical Features
let allCategoriesValues = dataFeatures.map{ row in categoricalColumns.map{ Int32(row[$0]) } }
                                .reduce(into: Array(repeating: [Int32](), count: 2)){ total, value in
                                    total[0].append(value[0])
                                    total[1].append(value[1]) }
                                .map{ Set($0).sorted() }

let categoricalFeatures = dataFeatures.map{ row in categoricalColumns.map{ Int32(row[$0]) } }

// Get Numerical Features
let numericalFeatures = dataFeatures.map{ row in numericalColumns.map{ row[$0] } }<jupyter_output><empty_output><jupyter_text>Categorize Categorical Features with Ordinal values<jupyter_code>var categoricalValues = Array(repeating: Set<Int32>(), count: 2)

for record in categoricalFeatures {
    categoricalValues[0].insert(record[0])
    categoricalValues[1].insert(record[1])
}

let sortedCategoricalValues = [categoricalValues[0].sorted(), categoricalValues[1].sorted()]

let ordinalCategoricalFeatures = categoricalFeatures.map{ [Int32(sortedCategoricalValues[0].firstIndex(of:$0[0])!), 
                                                           Int32(sortedCategoricalValues[1].firstIndex(of:$0[1])!)] }<jupyter_output><empty_output><jupyter_text>Split Train and Test<jupyter_code>let trainPercentage:Float = 0.8
let numTrainRecords = Int(ceil(Float(numRecords) * trainPercentage))
let numTestRecords = numRecords - numTrainRecords

func matrixTranspose<T>(_ matrix: [[T]]) -> [[T]] {
    if matrix.isEmpty {return matrix}
    var result = [[T]]()
    for index in 0..<matrix.first!.count {
        result.append(matrix.map{$0[index]})
    }
    return result
}

let xCategoricalAllTrain = matrixTranspose(Array(ordinalCategoricalFeatures[0..<numTrainRecords]))
let xCategoricalAllTest = matrixTranspose(Array(ordinalCategoricalFeatures[numTrainRecords...]))
let xNumericalAllTrain = Array(Array(numericalFeatures[0..<numTrainRecords]).joined())
let xNumericalAllTest = Array(Array(numericalFeatures[numTrainRecords...]).joined())
let yAllTrain = Array(Array(dataLabels[0..<numTrainRecords]).joined())
let yAllTest = Array(Array(dataLabels[numTrainRecords...]).joined())

let XCategoricalTrain = xCategoricalAllTrain.enumerated().map{ (offset, element) in 
    Tensor<Int32>(element).reshaped(to: TensorShape([numTrainRecords, 1]))
}
let XCategoricalTest = xCategoricalAllTest.enumerated().map{ (offset, element) in 
    Tensor<Int32>(element).reshaped(to: TensorShape([numTestRecords, 1]))
}

let XNumericalTrainDeNorm = Tensor<Float>(xNumericalAllTrain).reshaped(to: TensorShape([numTrainRecords, numNumericalFeatures]))
let XNumericalTestDeNorm = Tensor<Float>(xNumericalAllTest).reshaped(to: TensorShape([numTestRecords, numNumericalFeatures]))
let YTrain = Tensor<Float>(yAllTrain).reshaped(to: TensorShape([numTrainRecords, numLabels]))
let YTest = Tensor<Float>(yAllTest).reshaped(to: TensorShape([numTestRecords, numLabels]))<jupyter_output><empty_output><jupyter_text>Normalize Numerical Features<jupyter_code>let mean = XNumericalTrainDeNorm.mean(alongAxes: 0)
let std = XNumericalTrainDeNorm.standardDeviation(alongAxes: 0)

print(mean, std)
let XNumericalTrain = (XNumericalTrainDeNorm - mean)/std
let XNumericalTest = (XNumericalTestDeNorm - mean)/std
print("Training shapes \(XNumericalTrain.shape) \(XCategoricalTrain[0].shape) \(XCategoricalTrain[1].shape) \(YTrain.shape)")
print("Testing shapes  \(XNumericalTest.shape) \(XCategoricalTest[0].shape) \(XCategoricalTest[1].shape) \(YTest.shape)")<jupyter_output>Training shapes [405, 11] [405, 1] [405, 1] [405, 1]
Testing shapes  [101, 11] [101, 1] [101, 1] [101, 1]<jupyter_text>Model<jupyter_code>struct MultiInputs<N: Differentiable, C>: Differentiable {
  var numerical: N
  
  @noDerivative
  var categorical: C

  @differentiable
  init(numerical: N, categorical: C) {
    self.numerical = numerical
    self.categorical = categorical
  }
}

struct RegressionModel: Module {
    var numericalLayer = Dense<Float>(inputSize: 11, outputSize: 32, activation: relu)
    var embedding1 = Embedding<Float>(vocabularySize: 2, embeddingSize: 2)
    var embedding2 = Embedding<Float>(vocabularySize: 9, embeddingSize: 5)
    var embeddingLayer = Dense<Float>(inputSize: (2 + 5), outputSize: 32, activation: relu)
    var allInputConcatLayer = Dense<Float>(inputSize: (32 + 32), outputSize: 128, activation: relu)
    var hiddenLayer = Dense<Float>(inputSize: 128, outputSize: 32, activation: relu)
    var outputLayer = Dense<Float>(inputSize: 32, outputSize: 1)
    
    @differentiable
    func callAsFunction(_ input: MultiInputs<[Tensor<Float>], [Tensor<Int32>]>) -> Tensor<Float> {
        let numericalInput = numericalLayer(input.numerical[0])
        let embeddingOutput1 = embedding1(input.categorical[0])
        let embeddingOutput1Reshaped = embeddingOutput1.reshaped(to: 
            TensorShape([embeddingOutput1.shape[0], embeddingOutput1.shape[1] * embeddingOutput1.shape[2]]))
        let embeddingOutput2 = embedding2(input.categorical[1])
        let embeddingOutput2Reshaped = embeddingOutput2.reshaped(to: 
            TensorShape([embeddingOutput2.shape[0], embeddingOutput2.shape[1] * embeddingOutput2.shape[2]]))
        let embeddingConcat = Tensor<Float>(concatenating: [embeddingOutput1Reshaped, embeddingOutput2Reshaped], alongAxis: 1)
        let embeddingInput = embeddingLayer(embeddingConcat)
        let allConcat = Tensor<Float>(concatenating: [numericalInput, embeddingInput], alongAxis: 1)
        return allConcat.sequenced(through: allInputConcatLayer, hiddenLayer, outputLayer)
    }
}

var model = RegressionModel()<jupyter_output><empty_output><jupyter_text>Training<jupyter_code>let optimizer = RMSProp(for: model, learningRate: 0.001)
Context.local.learningPhase = .training
let epochCount = 500
let batchSize = 32
let numberOfBatch = Int(ceil(Double(numTrainRecords) / Double(batchSize)))
let shuffle = true

func mae(predictions: Tensor<Float>, truths: Tensor<Float>) -> Float {
    return abs(Tensor<Float>(predictions - truths)).mean().scalarized()
}
for epoch in 1...epochCount {
    var epochLoss: Float = 0
    var epochMAE: Float = 0
    var batchCount: Int = 0
    var batchArray = Array(repeating: false, count: numberOfBatch)
    for batch in 0..<numberOfBatch {
        var r = batch
        if shuffle {
            while true {
                r = Int.random(in: 0..<numberOfBatch)
                if !batchArray[r] {
                    batchArray[r] = true
                    break
                }
            }
        }
        
        let batchStart = r * batchSize
        let batchEnd = min(numTrainRecords, batchStart + batchSize)
        let (loss, grad) = model.valueWithGradient { (model: RegressionModel) -> Tensor<Float> in
            let multiInput = MultiInputs(numerical: [XNumericalTrain[batchStart..<batchEnd]],
                                         categorical: [XCategoricalTrain[0][batchStart..<batchEnd],
                                                       XCategoricalTrain[1][batchStart..<batchEnd]])
            let logits = model(multiInput)
            return meanSquaredError(predicted: logits, expected: YTrain[batchStart..<batchEnd])
        }
        optimizer.update(&model, along: grad)
        
        let multiInput = MultiInputs(numerical: [XNumericalTrain[batchStart..<batchEnd]],
                                     categorical: [XCategoricalTrain[0][batchStart..<batchEnd],
                                                   XCategoricalTrain[1][batchStart..<batchEnd]])
        let logits = model(multiInput)
        epochMAE += mae(predictions: logits, truths: YTrain[batchStart..<batchEnd])
        epochLoss += loss.scalarized()
        batchCount += 1
    }
    epochMAE /= Float(batchCount)
    epochLoss /= Float(batchCount)

    print("Epoch \(epoch): MSE: \(epochLoss), MAE: \(epochMAE)")
}<jupyter_output>Epoch 1: MSE: 491.70114, MAE: 19.796753
Epoch 2: MSE: 319.2449, MAE: 15.063409
Epoch 3: MSE: 175.89833, MAE: 10.81574
Epoch 4: MSE: 87.37532, MAE: 7.176715
Epoch 5: MSE: 47.04053, MAE: 4.900528
Epoch 6: MSE: 33.370583, MAE: 4.0233173
Epoch 7: MSE: 28.9492, MAE: 3.6320593
Epoch 8: MSE: 27.033323, MAE: 3.4715817
Epoch 9: MSE: 25.723206, MAE: 3.3521864
Epoch 10: MSE: 24.57535, MAE: 3.2877529
Epoch 11: MSE: 23.5631, MAE: 3.1904118
Epoch 12: MSE: 23.107466, MAE: 3.1184504
Epoch 13: MSE: 22.688564, MAE: 3.0788329
Epoch 14: MSE: 21.615004, MAE: 3.018838
Epoch 15: MSE: 21.367666, MAE: 2.9685915
Epoch 16: MSE: 20.618574, MAE: 2.93647
Epoch 17: MSE: 20.335337, MAE: 2.905603
Epoch 18: MSE: 19.758018, MAE: 2.8837278
Epoch 19: MSE: 19.396109, MAE: 2.8424964
Epoch 20: MSE: 19.329313, MAE: 2.8107526
Epoch 21: MSE: 18.523045, MAE: 2.8074877
Epoch 22: MSE: 18.752144, MAE: 2.774503
Epoch 23: MSE: 18.14185, MAE: 2.7374887
Epoch 24: MSE: 17.834278, MAE: 2.7134554
Epoch 25: MSE: 17.616562, MAE: 2.6849096
E[...]<jupyter_text>Test<jupyter_code>Context.local.learningPhase = .inference

let multiInputTest = MultiInputs(numerical: [XNumericalTest],
                                 categorical: [XCategoricalTest[0],
                                               XCategoricalTest[1]])

let prediction = model(multiInputTest)

let predictionMse = meanSquaredError(predicted: prediction, expected: YTest).scalarized()/Float(numTestRecords)
let predictionMae = mae(predictions: prediction, truths: YTest)/Float(numTestRecords)

print("MSE: \(predictionMse), MAE: \(predictionMae)")<jupyter_output>MSE: 0.09474914, MAE: 0.023811752
{'fraction_non_alphanumeric': 0.12319127065707282, 'fraction_numerical': 0.053214200996283705, 'mean_word_length': 3.769230769230769, 'pattern_counts': {'":': 0, '<': 84, '<?xml version=': 0, '>': 71, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 15, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '270395', 'license': 'MIT', 'n_tokens_mistral': 4557, 'n_tokens_neox': 3995, 'n_words': 917, 'path': 'S4TF_House.ipynb', 'repo_name': 'JacopoMangiavacchi/S4TF-EmbeddingMultiInput', 'size': '12660'}