<jupyter_start><jupyter_text>[this doc on github](https://github.com/dotnet/interactive/tree/main/samples/notebooks/csharp/Samples) Machine Learning over House Prices with ML.NET<jupyter_code>#i "nuget:https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json" 
#i "nuget:https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json" 

#r "nuget:Microsoft.ML, 1.7.0"
#r "nuget:Microsoft.ML.AutoML, 0.19.0"
#r "nuget:Microsoft.Data.Analysis, 0.19.0"
#r "nuget:XPlot.Plotly.Interactive, 4.0.6"
using static Microsoft.DotNet.Interactive.Formatting.PocketViewTags;
using Microsoft.DotNet.Interactive.Formatting;
using Microsoft.Data.Analysis;
using XPlot.Plotly;
using System.IO;
using System.Net.Http;
string housingPath = "housing.csv";

if (!File.Exists(housingPath))
{
    var contents = await new HttpClient()
        .GetStringAsync("https://raw.githubusercontent.com/ageron/handson-ml2/master/datasets/housing/housing.csv");
        
    File.WriteAllText(housingPath, contents);
}
var housingData = DataFrame.LoadCsv(housingPath);
housingData
housingData.Description()
Chart.Plot(
    new Histogram()
    {
        x = housingData.Columns["median_house_value"],
        nbinsx = 20
    }
)
var chart = Chart.Plot(
    new Scattergl()
    {
        x = housingData.Columns["longitude"],
        y = housingData.Columns["latitude"],
        mode = "markers",
        marker = new Marker()
        {
            color = housingData.Columns["median_house_value"],
            colorscale = "Jet"
        }
    }
);

chart.Width = 600;
chart.Height = 600;
chart.Display();
static T[] Shuffle<T>(T[] array)
{
    Random rand = new Random();
    for (int i = 0; i < array.Length; i++)
    {
        int r = i + rand.Next(array.Length - i);
        T temp = array[r];
        array[r] = array[i];
        array[i] = temp;
    }
    return array;
}

int[] randomIndices = Shuffle(Enumerable.Range(0, (int)housingData.Rows.Count).ToArray());
int testSize = (int)(housingData.Rows.Count * .1);
int[] trainRows = randomIndices[testSize..];
int[] testRows = randomIndices[..testSize];

DataFrame housing_train = housingData[trainRows];
DataFrame housing_test = housingData[testRows];

housing_train.Rows.Count.Display();
housing_test.Rows.Count.Display();
using Microsoft.ML;
using Microsoft.ML.Data;
using Microsoft.ML.AutoML;
#!time

var mlContext = new MLContext();

var experiment = mlContext.Auto().CreateRegressionExperiment(maxExperimentTimeInSeconds: 15);
var result = experiment.Execute(housing_train, labelColumnName:"median_house_value");
var scatters = result.RunDetails.Where(d => d.ValidationMetrics != null).GroupBy(
    r => r.TrainerName,
    (name, details) => new Scattergl()
    {
        name = name,
        x = details.Select(r => r.RuntimeInSeconds),
        y = details.Select(r => r.ValidationMetrics.MeanAbsoluteError),
        mode = "markers",
        marker = new Marker() { size = 12 }
    });

var chart = Chart.Plot(scatters);
chart.WithXTitle("Training Time");
chart.WithYTitle("Error");
chart.Display();

Console.WriteLine($"Best Trainer:{result.BestRun.TrainerName}");
var testResults = result.BestRun.Model.Transform(housing_test);

var trueValues = testResults.GetColumn<float>("median_house_value");
var predictedValues = testResults.GetColumn<float>("Score");

var predictedVsTrue = new Scattergl()
{
    x = trueValues,
    y = predictedValues,
    mode = "markers",
};

var maximumValue = Math.Max(trueValues.Max(), predictedValues.Max());

var perfectLine = new Scattergl()
{
    x = new[] {0, maximumValue},
    y = new[] {0, maximumValue},
    mode = "lines",
};

var chart = Chart.Plot(new[] {predictedVsTrue, perfectLine });
chart.WithXTitle("True Values");
chart.WithYTitle("Predicted Values");
chart.WithLegend(false);
chart.Width = 600;
chart.Height = 600;
chart.Display();<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1481100539984572, 'fraction_numerical': 0.010542555926973516, 'mean_word_length': 4.0519480519480515, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 13, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '28388', 'license': 'MIT', 'n_tokens_mistral': 1336, 'n_tokens_neox': 1290, 'n_words': 244, 'path': 'samples/notebooks/csharp/Samples/HousingML.ipynb', 'repo_name': 'flcdrg/dotnet-interactive', 'size': '3889'}