<jupyter_start><jupyter_text>Structured Data Solution In this notebook, we will use the structured data package in Datalab to build a model to predict taxifares.<jupyter_code>import os
PROJECT = 'cloud-training-demos'    # CHANGE THIS
BUCKET = 'cloud-training-demos-ml'  # CHANGE THIS
REGION = 'us-central1' # CHANGE THIS

os.environ['PROJECT'] = PROJECT # for bash
os.environ['BUCKET'] = BUCKET # for bash
os.environ['REGION'] = REGION # for bash
%bash
echo "project=$PROJECT"
echo "bucket=$BUCKET"
echo "region=$REGION"
gcloud config set project $PROJECT
gcloud config set compute/region $REGION
gcloud beta ml init-project -q
import tensorflow as tf
import datalab.ml as ml
import google.cloud.ml as cml
import datalab_solutions.structured_data as sd
from tensorflow.python.lib.io import file_io
import json
import shutil

print('tf ' + str(tf.__version__))
print('sd ' + str(sd.__version__))
print('cml ' + str(cml.__version__))
INDIR = '../feateng/sample'
OUTDIR = '.'

# for bash
os.environ['OUTDIR'] = OUTDIR<jupyter_output><empty_output><jupyter_text>Set up schema file Schema of training/test. Same format as BigQuery.  STRING/INTEGER/FLOAT only.<jupyter_code>%writefile $OUTDIR/taxifare.json
[
    {
        "mode": "NULLABLE",
        "name": "fare_amount",
        "type": "FLOAT"
    }, 
    {
        "mode": "NULLABLE",
        "name": "dayofweek",
        "type": "STRING"
    },
    {
        "mode": "NULLABLE",
        "name": "hourofday",
        "type": "STRING"
    },
    {
        "mode": "NULLABLE",
        "name": "pickuplon",
        "type": "FLOAT"
    },
    {
        "mode": "NULLABLE",
        "name": "pickuplat",
        "type": "FLOAT"
    },
    {
        "mode": "NULLABLE",
        "name": "dropofflon",
        "type": "FLOAT"
    },
    {
        "mode": "NULLABLE",
        "name": "dropofflat",
        "type": "FLOAT"
    },
    {
        "mode": "NULLABLE",
        "name": "passengers",
        "type": "FLOAT"
    },
    {
        "mode": "REQUIRED",
        "name": "key",
        "type": "STRING"
    } 
]<jupyter_output>Overwriting ./taxifare.json<jupyter_text>Preprocessing The first step of preprocessing is to compute the min, max, etc. for scaling purposes.<jupyter_code>!rm -rf $OUTDIR/taxi_preproc $OUTDIR/taxi_trained
train_csv = ml.CsvDataSet(
  file_pattern=os.path.join(INDIR, 'train*'),
  schema_file=os.path.join(OUTDIR, 'taxifare.json'))
sd.local_preprocess(
  dataset=train_csv,
  output_dir=os.path.join(OUTDIR, 'taxi_preproc'),
)<jupyter_output>Starting local preprocessing.
Local preprocessing done.<jupyter_text>The second step is to specify the feature columns and transformations.  The target and key transforms are required. Everything else is optional.<jupyter_code>transforms = {
  "fare_amount": {"transform": "target"},
  "key": {"transform": "key"}, 
  "dayofweek": {"transform": "one_hot"},
  "hourofday": {"transform": "embedding", "embedding_dim": 2}, # group-combine the hour
}
file_io.write_string_to_file(os.path.join(OUTDIR, 'taxi_preproc/transforms.json'),
                             json.dumps(transforms, indent=2))
%bash
ls $OUTDIR/taxi_preproc
!cat $OUTDIR/taxi_preproc/num*json
!cat $OUTDIR/taxi_preproc/vocab_day*<jupyter_output>Wed
Sun
Fri
Tue
Mon
Thu
Sat<jupyter_text>Local Training and prediction Train using the preproprocessed data.<jupyter_code>eval_csv = ml.CsvDataSet(
  file_pattern=os.path.join(INDIR, 'valid*'),
  schema_file=os.path.join(OUTDIR, 'taxifare.json'))

shutil.rmtree(os.path.join(OUTDIR, 'taxi_trained'), ignore_errors=True)
sd.local_train(
  train_dataset=train_csv,
  eval_dataset=eval_csv,
  preprocess_output_dir=os.path.join(OUTDIR, 'taxi_preproc'),
  transforms=os.path.join(OUTDIR, 'taxi_preproc/transforms.json'),
  output_dir=os.path.join(OUTDIR, 'taxi_trained'),
  model_type='dnn_regression',
  max_steps=2500,
  layer_sizes=[64, 4]
)
!ls taxi_trained
sd.local_predict(
  training_ouput_dir=os.path.join(OUTDIR, 'taxi_trained'),
  data=['Sun,0,-73.984685,40.769262,-73.991065,40.728145,5.0,row_01',
        'Mon,12,-74.006927,40.739993,-73.950025,40.773403,1.0,row_02',
        'Tues,8,-73.977345,40.779387,-73.97615,40.778867,1.0,row_03',
        'Fri,17,-73.97136,40.794413,-73.99623,40.74524,1.0,row_04',
        'Sun,0,-73.997642,40.763853,-73.99485,40.750282,1.0,row_05',
        'Sun,0,-74.004538,40.742202,-73.955823,40.773485,1.0,row_06',
        'Sun,0,-74.000589,40.73731,-73.985902,40.692725,1.0,row_07',
        'Sun,0,-73.995432,40.72114,-73.992403,40.719745,1.0,row_08',
        'Sun,0,-73.945033,40.779203,-73.952037,40.766802,1.0,row_09',
        'Sun,0,-73.968592,40.693262,-73.99231,40.694317,1.0,row_10']
)
shutil.rmtree(os.path.join(OUTDIR,'batch_predict'), ignore_errors=True)
sd.local_batch_predict(
  training_ouput_dir=os.path.join(OUTDIR, 'taxi_trained'),
  prediction_input_file=os.path.join(INDIR, 'valid*'),
  output_dir=os.path.join(OUTDIR, 'batch_predict'),
  output_format='csv',
  mode='evaluation'   # mode=prediction if you don't have target column
)
!head $OUTDIR/batch_predict/predictions-00000-*<jupyter_output>2009-04-28 01:01:23.000000-73.980640.750940.7533-73.9696,11.4058189392,4.90000009537
2009-10-27 02:46:24.000000-73.980840.758840.7605-73.9931,11.1602220535,4.90000009537
2010-10-05 02:09:00.000000-74.000140.730240.7526-74.0039,11.1707496643,7.69999980927
2012-05-08 03:50:00.000000-74.003840.73840.7591-73.9904,11.6874761581,6.90000009537
2012-05-08 03:50:00.000000-73.988840.736540.7193-73.9436,11.727016449,12.1000003815
2012-12-04 06:52:02.000000-73.982240.731840.7725-73.951,11.7066316605,11.0
2012-11-20 06:53:30.000000-73.94740.771940.7554-73.9759,11.7086172104,9.0
2014-05-13 07:33:52.000000-73.962340.79540.7853-73.973,10.7292032242,5.0
2011-05-17 07:53:55.000000-73.970540.752340.7467-73.9817,10.7170114517,5.30000019073
2012-10-09 07:47:46.000000-73.973940.747740.7256-74.0055,10.7183790207,13.5
{'fraction_non_alphanumeric': 0.16098388464800678, 'fraction_numerical': 0.17217981340118746, 'mean_word_length': 4.104761904761904, 'pattern_counts': {'":': 36, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '521314', 'license': 'Apache-2.0', 'n_tokens_mistral': 2961, 'n_tokens_neox': 2385, 'n_words': 347, 'path': 'courses/machine_learning/packaged/sdsoln.ipynb', 'repo_name': 'bjoernrost/training-data-analyst', 'size': '5895'}