<commit_before>{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using default config.\n",
      "[2018-01-27 20:10:54,432] {tf_logging.py:82} INFO - Using default config.\n",
      "INFO:tensorflow:Using config: {'_save_checkpoints_secs': 600, '_session_config': None, '_keep_checkpoint_max': 5, '_task_type': 'worker', '_is_chief': True, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7f1ffcdcee50>, '_save_checkpoints_steps': None, '_keep_checkpoint_every_n_hours': 10000, '_service': None, '_num_ps_replicas': 0, '_tf_random_seed': None, '_master': '', '_num_worker_replicas': 1, '_task_id': 0, '_log_step_count_steps': 100, '_model_dir': './model_trained', '_save_summary_steps': 100}\n",
      "[2018-01-27 20:10:54,434] {tf_logging.py:82} INFO - Using config: {'_save_checkpoints_secs': 600, '_session_config': None, '_keep_checkpoint_max': 5, '_task_type': 'worker', '_is_chief': True, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7f1ffcdcee50>, '_save_checkpoints_steps': None, '_keep_checkpoint_every_n_hours': 10000, '_service': None, '_num_ps_replicas': 0, '_tf_random_seed': None, '_master': '', '_num_worker_replicas': 1, '_task_id': 0, '_log_step_count_steps': 100, '_model_dir': './model_trained', '_save_summary_steps': 100}\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "[2018-01-27 20:10:54,545] {tf_logging.py:82} INFO - Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Saving checkpoints for 1 into ./model_trained/model.ckpt.\n",
      "[2018-01-27 20:10:54,899] {tf_logging.py:82} INFO - Saving checkpoints for 1 into ./model_trained/model.ckpt.\n",
      "INFO:tensorflow:loss = 12500.0, step = 1\n",
      "[2018-01-27 20:10:54,948] {tf_logging.py:82} INFO - loss = 12500.0, step = 1\n",
      "INFO:tensorflow:Saving checkpoints for 100 into ./model_trained/model.ckpt.\n",
      "[2018-01-27 20:10:55,028] {tf_logging.py:82} INFO - Saving checkpoints for 100 into ./model_trained/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 0.0004915521.\n",
      "[2018-01-27 20:10:55,072] {tf_logging.py:82} INFO - Loss for final step: 0.0004915521.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.estimator.canned.linear.LinearRegressor at 0x7f202d4e8110>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import shutil\n",
    "featcols = [\n",
    "  tf.feature_column.numeric_column(\"sq_footage\")\n",
    "]\n",
    "\n",
    "shutil.rmtree('./model_trained', ignore_errors=True)\n",
    "model = tf.estimator.LinearRegressor(featcols, './model_trained')\n",
    "\n",
    "def train_input_fn():\n",
    "  features = {\n",
    "    \"sq_footage\": tf.constant([1000, 2000])\n",
    "  }\n",
    "  labels = tf.constant([50, 100]) # in thousands\n",
    "  return features, labels\n",
    "  \n",
    "model.train(train_input_fn, steps=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "checkpoint\t\t\t    model.ckpt-100.meta\r\n",
      "graph.pbtxt\t\t\t    model.ckpt-1.data-00000-of-00001\r\n",
      "model.ckpt-100.data-00000-of-00001  model.ckpt-1.index\r\n",
      "model.ckpt-100.index\t\t    model.ckpt-1.meta\r\n"
     ]
    }
   ],
   "source": [
    "!ls model_trained"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using default config.\n",
      "[2018-01-27 20:11:02,228] {tf_logging.py:82} INFO - Using default config.\n",
      "INFO:tensorflow:Using config: {'_save_checkpoints_secs': 600, '_session_config': None, '_keep_checkpoint_max': 5, '_task_type': 'worker', '_is_chief': True, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7f202cc629d0>, '_save_checkpoints_steps': None, '_keep_checkpoint_every_n_hours': 10000, '_service': None, '_num_ps_replicas': 0, '_tf_random_seed': None, '_master': '', '_num_worker_replicas': 1, '_task_id': 0, '_log_step_count_steps': 100, '_model_dir': './model_trained', '_save_summary_steps': 100}\n",
      "[2018-01-27 20:11:02,231] {tf_logging.py:82} INFO - Using config: {'_save_checkpoints_secs': 600, '_session_config': None, '_keep_checkpoint_max': 5, '_task_type': 'worker', '_is_chief': True, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7f202cc629d0>, '_save_checkpoints_steps': None, '_keep_checkpoint_every_n_hours': 10000, '_service': None, '_num_ps_replicas': 0, '_tf_random_seed': None, '_master': '', '_num_worker_replicas': 1, '_task_id': 0, '_log_step_count_steps': 100, '_model_dir': './model_trained', '_save_summary_steps': 100}\n"
     ]
    }
   ],
   "source": [
    "# restore model and predict with it\n",
    "import tensorflow as tf\n",
    "featcols = [\n",
    "  tf.feature_column.numeric_column(\"sq_footage\")\n",
    "]\n",
    "trained = tf.estimator.LinearRegressor(featcols, './model_trained')\n",
    "def pred_input_fn():\n",
    "  features = {\n",
    "    \"sq_footage\": tf.constant([1500, 1800])\n",
    "  }\n",
    "  return features\n",
    "out = trained.predict(pred_input_fn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Input graph does not contain a QueueRunner. That means predict yields forever. This is probably a mistake.\n",
      "[2018-01-27 20:11:04,618] {tf_logging.py:90} WARNING - Input graph does not contain a QueueRunner. That means predict yields forever. This is probably a mistake.\n",
      "INFO:tensorflow:Restoring parameters from ./model_trained/model.ckpt-100\n",
      "[2018-01-27 20:11:04,692] {tf_logging.py:82} INFO - Restoring parameters from ./model_trained/model.ckpt-100\n",
      "{'predictions': array([75.00495], dtype=float32)}\n",
      "{'predictions': array([89.996025], dtype=float32)}\n"
     ]
    }
   ],
   "source": [
    "print out.next()\n",
    "print out.next()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "__init__(\n",
    "    feature_columns,\n",
    "    model_dir=None,\n",
    "    label_dimension=1,\n",
    "    weight_column=None,\n",
    "    optimizer='Ftrl',\n",
    "    config=None,\n",
    "    partitioner=None\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}<commit_msg>try out pandas input function<commit_after>{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using default config.\n",
      "INFO:tensorflow:Using config: {'_save_checkpoints_secs': 600, '_session_config': None, '_keep_checkpoint_max': 5, '_task_type': 'worker', '_is_chief': True, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7f574034bad0>, '_save_checkpoints_steps': None, '_keep_checkpoint_every_n_hours': 10000, '_service': None, '_num_ps_replicas': 0, '_tf_random_seed': None, '_master': '', '_num_worker_replicas': 1, '_task_id': 0, '_log_step_count_steps': 100, '_model_dir': './model_trained', '_save_summary_steps': 100}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/envs/py2env/lib/python2.7/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Saving checkpoints for 1 into ./model_trained/model.ckpt.\n",
      "INFO:tensorflow:loss = 12500.0, step = 1\n",
      "INFO:tensorflow:Saving checkpoints for 100 into ./model_trained/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 0.0004915521.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.estimator.canned.linear.LinearRegressor at 0x7f5787cab450>"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import shutil\n",
    "featcols = [\n",
    "  tf.feature_column.numeric_column(\"sq_footage\")\n",
    "]\n",
    "\n",
    "shutil.rmtree('./model_trained', ignore_errors=True)\n",
    "model = tf.estimator.LinearRegressor(featcols, './model_trained')\n",
    "\n",
    "def train_input_fn():\n",
    "  features = {\n",
    "    \"sq_footage\": tf.constant([1000, 2000])\n",
    "  }\n",
    "  labels = tf.constant([50, 100]) # in thousands\n",
    "  return features, labels\n",
    "  \n",
    "model.train(train_input_fn, steps=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "checkpoint\t\t\t    model.ckpt-100.meta\r\n",
      "graph.pbtxt\t\t\t    model.ckpt-1.data-00000-of-00001\r\n",
      "model.ckpt-100.data-00000-of-00001  model.ckpt-1.index\r\n",
      "model.ckpt-100.index\t\t    model.ckpt-1.meta\r\n"
     ]
    }
   ],
   "source": [
    "!ls model_trained"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using default config.\n",
      "[2018-01-27 20:11:02,228] {tf_logging.py:82} INFO - Using default config.\n",
      "INFO:tensorflow:Using config: {'_save_checkpoints_secs': 600, '_session_config': None, '_keep_checkpoint_max': 5, '_task_type': 'worker', '_is_chief': True, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7f202cc629d0>, '_save_checkpoints_steps': None, '_keep_checkpoint_every_n_hours': 10000, '_service': None, '_num_ps_replicas': 0, '_tf_random_seed': None, '_master': '', '_num_worker_replicas': 1, '_task_id': 0, '_log_step_count_steps': 100, '_model_dir': './model_trained', '_save_summary_steps': 100}\n",
      "[2018-01-27 20:11:02,231] {tf_logging.py:82} INFO - Using config: {'_save_checkpoints_secs': 600, '_session_config': None, '_keep_checkpoint_max': 5, '_task_type': 'worker', '_is_chief': True, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7f202cc629d0>, '_save_checkpoints_steps': None, '_keep_checkpoint_every_n_hours': 10000, '_service': None, '_num_ps_replicas': 0, '_tf_random_seed': None, '_master': '', '_num_worker_replicas': 1, '_task_id': 0, '_log_step_count_steps': 100, '_model_dir': './model_trained', '_save_summary_steps': 100}\n"
     ]
    }
   ],
   "source": [
    "# restore model and predict with it\n",
    "import tensorflow as tf\n",
    "featcols = [\n",
    "  tf.feature_column.numeric_column(\"sq_footage\")\n",
    "]\n",
    "trained = tf.estimator.LinearRegressor(featcols, './model_trained')\n",
    "def pred_input_fn():\n",
    "  features = {\n",
    "    \"sq_footage\": tf.constant([1500, 1800])\n",
    "  }\n",
    "  return features\n",
    "out = trained.predict(pred_input_fn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Input graph does not contain a QueueRunner. That means predict yields forever. This is probably a mistake.\n",
      "[2018-01-27 20:11:04,618] {tf_logging.py:90} WARNING - Input graph does not contain a QueueRunner. That means predict yields forever. This is probably a mistake.\n",
      "INFO:tensorflow:Restoring parameters from ./model_trained/model.ckpt-100\n",
      "[2018-01-27 20:11:04,692] {tf_logging.py:82} INFO - Restoring parameters from ./model_trained/model.ckpt-100\n",
      "{'predictions': array([75.00495], dtype=float32)}\n",
      "{'predictions': array([89.996025], dtype=float32)}\n"
     ]
    }
   ],
   "source": [
    "print out.next()\n",
    "print out.next()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using default config.\n",
      "WARNING:tensorflow:Using temporary folder as model directory: /tmp/tmp90etbo\n",
      "INFO:tensorflow:Using config: {'_save_checkpoints_secs': 600, '_session_config': None, '_keep_checkpoint_max': 5, '_task_type': 'worker', '_is_chief': True, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7f56fe38efd0>, '_save_checkpoints_steps': None, '_keep_checkpoint_every_n_hours': 10000, '_service': None, '_num_ps_replicas': 0, '_tf_random_seed': None, '_master': '', '_num_worker_replicas': 1, '_task_id': 0, '_log_step_count_steps': 100, '_model_dir': '/tmp/tmp90etbo', '_save_summary_steps': 100}\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Saving checkpoints for 1 into /tmp/tmp90etbo/model.ckpt.\n",
      "INFO:tensorflow:loss = 807500.0, step = 1\n",
      "INFO:tensorflow:Saving checkpoints for 16 into /tmp/tmp90etbo/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 0.020175744.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.estimator.canned.linear.LinearRegressor at 0x7f56fe38edd0>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "\n",
    "featcols = [\n",
    "  tf.feature_column.numeric_column(\"sq_footage\")\n",
    "]\n",
    "\n",
    "df = pd.DataFrame({\n",
    "    \"sq_footage\": [1000, 2000],\n",
    "    \"price\":      [50, 100] # in thousands\n",
    "})\n",
    "\n",
    "model = tf.estimator.LinearRegressor(featcols)\n",
    "\n",
    "def train_input_fn(df, num_epochs):  # a Pandas dataframe\n",
    "  return tf.estimator.inputs.pandas_input_fn(\n",
    "    x = df,\n",
    "    y = df['price'],\n",
    "    num_epochs=num_epochs,\n",
    "    shuffle = True\n",
    "  )\n",
    "\n",
    "model.train(train_input_fn(df, 1000), steps=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
{'fraction_non_alphanumeric': 0.1854970618525474, 'fraction_numerical': 0.05112982371115284, 'mean_word_length': 3.3729801324503312, 'pattern_counts': {'":': 174, '<': 14, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5013823', 'n_tokens_mistral': 6726, 'n_tokens_neox': 6076, 'n_words': 1115}