<commit_before>{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Creates TensorFlow Graphs for Spark NLP NerDLApproach\n",
    "TensorFlow: `1.15.0`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import os\n",
    "import logging\n",
    "\n",
    "import tensorflow.compat.v1 as tf\n",
    "import string\n",
    "import random\n",
    "import math\n",
    "import sys\n",
    "import shutil\n",
    "\n",
    "sys.path.append('../lib/ner')\n",
    "from ner_model import NerModel\n",
    "from dataset_encoder import DatasetEncoder\n",
    "from ner_model_saver import NerModelSaver\n",
    "from pathlib import Path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(tf.__version__)\n",
    "print(tf.keras.__version__)\n",
    "\n",
    "logger = tf.get_logger()\n",
    "logger.setLevel(logging.ERROR)\n",
    "\n",
    "tf.get_logger().setLevel('ERROR')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## SETTINGS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# By default the first GPU is used.\n",
    "# If you have multiple GPU devices and wish to\n",
    "# use a different device, you can set that here. (Make suer that device is available!)\n",
    "# In case there is no GPU, it falls back on CPU\n",
    "\n",
    "gpu_device=0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def create_graph(ntags, embeddings_dim, nchars, lstm_size = 128):\n",
    "    if sys.version_info[0] != 3 or sys.version_info[1] >= 7:\n",
    "        print('Python 3.6 or above not supported by tensorflow')\n",
    "        return\n",
    "    if tf.__version__ != '1.15.0':\n",
    "        print('Spark NLP is compiled with TensorFlow 1.15.0, Please use such version.')\n",
    "        print('Current TensorFlow version: ', tf.__version__)\n",
    "        return\n",
    "    tf.disable_v2_behavior()\n",
    "    tf.reset_default_graph()\n",
    "    model_name = 'blstm'+'_{}_{}_{}_{}'.format(ntags, embeddings_dim, lstm_size, nchars)\n",
    "    with tf.Session() as session:\n",
    "        ner = NerModel(session=None, use_gpu_device=gpu_device)\n",
    "        ner.add_cnn_char_repr(nchars, 25, 30)\n",
    "        ner.add_bilstm_char_repr(nchars, 25, 30)\n",
    "        ner.add_pretrained_word_embeddings(embeddings_dim)\n",
    "        ner.add_context_repr(ntags, lstm_size, 3)\n",
    "        ner.add_inference_layer(True)\n",
    "        ner.add_training_op(5)\n",
    "        ner.init_variables()\n",
    "        saver = tf.train.Saver()\n",
    "        file_name = model_name + '.pb'\n",
    "        tf.io.write_graph(ner.session.graph, './', file_name, False)\n",
    "        ner.close()\n",
    "        session.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Attributes info\n",
    "- 1st attribute: max number of tags (Must be at least equal to the number of unique labels, including O if IOB)\n",
    "- 2nd attribute: embeddings dimension\n",
    "- 3rd attribute: max number of characters processed (Must be at least the largest possible amount of characters)\n",
    "- 4th attribute: LSTM Size (128)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# CoNLL 2003 - English - GloVe 100d\n",
    "create_graph(10, 100, 120)\n",
    "\n",
    "# CoNLL 2003 - English - GloVe 300d\n",
    "create_graph(10, 300, 120)\n",
    "\n",
    "# CoNLL 2003 - English - BERT Base\n",
    "create_graph(10, 768, 120)\n",
    "\n",
    "# CoNLL 2003 - English - BERT Large\n",
    "create_graph(10, 1024, 120)\n",
    "\n",
    "# Catch some generic situations\n",
    "create_graph(20, 100, 120)\n",
    "create_graph(20, 300, 120)\n",
    "create_graph(30, 100, 600)\n",
    "create_graph(30, 300, 600)\n",
    "create_graph(30, 768, 600)\n",
    "create_graph(30, 1024, 600)\n",
    "\n",
    "# You got the idea :)\n",
    "# Set the numbers according to your own dataset if the current graphs failed!\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },<commit_msg>[skip travis] Update NER create model notebook<commit_after>{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Creates TensorFlow Graphs for Spark NLP NerDLApproach\n",
    "TensorFlow: `1.15.0`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import os\n",
    "import logging\n",
    "\n",
    "import tensorflow.compat.v1 as tf\n",
    "import string\n",
    "import random\n",
    "import math\n",
    "import sys\n",
    "import shutil\n",
    "\n",
    "sys.path.append('../lib/ner')\n",
    "from ner_model import NerModel\n",
    "from dataset_encoder import DatasetEncoder\n",
    "from ner_model_saver import NerModelSaver\n",
    "from pathlib import Path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.15.0\n",
      "2.2.4-tf\n"
     ]
    }
   ],
   "source": [
    "print(tf.__version__)\n",
    "print(tf.keras.__version__)\n",
    "\n",
    "logger = tf.get_logger()\n",
    "logger.setLevel(logging.ERROR)\n",
    "\n",
    "tf.get_logger().setLevel('ERROR')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## SETTINGS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# By default the first GPU is used.\n",
    "# If you have multiple GPU devices and wish to\n",
    "# use a different device, you can set that here. (Make suer that device is available!)\n",
    "# In case there is no GPU, it falls back on CPU\n",
    "\n",
    "gpu_device=0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def create_graph(ntags, embeddings_dim, nchars, lstm_size = 128):\n",
    "    if sys.version_info[0] != 3 or sys.version_info[1] >= 7:\n",
    "        print('Python 3.6 or above not supported by tensorflow')\n",
    "        return\n",
    "    if tf.__version__ != '1.15.0':\n",
    "        print('Spark NLP is compiled with TensorFlow 1.15.0, Please use such version.')\n",
    "        print('Current TensorFlow version: ', tf.__version__)\n",
    "        return\n",
    "    tf.disable_v2_behavior()\n",
    "    tf.reset_default_graph()\n",
    "    model_name = 'blstm'+'_{}_{}_{}_{}'.format(ntags, embeddings_dim, lstm_size, nchars)\n",
    "    with tf.Session() as session:\n",
    "        ner = NerModel(session=None, use_gpu_device=gpu_device)\n",
    "        ner.add_cnn_char_repr(nchars, 25, 30)\n",
    "        ner.add_bilstm_char_repr(nchars, 25, 30)\n",
    "        ner.add_pretrained_word_embeddings(embeddings_dim)\n",
    "        ner.add_context_repr(ntags, lstm_size, 3)\n",
    "        ner.add_inference_layer(True)\n",
    "        ner.add_training_op(5)\n",
    "        ner.init_variables()\n",
    "        saver = tf.train.Saver()\n",
    "        file_name = model_name + '.pb'\n",
    "        tf.io.write_graph(ner.session.graph, './', file_name, False)\n",
    "        ner.close()\n",
    "        session.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Attributes info\n",
    "- 1st attribute: max number of tags (Must be at least equal to the number of unique labels, including O if IOB)\n",
    "- 2nd attribute: embeddings dimension\n",
    "- 3rd attribute: max number of characters processed (Must be at least the largest possible amount of characters)\n",
    "- 4th attribute: LSTM Size (128)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# CoNLL 2003 - English - GloVe 100d\n",
    "create_graph(10, 100, 120)\n",
    "\n",
    "# CoNLL 2003 - English - GloVe 300d\n",
    "create_graph(10, 300, 120)\n",
    "\n",
    "# CoNLL 2003 - English - ELMO\n",
    "create_graph(10, 512, 120)\n",
    "\n",
    "# CoNLL 2003 - English - BERT Base\n",
    "create_graph(10, 768, 120)\n",
    "\n",
    "# CoNLL 2003 - English - BERT Large and ELMO\n",
    "create_graph(10, 1024, 120)\n",
    "\n",
    "# OntoNotes - English - GloVe 100d\n",
    "create_graph(38, 100, 200)\n",
    "\n",
    "# OntoNotes - English - GloVe 300d\n",
    "create_graph(38, 300, 200)\n",
    "\n",
    "# OntoNotes - English - ELMO\n",
    "create_graph(38, 512, 200)\n",
    "\n",
    "# OntoNotes - English - BERT Base\n",
    "create_graph(38, 768, 200)\n",
    "\n",
    "# OntoNotes - English - BERT Large and ELMO\n",
    "create_graph(38, 1024, 200)\n",
    "\n",
    "# You got the idea :)\n",
    "# Set the numbers according to your own dataset if the current graphs failed!\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
{'fraction_non_alphanumeric': 0.18881404368238416, 'fraction_numerical': 0.03214941824862217, 'mean_word_length': 2.464992927864215, 'pattern_counts': {'":': 99, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4727532', 'n_tokens_mistral': 3904, 'n_tokens_neox': 3619, 'n_words': 793}