<commit_before>{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Rule based Matching in Linguistic Features\n",
    "\n",
    "Link [here](https://spacy.io/usage/linguistic-features#rule-based-matching)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Adding Patterns https://spacy.io/usage/linguistic-features#adding-patterns\n",
    "\n",
    "# Importing spaCy\n",
    "import spacy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load the language model\n",
    "nlp = spacy.load('en')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "first sentence match - [(15578876784678163569, 0, 3)]\n"
     ]
    }
   ],
   "source": [
    "# Importing Matcher from spacy\n",
    "from spacy.matcher import Matcher\n",
    "\n",
    "# The matcher must always share the same vocab with the documents it will operate on.\n",
    "matcher = Matcher(nlp.vocab)\n",
    "# add match ID \"HelloWorld\" with no callback and one pattern\n",
    "pattern = [{'LOWER': 'hello'}, {'IS_PUNCT': True}, {'LOWER': 'world'}]\n",
    "\n",
    "# The second argument lets you pass in an optional callback function \"on_match\" to invoke on a successful match.\n",
    "matcher.add('HelloWorld', None, pattern)\n",
    "\n",
    "# Getting the document from the sentence\n",
    "doc = nlp(u'Hello, world! Hello world!')\n",
    "\n",
    "# the matcher will only return the matches and not do anything else, like merge entities or assign labels\n",
    "# we can implement custom logic by passing in a callback function as the on_match argument on add()\n",
    "matches = matcher(doc)\n",
    "\n",
    "# TODO: spacy.io Documentation update, it refers as [('HelloWorld', 0, 2)]\n",
    "print('first sentence match -', matches)\n",
    "# Should match and return the result, TODO: Should it return 2 ?\n",
    "assert len(matches) == 1\n",
    "\n",
    "# New sentence which doesnt match the matcher\n",
    "# TODO: this seems to be the 64bit hash id, use stringstore to retrieve the original string ?\n",
    "doc = nlp(u'another arbitary sentence')\n",
    "matches = matcher(doc)\n",
    "assert len(matches) == 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Patterns -  ['<NAME>', '<NAME>', 'Washington, D.C.']\n",
      "Matches  [(3766102292120407359, 2, 4), (3766102292120407359, 7, 9), (3766102292120407359, 19, 22)]\n"
     ]
    }
   ],
   "source": [
    "# Adding Phrase Patterns\n",
    "\n",
    "# If you need to match large terminology lists, prefer to use PhraseMatcher\n",
    "\n",
    "# Importing PhraseMatcher from spacy\n",
    "from spacy.matcher import PhraseMatcher\n",
    "\n",
    "# Creating new PhraseMatcher\n",
    "matcher = PhraseMatcher(nlp.vocab)\n",
    "# Preparing terminology list\n",
    "terminology_list = ['<NAME>', '<NAME>', 'Washington, D.C.']\n",
    "# Preparing the patterns\n",
    "\n",
    "patterns = [nlp(text) for text in terminology_list]\n",
    "print('Patterns - ', [patrn.text for patrn in patterns])\n",
    "# Adding to matcher\n",
    "matcher.add('TerminologyList', None, *patterns)\n",
    "\n",
    "doc = nlp(u\"German Chancellor <NAME> and US President <NAME> \"\n",
    "          u\"converse in the Oval Office inside the White House in Washington, D.C.\")\n",
    "matches = matcher(doc)\n",
    "# TODO: Why all them matched with same id?\n",
    "print('Matches ', matches)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Adding on_match rules\n",
    "# https://spacy.io/usage/linguistic-features#on_match"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}<commit_msg>Updated Rule based matching Linguistic features<commit_after>{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Rule based Matching in Linguistic Features\n",
    "\n",
    "Link [here](https://spacy.io/usage/linguistic-features#rule-based-matching)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Adding Patterns https://spacy.io/usage/linguistic-features#adding-patterns\n",
    "\n",
    "# Importing spaCy\n",
    "import spacy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load the language model\n",
    "nlp = spacy.load('en')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Importing Matcher from spacy\n",
    "from spacy.matcher import Matcher\n",
    "\n",
    "# The matcher must always share the same vocab with the documents it will operate on.\n",
    "matcher = Matcher(nlp.vocab)\n",
    "# add match ID \"HelloWorld\" with no callback and one pattern\n",
    "pattern = [{'LOWER': 'hello'}, {'IS_PUNCT': True}, {'LOWER': 'world'}]\n",
    "\n",
    "# The second argument lets you pass in an optional callback function \"on_match\" to invoke on a successful match.\n",
    "matcher.add('HelloWorld', None, pattern)\n",
    "\n",
    "# Getting the document from the sentence\n",
    "doc = nlp(u'Hello, world! Hello world!')\n",
    "\n",
    "# the matcher will only return the matches and not do anything else, like merge entities or assign labels\n",
    "# we can implement custom logic by passing in a callback function as the on_match argument on add()\n",
    "matches = matcher(doc)\n",
    "\n",
    "# TODO: spacy.io Documentation update, it refers as [('HelloWorld', 0, 2)]\n",
    "print('first sentence match -', matches)\n",
    "# Should match and return the result, TODO: Should it return 2 ?\n",
    "assert len(matches) == 1\n",
    "\n",
    "# New sentence which doesnt match the matcher\n",
    "# TODO: this seems to be the 64bit hash id, use stringstore to retrieve the original string ?\n",
    "doc = nlp(u'another arbitary sentence')\n",
    "matches = matcher(doc)\n",
    "assert len(matches) == 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Adding Phrase Patterns\n",
    "\n",
    "# If you need to match large terminology lists, prefer to use PhraseMatcher\n",
    "\n",
    "# Importing PhraseMatcher from spacy\n",
    "from spacy.matcher import PhraseMatcher\n",
    "\n",
    "# Creating new PhraseMatcher\n",
    "matcher = PhraseMatcher(nlp.vocab)\n",
    "# Preparing terminology list\n",
    "terminology_list = ['Barack Obama', '<NAME>', 'Washington, D.C.']\n",
    "# Preparing the patterns\n",
    "\n",
    "patterns = [nlp(text) for text in terminology_list]\n",
    "print('Patterns - ', [patrn.text for patrn in patterns])\n",
    "# Adding to matcher\n",
    "matcher.add('TerminologyList', None, *patterns)\n",
    "\n",
    "doc = nlp(u\"German Chancellor <NAME> and US President <NAME> \"\n",
    "          u\"converse in the Oval Office inside the White House in Washington, D.C.\")\n",
    "matches = matcher(doc)\n",
    "# TODO: Why all them matched with same id?\n",
    "print('Matches ', matches)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Entities Without matcher\n",
      "ent - \" next Friday \"\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div class=\"entities\" style=\"line-height: 2.5\">Google I/O event on \n",
       "<mark class=\"entity\" style=\"background: #bfe1d9; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "    next Friday\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">DATE</span>\n",
       "</mark>\n",
       "</div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Adding on_match rules\n",
    "# https://spacy.io/usage/linguistic-features#on_match\n",
    "\n",
    "# Loading language model\n",
    "nlp = spacy.load('en')\n",
    "\n",
    "# Loading displacy to visualize\n",
    "from spacy import displacy\n",
    "\n",
    "doc = nlp('Google I/O event on next Friday')\n",
    "\n",
    "print('Entities Without matcher')\n",
    "# Initially without matcher, Observe that Google I/O is not matched as entity\n",
    "# Is it because we loaded the small model, check with medium and large models\n",
    "for ent in doc.ents:\n",
    "    print('ent - \"', ent,'\"')\n",
    "\n",
    "# next Friday is entity as DATE\n",
    "displacy.render(doc, style='ent', jupyter=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Now new entity With matcher\n",
      "ent -\" Google I/O \"\n",
      "ent -\" next Friday \"\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div class=\"entities\" style=\"line-height: 2.5\">\n",
       "<mark class=\"entity\" style=\"background: #ffeb80; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "    Google I/O\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EVENT</span>\n",
       "</mark>\n",
       " event on \n",
       "<mark class=\"entity\" style=\"background: #bfe1d9; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "    next Friday\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">DATE</span>\n",
       "</mark>\n",
       "</div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# want to match all mentions of \"Google I/O\" \n",
    "# add a second pattern with an added {IS_DIGIT: True} token – this will make sure you also match on \"Google I/O 2017\"\n",
    "\n",
    "# Loading Matcher\n",
    "\n",
    "from spacy.matcher import Matcher\n",
    "# Creating Matcher\n",
    "matcher = Matcher(nlp.vocab)\n",
    "\n",
    "# Get the ID of the 'EVENT' entity type. This is required to set an entity.\n",
    "EVENT = nlp.vocab.strings['EVENT']\n",
    "\n",
    "# on_match callback function\n",
    "def add_event_ent(matcher, doc, i, matches):\n",
    "    # Get the current match and create tuple of entity label, start and end.\n",
    "    # Append entity to the doc's entity. (Don't overwrite doc.ents!)\n",
    "    match_id, start, end = matches[i]\n",
    "    # Adding the EVENT to the document entities\n",
    "    doc.ents += ((EVENT, start, end),)\n",
    "\n",
    "matcher.add('GoogleIO', add_event_ent,\n",
    "            [{'ORTH': 'Google'}, {'ORTH': 'I'}, {'ORTH': '/'}, {'ORTH': 'O'}],\n",
    "            [{'ORTH': 'Google'}, {'ORTH': 'I'}, {'ORTH': '/'}, {'ORTH': 'O'}, {'IS_DIGIT': True}])\n",
    "\n",
    "doc = nlp('Google I/O event on next Friday')\n",
    "\n",
    "matches = matcher(doc)\n",
    "\n",
    "print('Now new entity With matcher')\n",
    "# Observation - see that Google I/O is matched as entity\n",
    "for ent in doc.ents:\n",
    "    print('ent -\"', ent,'\"')\n",
    "\n",
    "# Now we can see that \"Google I/O\" is marked as entity EVENT\n",
    "displacy.render(doc, style='ent', jupyter=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ent -\" Google I/O 2018 \"\n",
      "ent -\" next Friday \"\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div class=\"entities\" style=\"line-height: 2.5\">\n",
       "<mark class=\"entity\" style=\"background: #ffeb80; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "    Google I/O 2018\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EVENT</span>\n",
       "</mark>\n",
       " event on \n",
       "<mark class=\"entity\" style=\"background: #bfe1d9; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "    next Friday\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">DATE</span>\n",
       "</mark>\n",
       "</div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Matching with Ending with digit(year in this case)\n",
    "doc = nlp('Google I/O 2018 event on next Friday')\n",
    "matches = matcher(doc)\n",
    "for ent in doc.ents:\n",
    "    print('ent -\"', ent,'\"')\n",
    "\n",
    "# Now we can see that \"Google I/O 2018\" is marked as entity EVENT\n",
    "displacy.render(doc, style='ent', jupyter=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
{'fraction_non_alphanumeric': 0.1956320317670417, 'fraction_numerical': 0.01786896095301125, 'mean_word_length': 2.700048971596474, 'pattern_counts': {'":': 137, '<': 41, '<?xml version=': 0, '>': 41, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6723535', 'n_tokens_mistral': 5558, 'n_tokens_neox': 5356, 'n_words': 1362}