<commit_before>{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "from urllib.parse import quote_plus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get(engine, url, query):\n",
    "    quoted_query = quote_plus(query)\n",
    "    underscored_query = query.replace(' ', '_')\n",
    "    full_url = '{}?q={}'.format(url, quoted_query)\n",
    "    filename = '{}-{}.html'.format(engine, underscored_query)\n",
    "    r = requests.get(full_url)\n",
    "    status = r.status_code\n",
    "    if status is 200:\n",
    "        soup = BeautifulSoup(r.text, 'html.parser')\n",
    "        with open(filename, 'w') as f:\n",
    "            f.write(soup.prettify())\n",
    "        print('Saved search page to file: {}'.format(filename))\n",
    "        return (soup, filename)\n",
    "    else:\n",
    "        print('Error code: {}; URL: {}'.format(status, full_url))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved search page to file: google-trump_clinton.html\n"
     ]
    }
   ],
   "source": [
    "url = 'https://www.google.com/search'\n",
    "engine = 'google'\n",
    "query = 'trump clinton'\n",
    "soup, filename = get(engine, url, query)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def filter_1(tag):\n",
    "    return (tag.name == 'div'\n",
    "            and tag.parent.name == 'ol'\n",
    "            and tag['class'] == ['g'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def filter_2(tag):\n",
    "    bad_text = 'News for {}'.format(query)\n",
    "    return (tag.name == 'a'\n",
    "            and tag.parent.name == 'h3'\n",
    "            and tag.text != bad_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def strip_href(href):\n",
    "    result = href.strip('/url?q=')\n",
    "    return result.split('&')[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Results\n",
      "Engine: google\n",
      "Query: trump clinton\n",
      "\n",
      "For comparison, see the downloaded html page:\n",
      "google-trump_clinton.html\n",
      "\n",
      "1 http://projects.fivethirtyeight.com/clinton-trump-vote-maps-2016/\n",
      "2 http://www.wsj.com/articles/donald-trumps-new-attack-strategy-keep-clinton-voters-home-1476221895\n",
      "3 https://www.hillaryclinton.com/literallytrump/\n",
      "4 https://www.yahoo.com/news/latest-trump-serious-jailing-clinton-115529857--election.html\n",
      "5 http://www.cnn.com/2016/10/12/politics/hillary-clinton-utah-poll/\n",
      "6 http://www.realclearpolitics.com/epolls/2016/president/us/general_election_trump_vs_clinton-5491.html\n",
      "7 http://abcnews.go.com/Politics/donald-trump-slams-clinton-wikileaks-emails/story%3Fid%3D42743927\n",
      "8 http://www.latimes.com/nation/politics/trailguide/la-na-trailguide-updates-mike-pence-halts-trump-supporter-for-1476223249-htmlstory.html\n",
      "9 http://www.politico.com/story/2016/10/trump-ad-clinton-corruption-229662\n"
     ]
    }
   ],
   "source": [
    "results = [strip_href(tag.h3.a['href'])\n",
    "           for tag in soup.find_all(filter_1)\n",
    "           if tag.find_all(filter_2)]\n",
    "\n",
    "print('Results\\nEngine: {}\\nQuery: {}\\n'.format(engine, query))\n",
    "print('For comparison, see the downloaded html page:\\n{}\\n'.format(filename))\n",
    "for i, r in enumerate(results):\n",
    "    print((i + 1), r)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"<commit_msg>Add bing support, plus refactoring<commit_after>{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "from urllib.parse import quote_plus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get(engine, url, query):\n",
    "    quoted_query = quote_plus(query)\n",
    "    underscored_query = query.replace(' ', '_')\n",
    "    full_url = '{}?q={}'.format(url, quoted_query)\n",
    "    filename = '{}-{}.html'.format(engine, underscored_query)\n",
    "    r = requests.get(full_url)\n",
    "    status = r.status_code\n",
    "    if status is 200:\n",
    "        soup = BeautifulSoup(r.text, 'html.parser')\n",
    "        with open(filename, 'w') as f:\n",
    "            f.write(soup.prettify())\n",
    "        print('Saved search page to file: {}'.format(filename))\n",
    "        return (soup, filename)\n",
    "    else:\n",
    "        print('Error code: {}; URL: {}'.format(status, full_url))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# \n",
    "# Google\n",
    "#\n",
    "def parse_google(soup):\n",
    "    return [strip_href(tag.h3.a['href'])\n",
    "           for tag in soup.find_all(filter_1)\n",
    "           if tag.find_all(filter_2)]\n",
    "def filter_1(tag):\n",
    "    return (tag.name == 'div'\n",
    "            and tag.parent.name == 'ol'\n",
    "            and tag['class'] == ['g'])\n",
    "def filter_2(tag):\n",
    "    bad_text = 'News for {}'.format(query)\n",
    "    return (tag.name == 'a'\n",
    "            and tag.parent.name == 'h3'\n",
    "            and tag.text != bad_text)\n",
    "def strip_href(href):\n",
    "    result = href.strip('/url?q=')\n",
    "    return result.split('&')[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#\n",
    "# Bing\n",
    "#\n",
    "def parse_bing(soup):\n",
    "    return [t.h2.a['href']\n",
    "            for t in soup.find_all('li', 'b_algo')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def print_results(engine, query, filename, results):\n",
    "    print('Results\\nEngine: {}\\nQuery: {}\\n'.format(engine, query))\n",
    "    print('For comparison, see the downloaded html page:\\n{}\\n'.format(filename))\n",
    "    for i, r in enumerate(results):\n",
    "        print((i + 1), r)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved search page to file: google-trump_clinton.html\n",
      "Results\n",
      "Engine: google\n",
      "Query: trump clinton\n",
      "\n",
      "For comparison, see the downloaded html page:\n",
      "google-trump_clinton.html\n",
      "\n",
      "1 http://projects.fivethirtyeight.com/clinton-trump-vote-maps-2016/\n",
      "2 http://abcnews.go.com/Politics/donald-trump-slams-clinton-wikileaks-emails/story%3Fid%3D42743927\n",
      "3 https://www.hillaryclinton.com/literallytrump/\n",
      "4 http://www.wsj.com/articles/donald-trumps-new-attack-strategy-keep-clinton-voters-home-1476221895\n",
      "5 http://www.realclearpolitics.com/epolls/2016/president/us/general_election_trump_vs_clinton-5491.html\n",
      "6 http://www.motherjones.com/politics/2016/10/donald-trump-says-hell-imprison-hillary-clintons-lawyers-too\n",
      "7 http://www.msnbc.com/rachel-maddow-show/trump-may-not-believe-it-the-latest-polls-favor-clinton\n",
      "8 https://www.yahoo.com/news/the-final-meltdown-clinton-camp-taunts-trump-with-parody-music-video-150047438.html\n",
      "9 http://www.deseretnews.com/article/865664606/Poll-Trump-falls-into-tie-with-Clinton-among-Utah-voters.html%3Fpg%3Dall\n"
     ]
    }
   ],
   "source": [
    "url = 'https://www.google.com/search'\n",
    "engine = 'google'\n",
    "query = 'trump clinton'\n",
    "\n",
    "soup, filename = get(engine, url, query)\n",
    "results = parse_google(soup)\n",
    "print_results(engine, query, filename, results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved search page to file: bing-trump_clinton.html\n",
      "Results\n",
      "Engine: bing\n",
      "Query: trump clinton\n",
      "\n",
      "For comparison, see the downloaded html page:\n",
      "bing-trump_clinton.html\n",
      "\n",
      "1 http://www.cnn.com/2016/10/11/politics/donald-trump-hillary-clinton-2016-election-turnout/index.html\n",
      "2 http://www.cbsnews.com/news/why-ohio-is-leaving-donald-trump-for-hillary-clinton/\n",
      "3 http://www.deseretnews.com/article/865664606/Poll-Trump-falls-into-tie-with-Clinton-among-Utah-voters.html?pg=all\n",
      "4 http://www.usatoday.com/story/news/politics/elections/2016/10/10/fact-check-trump-clinton-false-misleading-claims-second-debate/91837294/\n",
      "5 http://www.newsday.com/long-island/politics/trump-clinton-exchange-bitter-words-in-second-presidential-debate-1.12426474\n",
      "6 http://www.wsj.com/livecoverage/hillary-clinton-and-donald-trump-second-debate-2016\n",
      "7 https://www.washingtonpost.com/politics/trump-remains-defiant-ahead-of-debate-as-surrogates-grapple-with-tape-fallout/2016/10/09/9a95a09a-8e28-11e6-9c85-ac42097b8cc0_story.html\n",
      "8 http://www.nbcnews.com/politics/first-read/poll-after-trump-tape-revelation-clinton-s-lead-double-digits-n663691\n",
      "9 http://www.bloomberg.com/politics/articles/2016-10-11/it-s-obama-versus-trump-as-clinton-leans-on-her-cleanup-hitter\n",
      "10 http://www.politico.com/story/2016/10/debate-fact-check-hillary-clinton-donald-trump-229527\n"
     ]
    }
   ],
   "source": [
    "url = 'http://www.bing.com/search'\n",
    "engine = 'bing'\n",
    "query = 'trump clinton'\n",
    "soup, filename = get(engine, url, query)\n",
    "results = parse_bing(soup)\n",
    "print_results(engine, query, filename, results)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
{'fraction_non_alphanumeric': 0.2056492027334852, 'fraction_numerical': 0.027881548974943052, 'mean_word_length': 2.5168215315603972, 'pattern_counts': {'":': 108, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 7, 'lorem ipsum': 0, 'www.': 27, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '642425', 'n_tokens_mistral': 4453, 'n_tokens_neox': 4231, 'n_words': 597}