<commit_before><commit_msg>LHYC safety boat scraper and report<commit_after>{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\ryangrosch\\Anaconda3\\lib\\site-packages\\bs4\\__init__.py:181: UserWarning: No parser was explicitly specified, so I'm using the best available HTML parser for this system (\"lxml\"). This usually isn't a problem, but if you run this code on another system, or in a different virtual environment, it may use a different parser and behave differently.\n",
      "\n",
      "The code that caused this warning is on line 193 of the file C:\\Users\\ryangrosch\\Anaconda3\\lib\\runpy.py. To get rid of this warning, change code that looks like this:\n",
      "\n",
      " BeautifulSoup(YOUR_MARKUP})\n",
      "\n",
      "to this:\n",
      "\n",
      " BeautifulSoup(YOUR_MARKUP, \"lxml\")\n",
      "\n",
      "  markup_type=markup_type))\n"
     ]
    }
   ],
   "source": [
    "# this is a calendar scraper and report of safety boat volunteers for LHYC\n",
    "# use this to contact volunteers and ensure coverage\n",
    "\n",
    "from bs4 import BeautifulSoup, SoupStrainer\n",
    "import re\n",
    "import urllib\n",
    "import webbrowser as wb\n",
    "import random\n",
    "import time\n",
    "import datetime as dt\n",
    "import pandas as pd\n",
    "\n",
    "###################\n",
    "# build calendars #\n",
    "###################\n",
    "\n",
    "month =  5\n",
    "calendars = []\n",
    "p1 = 'http://lhycsailing.com/calendar?EventViewMode=1&EventListViewMode=2&SelectedDate='\n",
    "p2 = '/10/2017&CalendarViewType=1'\n",
    "\n",
    "while month <= 9:\n",
    "    calendar_url = p1 + str(month) + p2\n",
    "    month = month + 1\n",
    "    calendars.append(calendar_url)\n",
    "    \n",
    "###################\n",
    "# find events    #\n",
    "###################  \n",
    "\n",
    "event_ids = []\n",
    "p3 = '^http://lhycsailing.com/event-'\n",
    "\n",
    "for calendar in calendars:\n",
    "    resp = urllib.request.urlopen(calendar)\n",
    "    soup = BeautifulSoup(resp, from_encoding=resp.info().get_param('charset'))\n",
    "    for link in soup.find_all(href=re.compile(p3)):\n",
    "        link = link['href']\n",
    "        link = re.search(p3 + '(.*)\\?', link).group(1)\n",
    "        event_ids.append(link)\n",
    "                      \n",
    "#print(event_ids)\n",
    "\n",
    "#####################\n",
    "# build event links #\n",
    "#####################  \n",
    "\n",
    "p4 = 'http://lhycsailing.com/event-'\n",
    "p5 = '/Attendees'\n",
    "\n",
    "events = []\n",
    "\n",
    "for event_id in event_ids:\n",
    "    event_url = p4 + str(event_id) + p5\n",
    "    events.append(event_url)\n",
    "\n",
    "#print(events)\n",
    "\n",
    "#####################\n",
    "# filled table data #\n",
    "#####################  \n",
    "\n",
    "p6 = '^http://lhycsailing.com/Sys/PublicProfile'\n",
    "members = []\n",
    "filled_events = []\n",
    "filled_dates = []\n",
    "\n",
    "registrations = []\n",
    "unfilled_events = []\n",
    "unfilled_dates = []\n",
    "\n",
    "for event in events:\n",
    "    try:\n",
    "        resp = urllib.request.urlopen(event)\n",
    "        soup = BeautifulSoup(resp, from_encoding=resp.info().get_param('charset'))\n",
    "        \n",
    "        ##  member extraction\n",
    "        for member in soup.find_all(href=re.compile(p6)):\n",
    "            member = member.text\n",
    "            members.append(member)\n",
    "            \n",
    "            event_name = soup.select('div.infoText')[0]\n",
    "            filled_events.extend(event_name.stripped_strings)\n",
    "            \n",
    "            date = soup.select('div.infoText')[1]\n",
    "            filled_dates.extend(date.stripped_strings)\n",
    "    except:\n",
    "        pass\n",
    "    \n",
    "#######################\n",
    "# unfilled table data #\n",
    "#######################  \n",
    "\n",
    "    try: \n",
    "        event_name = soup.select('div.infoText')[0]\n",
    "        unfilled_events.extend(event_name.stripped_strings)\n",
    "            \n",
    "        date = soup.select('div.infoText')[1]\n",
    "        unfilled_dates.extend(date.stripped_strings)\n",
    "        \n",
    "        registered = soup.select('h2.stepTitle')[0]\n",
    "        registrations.extend(registered.stripped_strings)\n",
    "    except:\n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Event</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Member</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th><NAME></th>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th><NAME></th>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th><NAME></th>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th><NAME></th>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th><NAME></th>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th><NAME></th>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th><NAME></th>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th><NAME></th>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th><NAME></th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th><NAME></th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th><NAME></th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th><NAME></th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th><NAME></th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th><NAME></th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th><NAME></th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th><NAME></th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th><NAME></th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th><NAME></th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th><NAME></th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th><NAME></th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   Event\n",
       "Member                  \n",
       "<NAME>         3\n",
       "<NAME>          3\n",
       "<NAME>         3\n",
       "<NAME>      3\n",
       "<NAME>           3\n",
       "<NAME>       3\n",
       "<NAME>            3\n",
       "<NAME>           3\n",
       "<NAME>      2\n",
       "<NAME>         2\n",
       "<NAME>          2\n",
       "<NAME>            2\n",
       "<NAME>        2\n",
       "<NAME>        1\n",
       "<NAME>            1\n",
       "<NAME>          1\n",
       "<NAME>           1\n",
       "<NAME>           1\n",
       "<NAME>      1\n",
       "<NAME>            1"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# How many times has each person volunteered?\n",
    "\n",
    "filled_raw = pd.DataFrame(\n",
    "    {'Member': members,\n",
    "     'Event': filled_events,\n",
    "     'Date': filled_dates\n",
    "    })\n",
    "filled = filled_raw.drop_duplicates()\n",
    "filled_summary = filled.groupby(by='Member').count().sort_values('Event',ascending=False)[['Event']]\n",
    "filled_summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Date</th>\n",
       "      <th>Event</th>\n",
       "      <th>Member</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>05/13/2017 12:00 PM - 5:00 PM</td>\n",
       "      <td>Saturday Safety Boat</td>\n",
       "      <td><NAME></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>05/13/2017 12:00 PM - 5:00 PM</td>\n",
       "      <td>Saturday Safety Boat</td>\n",
       "      <td><NAME></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>05/14/2017 10:30 AM - 1:00 PM</td>\n",
       "      <td>Sunday Safety Boat</td>\n",
       "      <td><NAME></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>05/14/2017 10:30 AM - 1:00 PM</td>\n",
       "      <td>Sunday Safety Boat</td>\n",
       "      <td><NAME></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>05/20/2017 12:00 PM - 5:00 PM</td>\n",
       "      <td>Saturday Safety Boat</td>\n",
       "      <td><NAME></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>05/20/2017 12:00 PM - 5:00 PM</td>\n",
       "      <td>Saturday Safety Boat</td>\n",
       "      <td><NAME></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>05/21/2017 10:30 AM - 1:00 PM</td>\n",
       "      <td>Sunday Safety Boat</td>\n",
       "      <td><NAME></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>05/21/2017 10:30 AM - 1:00 PM</td>\n",
       "      <td>Sunday Safety Boat</td>\n",
       "      <td><NAME></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>05/27/2017 12:00 PM - 5:00 PM</td>\n",
       "      <td>Saturday Safety Boat</td>\n",
       "      <td><NAME></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>05/28/2017 10:30 AM - 1:00 PM</td>\n",
       "      <td>Sunday Safety Boat</td>\n",
       "      <td><NAME></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>05/28/2017 10:30 AM - 1:00 PM</td>\n",
       "      <td>Sunday Safety Boat</td>\n",
       "      <td><NAME></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>05/29/2017 10:30 AM - 1:00 PM</td>\n",
       "      <td>Monday Safety Boat</td>\n",
       "      <td><NAME></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>05/29/2017 10:30 AM - 1:00 PM</td>\n",
       "      <td>Monday Safety Boat</td>\n",
       "      <td><NAME></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>06/03/2017 12:00 PM - 5:00 PM</td>\n",
       "      <td>Saturday Safety Boat</td>\n",
       "      <td><NAME></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>06/04/2017 10:30 AM - 1:00 PM</td>\n",
       "      <td>Sunday Safety Boat</td>\n",
       "      <td><NAME></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>06/10/2017 12:00 PM - 5:00 PM</td>\n",
       "      <td>Saturday Safety Boat</td>\n",
       "      <td><NAME></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>06/11/2017 10:30 AM - 1:00 PM</td>\n",
       "      <td>Sunday Safety Boat</td>\n",
       "      <td><NAME></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>06/18/2017 10:30 AM - 1:00 PM</td>\n",
       "      <td>Sunday Safety Boat</td>\n",
       "      <td><NAME></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>06/24/2017 12:00 PM - 5:00 PM</td>\n",
       "      <td>Saturday Safety Boat</td>\n",
       "      <td><NAME></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>06/24/2017 12:00 PM - 5:00 PM</td>\n",
       "      <td>Saturday Safety Boat</td>\n",
       "      <td><NAME></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>06/25/2017 10:30 AM - 1:00 PM</td>\n",
       "      <td>Sunday Safety Boat</td>\n",
       "      <td><NAME></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>06/25/2017 10:30 AM - 1:00 PM</td>\n",
       "      <td>Sunday Safety Boat</td>\n",
       "      <td><NAME></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>07/01/2017 12:00 PM - 5:00 PM</td>\n",
       "      <td>Saturday Safety Boat</td>\n",
       "      <td><NAME></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>07/01/2017 12:00 PM - 5:00 PM</td>\n",
       "      <td>Saturday Safety Boat</td>\n",
       "      <td><NAME></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>07/02/2017 10:30 AM - 1:00 PM</td>\n",
       "      <td>Sunday Safety Boat</td>\n",
       "      <td><NAME></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>07/04/2017 10:30 AM - 1:00 PM</td>\n",
       "      <td>Monday Safety Boat</td>\n",
       "      <td><NAME></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>07/04/2017 10:30 AM - 1:00 PM</td>\n",
       "      <td>Monday Safety Boat</td>\n",
       "      <td><NAME></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>07/08/2017 12:00 PM - 5:00 PM</td>\n",
       "      <td>Saturday Safety Boat</td>\n",
       "      <td><NAME></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>07/09/2017 10:30 AM - 1:00 PM</td>\n",
       "      <td>Sunday Safety Boat</td>\n",
       "      <td><NAME></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>07/15/2017 12:00 PM - 5:00 PM</td>\n",
       "      <td>Saturday Safety Boat</td>\n",
       "      <td><NAME></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>07/16/2017 10:30 AM - 1:00 PM</td>\n",
       "      <td>Sunday Safety Boat</td>\n",
       "      <td><NAME></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>07/16/2017 10:30 AM - 1:00 PM</td>\n",
       "      <td>Sunday Safety Boat</td>\n",
       "      <td><NAME></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>07/29/2017 12:00 PM - 5:00 PM</td>\n",
       "      <td>Saturday Safety Boat</td>\n",
       "      <td><NAME></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>07/30/2017 10:30 AM - 1:00 PM</td>\n",
       "      <td>Sunday Safety Boat</td>\n",
       "      <td><NAME></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>08/06/2017 10:30 AM - 1:00 PM</td>\n",
       "      <td>Sunday Safety Boat</td>\n",
       "      <td><NAME></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>08/19/2017 12:00 PM - 5:00 PM</td>\n",
       "      <td>Saturday Safety Boat</td>\n",
       "      <td><NAME></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>09/03/2017 10:30 AM - 1:00 PM</td>\n",
       "      <td>Sunday Safety Boat</td>\n",
       "      <td><NAME></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>09/04/2017 10:30 AM - 1:00 PM</td>\n",
       "      <td>Monday Safety Boat</td>\n",
       "      <td><NAME></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>09/09/2017 12:00 PM - 5:00 PM</td>\n",
       "      <td>Saturday Safety Boat</td>\n",
       "      <td><NAME></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58</th>\n",
       "      <td>09/10/2017 10:30 AM - 1:00 PM</td>\n",
       "      <td>Sunday Safety Boat</td>\n",
       "      <td><NAME></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>09/30/2017 12:00 PM - 5:00 PM</td>\n",
       "      <td>Saturday Safety Boat - Commodore Cup</td>\n",
       "      <td><NAME></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                             Date                                 Event  \\\n",
       "0   05/13/2017 12:00 PM - 5:00 PM                  Saturday Safety Boat   \n",
       "1   05/13/2017 12:00 PM - 5:00 PM                  Saturday Safety Boat   \n",
       "2   05/14/2017 10:30 AM - 1:00 PM                    Sunday Safety Boat   \n",
       "3   05/14/2017 10:30 AM - 1:00 PM                    Sunday Safety Boat   \n",
       "4   05/20/2017 12:00 PM - 5:00 PM                  Saturday Safety Boat   \n",
       "5   05/20/2017 12:00 PM - 5:00 PM                  Saturday Safety Boat   \n",
       "6   05/21/2017 10:30 AM - 1:00 PM                    Sunday Safety Boat   \n",
       "7   05/21/2017 10:30 AM - 1:00 PM                    Sunday Safety Boat   \n",
       "8   05/27/2017 12:00 PM - 5:00 PM                  Saturday Safety Boat   \n",
       "9   05/28/2017 10:30 AM - 1:00 PM                    Sunday Safety Boat   \n",
       "10  05/28/2017 10:30 AM - 1:00 PM                    Sunday Safety Boat   \n",
       "11  05/29/2017 10:30 AM - 1:00 PM                    Monday Safety Boat   \n",
       "12  05/29/2017 10:30 AM - 1:00 PM                    Monday Safety Boat   \n",
       "13  06/03/2017 12:00 PM - 5:00 PM                  Saturday Safety Boat   \n",
       "14  06/04/2017 10:30 AM - 1:00 PM                    Sunday Safety Boat   \n",
       "15  06/10/2017 12:00 PM - 5:00 PM                  Saturday Safety Boat   \n",
       "23  06/11/2017 10:30 AM - 1:00 PM                    Sunday Safety Boat   \n",
       "24  06/18/2017 10:30 AM - 1:00 PM                    Sunday Safety Boat   \n",
       "25  06/24/2017 12:00 PM - 5:00 PM                  Saturday Safety Boat   \n",
       "26  06/24/2017 12:00 PM - 5:00 PM                  Saturday Safety Boat   \n",
       "27  06/25/2017 10:30 AM - 1:00 PM                    Sunday Safety Boat   \n",
       "28  06/25/2017 10:30 AM - 1:00 PM                    Sunday Safety Boat   \n",
       "29  07/01/2017 12:00 PM - 5:00 PM                  Saturday Safety Boat   \n",
       "30  07/01/2017 12:00 PM - 5:00 PM                  Saturday Safety Boat   \n",
       "31  07/02/2017 10:30 AM - 1:00 PM                    Sunday Safety Boat   \n",
       "32  07/04/2017 10:30 AM - 1:00 PM                    Monday Safety Boat   \n",
       "33  07/04/2017 10:30 AM - 1:00 PM                    Monday Safety Boat   \n",
       "34  07/08/2017 12:00 PM - 5:00 PM                  Saturday Safety Boat   \n",
       "43  07/09/2017 10:30 AM - 1:00 PM                    Sunday Safety Boat   \n",
       "44  07/15/2017 12:00 PM - 5:00 PM                  Saturday Safety Boat   \n",
       "45  07/16/2017 10:30 AM - 1:00 PM                    Sunday Safety Boat   \n",
       "46  07/16/2017 10:30 AM - 1:00 PM                    Sunday Safety Boat   \n",
       "47  07/29/2017 12:00 PM - 5:00 PM                  Saturday Safety Boat   \n",
       "48  07/30/2017 10:30 AM - 1:00 PM                    Sunday Safety Boat   \n",
       "50  08/06/2017 10:30 AM - 1:00 PM                    Sunday Safety Boat   \n",
       "51  08/19/2017 12:00 PM - 5:00 PM                  Saturday Safety Boat   \n",
       "52  09/03/2017 10:30 AM - 1:00 PM                    Sunday Safety Boat   \n",
       "53  09/04/2017 10:30 AM - 1:00 PM                    Monday Safety Boat   \n",
       "54  09/09/2017 12:00 PM - 5:00 PM                  Saturday Safety Boat   \n",
       "58  09/10/2017 10:30 AM - 1:00 PM                    Sunday Safety Boat   \n",
       "59  09/30/2017 12:00 PM - 5:00 PM  Saturday Safety Boat - Commodore Cup   \n",
       "\n",
       "               Member  \n",
       "0         <NAME>  \n",
       "1      <NAME>  \n",
       "2        <NAME>  \n",
       "3   <NAME>  \n",
       "4   O'Brien, Jennifer  \n",
       "5        <NAME>  \n",
       "6   Loscheider, Steve  \n",
       "7      <NAME>  \n",
       "8        <NAME>  \n",
       "9     <NAME>  \n",
       "10        <NAME>  \n",
       "11        <NAME>  \n",
       "12      <NAME>  \n",
       "13   <NAME>  \n",
       "14   <NAME>  \n",
       "15   <NAME>  \n",
       "23     <NAME>  \n",
       "24     <NAME>  \n",
       "25      <NAME>  \n",
       "26        <NAME>  \n",
       "27        <NAME>  \n",
       "28    <NAME>  \n",
       "29       <NAME>  \n",
       "30      <NAME>  \n",
       "31     <NAME>  \n",
       "32       <NAME>  \n",
       "33      <NAME>  \n",
       "34        <NAME>  \n",
       "43    <NAME>  \n",
       "44      <NAME>  \n",
       "45     <NAME>  \n",
       "46  <NAME>  \n",
       "47      <NAME>  \n",
       "48     <NAME>  \n",
       "50       <NAME>  \n",
       "51        <NAME>  \n",
       "52       <NAME>  \n",
       "53     <NAME>  \n",
       "54  <NAME>  \n",
       "58  <NAME>  \n",
       "59       <NAME>  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# When did they volunteer?\n",
    "\n",
    "filled"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\ryangrosch\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:9: FutureWarning: currently extract(expand=None) means expand=False (return Index/Series/DataFrame) but in a future version of pandas this will be changed to expand=True (return DataFrame)\n",
      "  if __name__ == '__main__':\n",
      "C:\\Users\\ryangrosch\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:9: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  if __name__ == '__main__':\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Event</th>\n",
       "      <th>Registered</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Date</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2017-08-05</th>\n",
       "      <td>Saturday Safety Boat</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2017-08-12</th>\n",
       "      <td>Saturday Safety Boat</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2017-08-13</th>\n",
       "      <td>Sunday Safety Boat</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2017-08-19</th>\n",
       "      <td>Saturday Safety Boat</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2017-08-20</th>\n",
       "      <td>Sunday Safety Boat</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2017-08-26</th>\n",
       "      <td>Saturday Safety Boat</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2017-08-27</th>\n",
       "      <td>Sunday Safety Boat</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2017-09-02</th>\n",
       "      <td>Saturday Safety Boat</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2017-09-04</th>\n",
       "      <td>Monday Safety Boat</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2017-09-09</th>\n",
       "      <td>Saturday Safety Boat</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                           Event  Registered\n",
       "Date                                        \n",
       "2017-08-05  Saturday Safety Boat           1\n",
       "2017-08-12  Saturday Safety Boat           1\n",
       "2017-08-13    Sunday Safety Boat           1\n",
       "2017-08-19  Saturday Safety Boat           1\n",
       "2017-08-20    Sunday Safety Boat           1\n",
       "2017-08-26  Saturday Safety Boat           0\n",
       "2017-08-27    Sunday Safety Boat           1\n",
       "2017-09-02  Saturday Safety Boat           0\n",
       "2017-09-04    Monday Safety Boat           1\n",
       "2017-09-09  Saturday Safety Boat           1"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Where do we still need volunteers?\n",
    "\n",
    "unfilled_raw = pd.DataFrame(\n",
    "    {'Registered': registrations,\n",
    "     'Event': unfilled_events,\n",
    "     'Date': unfilled_dates\n",
    "    })\n",
    "unfilled = unfilled_raw.drop_duplicates()\n",
    "unfilled['Registered'] = unfilled['Registered'].str.extract('(\\d+)').fillna(0).astype(int)\n",
    "unfilled = unfilled.query('Registered < 2')\n",
    "unfilled['Date'] = unfilled['Date'].str[:10]\n",
    "unfilled['Date'] = pd.to_datetime(unfilled['Date'])\n",
    "unfilled = unfilled.set_index('Date')\n",
    "unfilled = unfilled['20170716':'20170922']\n",
    "unfilled"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
{'fraction_non_alphanumeric': 0.22664180650507562, 'fraction_numerical': 0.06045853186934604, 'mean_word_length': 1.2207483514798343, 'pattern_counts': {'":': 63, '<': 803, '<?xml version=': 0, '>': 801, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '574944', 'n_tokens_mistral': 12068, 'n_tokens_neox': 10792, 'n_words': 1943}