<commit_before><commit_msg>Adding PandasandJupyter plus updates<commit_after>{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Below is from https://dev.socrata.com/blog/2016/02/01/pandas-and-jupyter-notebook.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.0.3\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "print(pd.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.read_json?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>geography</th>\n",
       "      <th>geography_type</th>\n",
       "      <th>year</th>\n",
       "      <th>less_than_high_school_graduate</th>\n",
       "      <th>high_school_graduate</th>\n",
       "      <th>some_college_or_associate_s_degree</th>\n",
       "      <th>bachelor_s_degree_or_higher</th>\n",
       "      <th>location_1</th>\n",
       "      <th>:@computed_region_uph5_8hpn</th>\n",
       "      <th>:@computed_region_i2t2_cryp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Atherton</td>\n",
       "      <td>Town</td>\n",
       "      <td>2014-01-01T00:00:00.000</td>\n",
       "      <td>13.6</td>\n",
       "      <td>12.3</td>\n",
       "      <td>2.7</td>\n",
       "      <td>3.5</td>\n",
       "      <td>{'type': 'Point', 'coordinates': [-122.2, 37.4...</td>\n",
       "      <td>2.0</td>\n",
       "      <td>28596</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Colma</td>\n",
       "      <td>Town</td>\n",
       "      <td>2014-01-01T00:00:00.000</td>\n",
       "      <td>6.3</td>\n",
       "      <td>6.4</td>\n",
       "      <td>10.4</td>\n",
       "      <td>2.4</td>\n",
       "      <td>{'type': 'Point', 'coordinates': [-122.455556,...</td>\n",
       "      <td>4.0</td>\n",
       "      <td>28588</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Foster City</td>\n",
       "      <td>City</td>\n",
       "      <td>2014-01-01T00:00:00.000</td>\n",
       "      <td>11.9</td>\n",
       "      <td>9.7</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.9</td>\n",
       "      <td>{'type': 'Point', 'coordinates': [-122.266389,...</td>\n",
       "      <td>6.0</td>\n",
       "      <td>319</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Portola Valley</td>\n",
       "      <td>Town</td>\n",
       "      <td>2014-01-01T00:00:00.000</td>\n",
       "      <td>48.1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.8</td>\n",
       "      <td>{'type': 'Point', 'coordinates': [-122.218611,...</td>\n",
       "      <td>14.0</td>\n",
       "      <td>28597</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Redwood City</td>\n",
       "      <td>City</td>\n",
       "      <td>2014-01-01T00:00:00.000</td>\n",
       "      <td>16.4</td>\n",
       "      <td>10.6</td>\n",
       "      <td>6.6</td>\n",
       "      <td>3.0</td>\n",
       "      <td>{'type': 'Point', 'coordinates': [-122.236111,...</td>\n",
       "      <td>21.0</td>\n",
       "      <td>28607</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        geography geography_type                     year  \\\n",
       "0        Atherton           Town  2014-01-01T00:00:00.000   \n",
       "1           Colma           Town  2014-01-01T00:00:00.000   \n",
       "2     Foster City           City  2014-01-01T00:00:00.000   \n",
       "3  Portola Valley           Town  2014-01-01T00:00:00.000   \n",
       "4    Redwood City           City  2014-01-01T00:00:00.000   \n",
       "\n",
       "   less_than_high_school_graduate  high_school_graduate  \\\n",
       "0                            13.6                  12.3   \n",
       "1                             6.3                   6.4   \n",
       "2                            11.9                   9.7   \n",
       "3                            48.1                   0.0   \n",
       "4                            16.4                  10.6   \n",
       "\n",
       "   some_college_or_associate_s_degree  bachelor_s_degree_or_higher  \\\n",
       "0                                 2.7                          3.5   \n",
       "1                                10.4                          2.4   \n",
       "2                                 2.0                          2.9   \n",
       "3                                 0.0                          1.8   \n",
       "4                                 6.6                          3.0   \n",
       "\n",
       "                                          location_1  \\\n",
       "0  {'type': 'Point', 'coordinates': [-122.2, 37.4...   \n",
       "1  {'type': 'Point', 'coordinates': [-122.455556,...   \n",
       "2  {'type': 'Point', 'coordinates': [-122.266389,...   \n",
       "3  {'type': 'Point', 'coordinates': [-122.218611,...   \n",
       "4  {'type': 'Point', 'coordinates': [-122.236111,...   \n",
       "\n",
       "   :@computed_region_uph5_8hpn  :@computed_region_i2t2_cryp  \n",
       "0                          2.0                        28596  \n",
       "1                          4.0                        28588  \n",
       "2                          6.0                          319  \n",
       "3                         14.0                        28597  \n",
       "4                         21.0                        28607  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_json(\"https://data.smcgov.org/resource/mb6a-xn89.json\")\n",
    "\n",
    "df.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['In',\n",
       " 'Out',\n",
       " '_',\n",
       " '_3',\n",
       " '__',\n",
       " '___',\n",
       " '__builtin__',\n",
       " '__builtins__',\n",
       " '__doc__',\n",
       " '__loader__',\n",
       " '__name__',\n",
       " '__package__',\n",
       " '__spec__',\n",
       " '_dh',\n",
       " '_i',\n",
       " '_i1',\n",
       " '_i2',\n",
       " '_i3',\n",
       " '_i4',\n",
       " '_ih',\n",
       " '_ii',\n",
       " '_iii',\n",
       " '_oh',\n",
       " 'df',\n",
       " 'exit',\n",
       " 'get_ipython',\n",
       " 'pd',\n",
       " 'quit']"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dir()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(32, 10)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
{'fraction_non_alphanumeric': 0.22655883731833099, 'fraction_numerical': 0.058837318330989215, 'mean_word_length': 1.132183908045977, 'pattern_counts': {'":': 70, '<': 157, '<?xml version=': 0, '>': 157, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 36, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3704605', 'n_tokens_mistral': 3518, 'n_tokens_neox': 3222, 'n_words': 430}