<commit_before>{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",<commit_msg>Load with pandas<commit_after>{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Status</th>\n",
       "      <th>Sentence</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>False</td>\n",
       "      <td>Go until jurong point, crazy.. Available only ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>False</td>\n",
       "      <td>Ok lar... Joking wif u oni...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>True</td>\n",
       "      <td>Free entry in 2 a wkly comp to win FA Cup fina...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>False</td>\n",
       "      <td>U dun say so early hor... U c already then say...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>False</td>\n",
       "      <td>Nah I don't think he goes to usf, he lives aro...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Status                                           Sentence\n",
       "0  False  Go until jurong point, crazy.. Available only ...\n",
       "1  False                      Ok lar... Joking wif u oni...\n",
       "2   True  Free entry in 2 a wkly comp to win FA Cup fina...\n",
       "3  False  U dun say so early hor... U c already then say...\n",
       "4  False  Nah I don't think he goes to usf, he lives aro..."
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "data = pd.read_csv('data/smsspamcollection/SMSSpamCollection', \n",
    "            sep='\\t', \n",
    "            header=None, \n",
    "            names=['Status', 'Sentence'], \n",
    "            true_values=['spam'], \n",
    "            false_values=['ham'])\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data['lower'] = [sentence.lower() for sentence in data['Sentence']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Status</th>\n",
       "      <th>Sentence</th>\n",
       "      <th>lower</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>False</td>\n",
       "      <td>Go until jurong point, crazy.. Available only ...</td>\n",
       "      <td>go until jurong point, crazy.. available only ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>False</td>\n",
       "      <td>Ok lar... Joking wif u oni...</td>\n",
       "      <td>ok lar... joking wif u oni...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>True</td>\n",
       "      <td>Free entry in 2 a wkly comp to win FA Cup fina...</td>\n",
       "      <td>free entry in 2 a wkly comp to win fa cup fina...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>False</td>\n",
       "      <td>U dun say so early hor... U c already then say...</td>\n",
       "      <td>u dun say so early hor... u c already then say...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>False</td>\n",
       "      <td>Nah I don't think he goes to usf, he lives aro...</td>\n",
       "      <td>nah i don't think he goes to usf, he lives aro...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Status                                           Sentence  \\\n",
       "0  False  Go until jurong point, crazy.. Available only ...   \n",
       "1  False                      Ok lar... Joking wif u oni...   \n",
       "2   True  Free entry in 2 a wkly comp to win FA Cup fina...   \n",
       "3  False  U dun say so early hor... U c already then say...   \n",
       "4  False  Nah I don't think he goes to usf, he lives aro...   \n",
       "\n",
       "                                               lower  \n",
       "0  go until jurong point, crazy.. available only ...  \n",
       "1                      ok lar... joking wif u oni...  \n",
       "2  free entry in 2 a wkly comp to win fa cup fina...  \n",
       "3  u dun say so early hor... u c already then say...  \n",
       "4  nah i don't think he goes to usf, he lives aro...  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
{'fraction_non_alphanumeric': 0.21496398166339228, 'fraction_numerical': 0.007531106745252128, 'mean_word_length': 1.2983446200150488, 'pattern_counts': {'":': 56, '<': 127, '<?xml version=': 0, '>': 127, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4986882', 'n_tokens_mistral': 2238, 'n_tokens_neox': 2186, 'n_words': 517}