<commit_before>#!/usr/local/bin/python
# -- coding: utf-8 --
from __future__ import division
import nltk, re, pprint

# ------------------------------------------------------------------------------
# Work through http://nltk.org/book/ch03.html
# -- gene + github at ology dot net not dot com
# ------------------------------------------------------------------------------
<commit_msg>Started ch3 with a handy inspect() function<commit_after>#!/usr/local/bin/python
# -- coding: utf-8 --
from __future__ import division
import nltk, re, pprint

# ------------------------------------------------------------------------------
# Work through http://nltk.org/book/ch03.html
# -- gene + github at ology dot net not dot com
# ------------------------------------------------------------------------------

def inspect(it):
    print 'type=%s, len=%d' % (type(it).__name__, len(it)),
    if isinstance(it, str):
        print 'chars'
    elif isinstance(it, list):
        print 'elements'

from urllib import urlopen
url = "http://www.gutenberg.org/files/2554/2554.txt"
raw = urlopen(url).read()
# Or this:
#proxies = {'http': 'http://www.someproxy.com:3128'}
#raw = urlopen(url, proxies=proxies).read()
inspect(raw) #type=str, len=1176893 chars
raw[31:73] #'Crime and Punishment, by <NAME>'

tokens = nltk.word_tokenize(raw)
inspect(tokens) #type=list, len=244484 elements
tokens[5:12] #['Crime', 'and', 'Punishment', ',', 'by', 'Fyodor', 'Dostoevsky']
{'fraction_non_alphanumeric': 0.3489655172413793, 'fraction_numerical': 0.02689655172413793, 'mean_word_length': 1.722326454033771, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6209473', 'n_tokens_mistral': 456, 'n_tokens_neox': 430, 'n_words': 108}