<jupyter_start><jupyter_text>Built-in Data Structures, Functions,   Data Structures and Sequences  Tuple<jupyter_code>tup = 4, 5, 6
tup
# tuples are immutable. Can we modify it in any way?
dir(tup)
nested_tup = (4, 5, 6), (7, 8)
nested_tup
tuple([4, 0, 2])
tup = tuple('string')
tup
tup[0]
tup = tuple(['foo', [1, 2], True])
tup[2] = False
# note that mutable objects within a tuple can still be modified
tup[1].append(3)
print(tup)
tup[1] = tup[1].append(10)
(4, None, 'foo') + (6, 0) + ('bar',)
('foo', 'bar') * 4
('foo', 'bar') * 2 - ('foo', 'bar')
# note that objects themselves are not copied, only references to them
tup = ([1, 2], 'string') * 4
print(tup)
tup[2].append(3)
print(tup)
tup[2] is tup[0]<jupyter_output><empty_output><jupyter_text>Unpacking tuples<jupyter_code>tup = (4, 5, 6)
a, b, c = tup
b
tup = 4, 5, (6, 7)
a, b, (c, d) = tup
d<jupyter_output><empty_output><jupyter_text>Swapping variable names in many languages would look like sotmp = aa = bb = tmp<jupyter_code>a, b = 1, 2
a
b
b, a = a, b
a
b
a, b = b, a
a is b
seq = [(1, 2, 3), (4, 5, 6), (7, 8, 9)]
for a, b, c in seq:
    print('a={0}, b={1}, c={2}'.format(a, b, c))
values = 1, 2, 3, 4, 5
a, b, *rest = values
print(a, b)
print(rest)
# by convention, you can use an underscore for unwanted values. Nothing special about "rest"
a, b, *_ = values<jupyter_output><empty_output><jupyter_text>Tuple methods<jupyter_code>a = (1, 2, 2, 2, 3, 4, 2)

# count how many times an object occurs
print(a.count(2))

# find an element's first positional index
print(a.index(2))
print(a.index(4))<jupyter_output>4
1
5<jupyter_text>List<jupyter_code># list methods
dir(list)

a_list = [2, 3, 7, None]
tup = ('foo', 'bar', 'baz')
b_list = list(tup)
print(b_list)
b_list[1] = 'peekaboo'
b_list
# list funtion is frequently used to materialize an iterator or generator object
gen = range(10)
print(gen)
print(list(gen))
print([gen])
print((gen))
print(tuple(gen))<jupyter_output>range(0, 10)
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
[range(0, 10)]
range(0, 10)
(0, 1, 2, 3, 4, 5, 6, 7, 8, 9)<jupyter_text>Adding and removing elements<jupyter_code>print(b_list)
b_list.append('dwarf')
b_list
b_list.insert(1, 'red')
b_list
b_list.pop(2)
b_list
# pop returns last element and removes it from list. Has anyone ever used the returned value of pop?
tmp = b_list.pop()
print(tmp)
print(b_list)
b_list.append('foo')
print(b_list)
b_list.remove('foo')
b_list
'dwarf' in b_list
'dwarf' not in b_list<jupyter_output><empty_output><jupyter_text>Concatenating and combining lists<jupyter_code>[4, None, 'foo'] + [7, 8, (2, 3)]
x = [4, None, 'foo']
x.extend([7, 8, (2, 3)])
x
x.extend((4, 5))
x
x.extend([4, 5, 6])
x
# This is preferred compared to the below example as far less copying has to be done.
everything = []
for chunk in list_of_lists:
    everything.extend(chunk)
# everything = []
for chunk in list_of_lists:
    everything = everything + chunk<jupyter_output><empty_output><jupyter_text>Sorting<jupyter_code>a = [7, 2, 5, 1, 3]
a.sort()
print(a)
a.sort(reverse=True)
print(a)
list('foxiE'.lower()) #in ['a', 'e', 'i', 'o', 'u']
def count_vowels(string):
    count = 0
    for letter in string.lower():
        if letter in ['a', 'e', 'i', 'o', 'u']:
            count += 1
    
    return count
b = ['saw', 'small', 'He', 'foxes', 'six']
b.sort(key=len)
print(b)
b.sort(key=count_vowels, reverse=True)
print(b)<jupyter_output>['He', 'saw', 'six', 'small', 'foxes']
['foxes', 'He', 'saw', 'six', 'small']<jupyter_text>Binary search and maintaining a sorted list<jupyter_code># bisect module does not check if list is sorted and will run on an unsorted list. This can cause issues. 
import bisect
c = [1, 2, 3, 4, 2, 7, 2] # unsorted list
print(bisect.bisect(c, 2))
print(bisect.insort(c, 2))
print(bisect.bisect(c, 5))
print(bisect.insort(c, 6))
c<jupyter_output>2
None
6
None<jupyter_text>Slicing<jupyter_code>seq = [7, 2, 3, 7, 5, 6, 0, 1]
seq[1:5]
seq[3:4] = [6, 3]
seq
print(len(seq))
print(seq[7:8])
print(seq[7:])
seq[7:15] = [21, 22, 23]
print(seq[7:15])
print(seq[:5])
seq[3:]
seq[-4:]
seq[-6:-2]
# Can supply a third number signaling a step
seq[::2]
# reverse a list by print elemnts of the list backwards
seq[::-1]<jupyter_output><empty_output><jupyter_text>Built-in Sequence Functions  enumerate i = 0for value in collection:    do something with value   i += 1 for i, value in enumerate(collection):    do something with value<jupyter_code># this code creates a dict with the key being the object in a sequence and the value being its location
some_list = ['foo', 'bar', 'baz']
mapping = {}
for i, v in enumerate(some_list):
    mapping[v] = i
mapping
print(sorted([7, 1, 2, 6, 0, 3, 2], reverse=True))
print(sorted([7, 1, 2, 6, 0, 3, 2], key=lambda x: x * -1))
sorted('horse race')<jupyter_output>[7, 6, 3, 2, 2, 1, 0]
[7, 6, 3, 2, 2, 1, 0]<jupyter_text>zip<jupyter_code>seq1 = ['foo', 'bar', 'baz']
seq2 = ['one', 'two', 'three']
zipped = zip(seq1, seq2)
list(zipped)
# number of tuples produced is determined by the shortest sequence
seq3 = [False, True]
list(zip(seq1, seq2, seq3))
for i, (a, b) in enumerate(zip(seq1, seq2)):
    print('{0}: {1}, {2}'.format(i, a, b))
list(enumerate(zip(seq1, seq2)))
# unzipping using the zip command. This of this as transposing the tuples
pitchers = [('Nolan', 'Ryan'),
            ('Roger', 'Clemens'),
            ('Schilling', 'Curt')]
first_names, last_names = zip(*pitchers)
print(first_names)
print(last_names)
print(list(zip(*pitchers)))<jupyter_output>('Nolan', 'Roger', 'Schilling')
('Ryan', 'Clemens', 'Curt')
[('Nolan', 'Roger', 'Schilling'), ('Ryan', 'Clemens', 'Curt')]<jupyter_text>reversed<jupyter_code># Reversed creates a generator object, so reversed sequence isn't created until materialized
list(reversed(range(10)))<jupyter_output><empty_output><jupyter_text>dictCommon names for dict are a hash map or associative array<jupyter_code>empty_dict = {}
d1 = {'a': 'some value', 'b' : [1, 2, 3, 4]}
d1
# same syntax to access elements or set elements as lists and tuples
d1[7] = 'an integer'
print(d1)
d1['b']
# is there a key with name 'b' in d1?
'b' in d1
# remove key, value pairs using del keyword or pop method
d1[5] = 'some value'
print(d1)
d1['dummy'] = 'another value'
print(d1)
del d1[5]
print(d1)
ret = d1.pop('dummy')
print(ret)
d1
# keys and values methods return iterators. They will return in the same order as one another
print(list(d1.keys()))
print(list(d1.values()))
# merge on dict into another
d1.update({'b' : 'foo', 'c' : 12})
d1<jupyter_output><empty_output><jupyter_text>Creating dicts from sequences mapping = {}for key, value in zip(key_list, value_list):    mapping[key] = value<jupyter_code># The dict functiona accepts two tuples as arguments as well
mapping = dict(zip(range(5), reversed(range(5))))
mapping<jupyter_output><empty_output><jupyter_text>Default values<jupyter_code>if key in some_dict:
    value = some_dict[key]
else:
    value = default_value<jupyter_output><empty_output><jupyter_text>value = some_dict.get(key, default_value)<jupyter_code>print(mapping.get(3, 'not here'))
print(mapping.get('Waldo', "can't find him"))
words = ['apple', 'bat', 'bar', 'atom', 'book']
by_letter = {}
for word in words:
    letter = word[0]
    if letter not in by_letter:
        by_letter[letter] = [word]
    else:
        by_letter[letter].append(word)
by_letter
# can use setdefault method to accomplish the above task
by_letter_two = {}
for word in words:
    letter = word[0]
    by_letter_two.setdefault(letter, []).append(word)
by_letter_two
from collections import defaultdict

by_letter_three = defaultdict(list)
for word in words:
    by_letter_three[word[0]].append(word)
by_letter_three<jupyter_output><empty_output><jupyter_text>LEFT OFF HERE Valid dict key types<jupyter_code># dictionary keys should be hashable and immutable. If a tuple, all items of tuple should also be immutable
print(hash('string'))
print(hash((1, 2, (2, 3))))
hash((1, 2, [2, 3])) # fails because lists are mutable
# convert a list to a tuple to use it as a key in a dict
d = {}
d[tuple([1, 2, 3])] = 5
d<jupyter_output><empty_output><jupyter_text>setSet is an unordered collection of unique elements<jupyter_code>print(set([2, 2, 2, 1, 3, 3]))
{2, 2, 2, 1, 3, 3}
# sets support mathematical set operations
a = {1, 2, 3, 4, 5}
b = {3, 4, 5, 6, 7, 8}
print(a.union(b))
a | b
print(a.intersection(b))
a & b
# News to me that you can use the |=, &= operators
c = a.copy()
c |= b
print(c)
d = a.copy()
d &= b
print(d)
# set elements should generally be immutable and must be hashable
my_data = [1, 2, 3, 4]
my_set = {tuple(my_data)}
print(my_set)
{[1, 2, 3, 4]}
a_set = {1, 2, 3, 4, 5}
print({1, 2, 3}.issubset(a_set))
a_set.issuperset({1, 2, 3})
{1, 2, 3} == {3, 2, 1}<jupyter_output><empty_output><jupyter_text>List of most common set operations can be found on page 66  List, Set, and Dict Comprehensions<jupyter_code>nums = range(20)
result_1 = []
for num in nums:
    if num % 2 == 0:
        result_1.append(num)
result_2 = [num for num in nums if num % 2 == 0]

print(result_1)
print(result_2)
strings = ['a', 'as', 'bat', 'car', 'dove', 'python']
[x.upper() for x in strings if len(x) > 2]<jupyter_output><empty_output><jupyter_text>Can use dictionary and set comprehensions in the same waydict_comp = { set_comp = {<jupyter_code>unique_lengths = {len(x) for x in strings}
unique_lengths
print(map(len, strings))
set(map(len, strings))
loc_mapping = {val: index for index, val in enumerate(strings)}
loc_mapping<jupyter_output><empty_output><jupyter_text>Nested list comprehensions<jupyter_code>all_data = [['John', 'Emily', 'Michael', 'Mary', 'Steven'],
            ['Maria', 'Juan', 'Javier', 'Natalia', 'Pilar']]
# Here is one way to get all names in all_data that have 2 or more e's
names_of_interest = []
for names in all_data:
    enough_es = [name for name in names if name.count('e') >= 2]
    names_of_interest.extend(enough_es)
names_of_interest
# nested list comprehension solution
result = [name for names in all_data for name in names
          if name.count('e') >= 2]
result<jupyter_output><empty_output><jupyter_text>Does anyone have thoughts on the two above methods? Readability, conciseness?<jupyter_code>some_tuples = [(1, 2, 3), (4, 5, 6), (7, 8, 9)]
flattened = [x for tup in some_tuples for x in tup]
flattened<jupyter_output><empty_output><jupyter_text>flattened = []for tup in some_tuples:    for x in tup:        flattened.append(x)<jupyter_code># recreate the nested list from flattened list
[tuple(x for x in tup) for tup in some_tuples]<jupyter_output><empty_output><jupyter_text>Functions<jupyter_code>def my_function(x, y, z=1.5):
    if z > 1:
        return z * (x + y)
    else:
        return z / (x + y)
# all valid ways to call the function. Keyword arguments must follow positional arguments
my_function(5, 6, z=0.7)
my_function(3.14, 7, 3.5)
my_function(10, 20)<jupyter_output><empty_output><jupyter_text>Functions are created with the `def` keyword and complete with the `return` keyword. If there is no return statement, the function will return `None`.   Namespaces, Scope, and Local Functions<jupyter_code># a_list is created in local namespace and then destroyed
def func():
    a_list = []
    for i in range(5):
        a.append(i)
a_list
# a is modified everytime func is called because it lives in the global namespace
a = []
def func():
    for i in range(5):
        a.append(i)
func()
func()
a
# Must use global keyword within a function to assign value to variable in global scope
a = None
def bind_a_variable():
    global a
    a = []
bind_a_variable()
print(a)<jupyter_output>[]<jupyter_text>Thoughts (discuss)* Ideally functions should operate locally and not depend on the global namespace* Assigning variables to the global environment from within a function is not best practice  Returning Multiple Values<jupyter_code>def f():
    a = 5
    b = 6
    c = 7
    return a, b, c

a, b, c = f()
return_value = f()
def f():
    a = 5
    b = 6
    c = 7
    return {'a' : a, 'b' : b, 'c' : c}<jupyter_output><empty_output><jupyter_text>Functions Are Objects<jupyter_code>states = ['   Alabama ', 'Georgia!', 'Georgia', 'georgia', 'FlOrIda',
          'south   carolina##', 'West virginia?']
import re

def clean_strings(strings):
    result = []
    for value in strings:
        value = value.strip()
        value = re.sub('[!#?]', '', value)
        value = value.title()
        result.append(value)
    return result
clean_strings(states)
# functions are objects and therefore can be applied from a list
def remove_punctuation(value):
    return re.sub('[!#?]', '', value)

clean_ops = [str.strip, remove_punctuation, str.title]

def clean_strings(strings, ops):
    result = []
    for value in strings:
        for function in ops:
            value = function(value)
        result.append(value)
    return result
clean_strings(states, clean_ops)
for x in map(remove_punctuation, states):
    print(x)<jupyter_output>Alabama 
Georgia
Georgia
georgia
FlOrIda
south   carolina
West virginia<jupyter_text>Anonymous (Lambda) Functions<jupyter_code>def short_function(x):
    return x * 2

# Not best practice to store anonymous functions and reuse them more than once or twice. thoughts?
equiv_anon = lambda x: x * 2
equiv_anon(2)
def apply_to_list(some_list, f):
    return [f(x) for x in some_list]

ints = [4, 0, 1, 5, 6]
apply_to_list(ints, lambda x: x * 2)
strings = ['foo', 'card', 'bar', 'aaaa', 'abab']
strings.sort(key=lambda x: len(set(list(x))))
strings<jupyter_output><empty_output><jupyter_text>Currying: Partial Argument Application<jupyter_code>def add_numbers(x, y):
    return x + y
add_five = lambda y: add_numbers(5, y)
add_five(10)
from functools import partial
add_five = partial(add_numbers, x=5)
print(add_five(y=10))
add_five(10)<jupyter_output>15<jupyter_text>GeneratorsPython uses the iterator protocol to consistently iterate over different types of objects<jupyter_code># Iterating over dictionary yields dict keys
some_dict = {'a': 1, 'b': 2, 'c': 3}
for key in some_dict:
    print(key)
# python tries to create iterator object when you try to loop over dict
dict_iterator = iter(some_dict)
dict_iterator
list(dict_iterator)<jupyter_output><empty_output><jupyter_text>Most methods expecting a list or list-like object will also accept any iterable object.Using the `yield` keyword returns a generator<jupyter_code>def squares(n=10):
    print('Generating squares from 1 to {0}'.format(n ** 2))
    for i in range(1, n + 1):
        yield i ** 2
gen = squares()
gen
# The code is not actually run until we request the objects
for x in gen:
    print(x, end=' ')<jupyter_output>Generating squares from 1 to 100
1 4 9 16 25 36 49 64 81 100<jupyter_text>Generator expresssions<jupyter_code># wrap expression in parens instead of brackets / braces
gen = (x ** 2 for x in range(100))
gen
# equivalent to above expression
def _make_gen():
    for x in range(100):
        yield x ** 2
gen = _make_gen()
gen
# can supply generators as function args to many functions
print(sum(x ** 2 for x in range(100)))
dict((i, i **2) for i in range(5))<jupyter_output>328350<jupyter_text>itertools module<jupyter_code># itertools has many good functions for creating generators
import itertools
first_letter = lambda x: x[0]
names = ['Alan', 'Adam', 'Wes', 'Will', 'Albert', 'Steven']
for letter, names in itertools.groupby(names, first_letter):
    print(letter, list(names)) # names is a generator<jupyter_output>A ['Alan', 'Adam']
W ['Wes', 'Will']
A ['Albert']
S ['Steven']<jupyter_text>Errors and Exception Handling<jupyter_code>float('1.2345')
float('something')
def attempt_float(x):
    try:
        return float(x)
    except:
        return x
print(attempt_float('1.2345'))
attempt_float('something')
float((1, 2))
# can suppress specific exceptions
def attempt_float(x):
    try:
        return float(x)
    except ValueError:
        return x
attempt_float((1, 2))
def attempt_float(x):
    try:
        return float(x)
    except (TypeError, ValueError):
        return x
attempt_float((1, 2))
# code in a finally block will execute regardless of if code in try succeeds or not
f = open('requirements.txt', 'w')

try:
    write_to_file(f)
finally:
    print('executing finally')
    f.close()
# else will only run if try block succeeds
f = open('requirements.txt', 'w')

try:
    write_to_file(f)
except:
    print('Failed')
else:
    print('Succeeded')
finally:
    f.close()<jupyter_output>Failed<jupyter_text>Exceptions in IPython In [10]: %run examples/ipython_bug.py---------------------------------------------------------------------------AssertionError                            Traceback (most recent call last)/home/wesm/code/pydata-book/examples/ipython_bug.py in ()     13     throws_an_exception()     14---> 15 calling_things()/home/wesm/code/pydata-book/examples/ipython_bug.py in calling_things()     11 def calling_things():     12     works_fine()---> 13     throws_an_exception()     14     15 calling_things()/home/wesm/code/pydata-book/examples/ipython_bug.py in throws_an_exception()      7     a = 5      8     b = 6----> 9     assert(a + b == 10)     10     11 def calling_things():AssertionError:  Files and the Operating System<jupyter_code># file opens in read only mode by default
path = 'examples/segismundo.txt'
f = open(path)
for line in f:
    pass
# lines come out of file with EOL markers intact
lines = [x.rstrip() for x in open(path)]
lines
f.close()
# connection is automatically closed once all code in with block is executed
with open(path) as f:
    lines = [x.rstrip() for x in f]
lines[:5]
f = open(path)
print(f.read(10))
f2 = open(path, 'rb')  # Binary mode
f2.read(10)<jupyter_output>Sueña el r<jupyter_text>Table 3.3 on page 82 shows all modes in which a file can be opened<jupyter_code># above, read advances the file handle's position by the number of bytes read. Tell gives you the current position
print(f.tell())
f2.tell()
import sys
sys.getdefaultencoding()
# seek changes the handle to the given position in the file
f.seek(3)
f.read(1)
f.close()
f2.close()
# write out a file with no blank lines
with open('tmp.txt', 'w') as handle:
    handle.writelines(x for x in open(path) if len(x) > 1)
with open('tmp.txt') as f:
    lines = f.readlines()
lines
import os
os.remove('tmp.txt')<jupyter_output><empty_output><jupyter_text>Bytes and Unicode with Files<jupyter_code>with open(path) as f:
    chars = f.read(10)
chars
with open(path, 'rb') as f:
    data = f.read(10)
data
print(data.decode('utf8'))
data[:4].decode('utf8')
# converting from one unicode encoding to another
sink_path = 'sink.txt'
with open(path) as source:
    with open(sink_path, 'xt', encoding='iso-8859-1') as sink:
        sink.write(source.read())
with open(sink_path, encoding='iso-8859-1') as f:
    print(f.read(10))
os.remove(sink_path)
# error will be thrown when reading incomplete bytes
f = open(path)
print(f.read(5))
f.seek(4)
f.read(1)
f.close()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14407898210231773, 'fraction_numerical': 0.027928831635077346, 'mean_word_length': 4.558098591549296, 'pattern_counts': {'":': 0, '<': 139, '<?xml version=': 0, '>': 147, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '38', 'dir': 'jupyter-structured-clean-dedup', 'id': '178394', 'license': 'MIT', 'n_tokens_mistral': 7358, 'n_tokens_neox': 6909, 'n_words': 2199, 'path': 'ch03.ipynb', 'repo_name': 'zschuster/pydata-book', 'size': '18941'}