<jupyter_start><jupyter_text>Finding Key Connectors<jupyter_code>users = [
    {'id': 0, 'name': 'Hero'},
    {'id': 1, 'name': 'Dunn'},
    {'id': 2, 'name': 'Sue'},
    {'id': 3, 'name': 'Chi'},
    {'id': 4, 'name': 'Thor'},
    {'id': 5, 'name': 'Clive'},
    {'id': 6, 'name': 'Hicks'},
    {'id': 7, 'name': 'Devin'},
    {'id': 8, 'name': 'Kate'},
    {'id': 9, 'name': 'Klein'}
]
friendship = [(0, 1), (0, 2), (1, 2), (1, 3), (2, 3), (3, 4),
              (4, 5), (5, 6), (5, 7), (6, 8), (7, 8), (8, 9)]
users
friendship
for user in users:
    user['friends'] = []
users
for i, j in friendship:
    users[i]['friends'].append(users[j])
    users[j]['friends'].append(users[i])
def number_of_friends(user):
    '''how many friends do _user_ have'''
    return len(user['friends'])
help(number_of_friends)
avg = sum(number_of_friends(user) for user in users) / len(users)
num_friends_by_id = [
    (user['id'], number_of_friends(user))
    for user in users
]
num_friends_by_id
sorted(num_friends_by_id, key = lambda user: user[1], reverse = True)<jupyter_output><empty_output><jupyter_text>Data Scientists You May Know<jupyter_code>def friends_of_friend_id_in(user):
    return [foaf['id']
            for friend in user['friends']
            for foaf in friend['friends']]

friends_of_friend_id_in(users[0])
from collections import Counter
help(Counter)
def not_same(user1, user2):
    '''whether _user1_ is not the same as _user2_'''
    return user1 != user2

def not_friend(user1, user2):
    '''whether _user1_ is a friend of _user2_'''
    return user1 not in user2['friends']
def friends_of_friend_ids(user):
    return Counter([foaf['id'] 
                   for friend in user['friends']
                   for foaf in friend['friends']
                   if not_same(foaf, user)
                   if not_friend(foaf, user)])
friends_of_friend_ids(users[3])
interests = [
    (0, 'Hadoop'), (0, 'Big Data'), (0, 'HBase'), (0, 'Java'),
    (0, 'Spark'), (0, 'Storm'), (0, 'Cassandra'),
    (1, 'NoSQL'), (1, 'MongoDB'), (1, 'Cassandra'), (1, 'HBase'), 
    (1, 'Postgres'), (2, 'Python'), (2, 'scikit-learn'), (2, 'scipy'),
    (2, 'numpy'), (2, 'statsmodels'), (2, 'pandas'), (3, 'R'), (3, 'Python'),
    (3, 'statistics'), (3, 'regression'), (3, 'probability'),
    (4, 'machine learning'), (4, 'regression'), (4, 'decision trees'),
    (4, 'libsvm'), (5, 'Python'), (5, 'R'), (5, 'Java'), (5, 'C++'),
    (5, 'Haskell'), (5, 'programming languages'), (6, 'statistics'),
    (6, 'probability'), (6, 'mathematics'), (6, 'theory'),
    (7, 'machine learning'), (7, 'scikit-learn'), (7, 'Mahout'),
    (7, 'neural networks'), (8, 'neural networks'), (8, 'deep learning'),
    (8, 'Big Data'), (8, 'artificial intelligence'), (9, 'Hadoop'),
    (9, 'Java'), (9, 'MapReduce'), (9, 'Big Data')
]
def data_scientists_who_like(target_interest, interestsets):
    '''find data scientists via interests'''
    return [user_id
           for user_id, user_interest in interestsets
           if user_interest == target_interest]
data_scientists_who_like('Big Data', interests)
from collections import defaultdict
help(defaultdict)
user_ids_by_interest = defaultdict(list)
user_ids_by_interest
for user_id, interest in interests:
    user_ids_by_interest[interest].append(user_id)
user_ids_by_interest
interests_by_user_id = defaultdict(list)
for user_id, user_interest in interests:
    interests_by_user_id[user_id].append(user_interest)
interests_by_user_id
def common_interest_users(user):
    return Counter([interested_user_id
                   for interest in interests_by_user_id[user['id']]
                   for interested_user_id in user_ids_by_interest[interest]
                   if interested_user_id != user['id']])
common_interest_users(users[0])<jupyter_output><empty_output><jupyter_text>Salaries and Experience<jupyter_code>salaries_and_tenures = [(83000, 8.7), (88000, 8.1),
                        (48000, 0.7), (76000, 6),
                        (69000, 6.5), (76000, 7.5),
                        (60000, 2.5), (83000, 10),
                        (48000, 1.9), (63000, 4.2)]
salary_by_tenure = defaultdict(list)

for salary, tenure in salaries_and_tenures:
    salary_by_tenure[tenure].append(salary)
salary_by_tenure
average_salary_by_tenure = {
    tenure: sum(salaries) / len(salaries)
    for tenure, salaries in salary_by_tenure.items()
}
average_salary_by_tenure
def tenure_bucket(tenure):
    if tenure < 2:
        return 'less than two'
    elif tenure <= 5:
        return 'between two and five'
    else:
        return 'more than five'
salary_by_tenure_bucket = defaultdict(list)

for salary, tenure in salaries_and_tenures:
    bucket = tenure_bucket(tenure)
    salary_by_tenure_bucket[bucket].append(salary)
salary_by_tenure_bucket
average_salary_by_bucket = {
    bucket: sum(salaries) / len(salaries)
    for bucket, salaries in salary_by_tenure_bucket.items()
}
average_salary_by_bucket<jupyter_output><empty_output><jupyter_text>Paid Accounts<jupyter_code>def predict_paid_or_unpaid(years_experience):
    if years_experience < 3.0:
        return 'paid'
    elif years_experience < 8.5:
        return 'unpaid'
    else:
        return 'paid'<jupyter_output><empty_output><jupyter_text>Topics of Interest<jupyter_code>words_and_counts = Counter(word
                         for user, interest in interests
                         for word in interest.lower().split())
words_and_counts
for word, count in words_and_counts.most_common():
    if count > 1:
        print(word, count)<jupyter_output>java 3
big 3
data 3
python 3
learning 3
r 2
scikit-learn 2
hbase 2
neural 2
machine 2
regression 2
statistics 2
cassandra 2
hadoop 2
networks 2
probability 2
{'fraction_non_alphanumeric': 0.1551663747810858, 'fraction_numerical': 0.033450087565674254, 'mean_word_length': 3.7591666666666668, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '463619', 'license': 'MIT', 'n_tokens_mistral': 2175, 'n_tokens_neox': 2011, 'n_words': 509, 'path': 'Chapter1.ipynb', 'repo_name': 'nifannn/data-science-from-scratch', 'size': '5710'}