<jupyter_start><jupyter_text>Numpy ndarrays<jupyter_code># Subway ridership for 5 stations on 10 different days
ridership = np.array([
    [   0,    0,    2,    5,    0],
    [1478, 3877, 3674, 2328, 2539],
    [1613, 4088, 3991, 6461, 2691],
    [1560, 3392, 3826, 4787, 2613],
    [1608, 4802, 3932, 4477, 2705],
    [1576, 3933, 3909, 4979, 2685],
    [  95,  229,  255,  496,  201],
    [   2,    0,    1,   27,    0],
    [1438, 3785, 3589, 4174, 2215],
    [1342, 4043, 4009, 4665, 3033]
])
# Accessing elements

print(ridership[1, 3])
print()
print(ridership[1:3, 3:5])
print()
print(ridership[1, :])
    
# Vectorized operations on rows or columns

print(ridership[0, :] + ridership[1, :])
print()
print(ridership[:, 0] + ridership[:, 1])

# Vectorized operations on entire arrays

a = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]])
b = np.array([[1, 1, 1], [2, 2, 2], [3, 3, 3]])

print(a + b)

def mean_riders_for_max_station(ridership):
    '''
    Fill in this function to find the station with the maximum riders on the
    first day, then return the mean riders per day for that station. Also
    return the mean ridership overall for comparsion.
    
    Hint: NumPy's argmax() function might be useful:
    http://docs.scipy.org/doc/numpy/reference/generated/numpy.argmax.html
    '''
    overall_mean = ridership.mean()
    station_id = ridership[0, :].argmax()
    mean_for_max = ridership[:, station_id].mean()
    
    return (overall_mean, mean_for_max)

mean_riders_for_max_station(ridership)<jupyter_output><empty_output><jupyter_text>Numpy axis<jupyter_code># NumPy axis argument

a = np.array([
    [1, 2, 3],
    [4, 5, 6],
    [7, 8, 9]
    ])

# numpy arrays are indexed in row-major order, i.e. rows are axis 0, columns axis 1
    
print(a.sum())
print()
print(a.sum(axis=0))
print()
print(a.sum(axis=1))

def min_and_max_riders_per_day(ridership):
    '''
    Fill in this function. First, for each subway station, calculate the
    mean ridership per day. Then, out of all the subway stations, return the
    maximum and minimum of these values. That is, find the maximum
    mean-ridership-per-day and the minimum mean-ridership-per-day for any
    subway station.
    '''
    mean_ridership_per_station = ridership.mean(axis=0)
    max_daily_ridership = mean_ridership_per_station.max()
    min_daily_ridership = mean_ridership_per_station.min()
    
    return (max_daily_ridership, min_daily_ridership)

min_and_max_riders_per_day(ridership)<jupyter_output><empty_output><jupyter_text>NumPy and Pandas Data Types<jupyter_code># Subway ridership for 5 stations on 10 different days
ridership_df = pd.DataFrame(
    data=[[   0,    0,    2,    5,    0],
          [1478, 3877, 3674, 2328, 2539],
          [1613, 4088, 3991, 6461, 2691],
          [1560, 3392, 3826, 4787, 2613],
          [1608, 4802, 3932, 4477, 2705],
          [1576, 3933, 3909, 4979, 2685],
          [  95,  229,  255,  496,  201],
          [   2,    0,    1,   27,    0],
          [1438, 3785, 3589, 4174, 2215],
          [1342, 4043, 4009, 4665, 3033]],
    index=['05-01-11', '05-02-11', '05-03-11', '05-04-11', '05-05-11',
           '05-06-11', '05-07-11', '05-08-11', '05-09-11', '05-10-11'],
    columns=['R003', 'R004', 'R005', 'R006', 'R007']
)

ridership_df

# DataFrame creation

# You can create a DataFrame out of a dictionary mapping column names to values
df_1 = pd.DataFrame({'A': [0, 1, 2], 'B': [3, 4, 5]})

print(df_1)
# You can also use a list of lists or a 2D NumPy array
df_2 = pd.DataFrame([[0, 1, 2], [3, 4, 5]], columns=['A', 'B', 'C'])

print(df_2)
   
# Accessing elements

print(ridership_df.iloc[0])
print()
print(ridership_df.loc['05-05-11'])
print()
print(ridership_df['R003'])
print()
print(ridership_df.iloc[1, 3])
print()

# Accessing multiple rows

print(ridership_df.iloc[1: 4])
    
# Accessing multiple columns

print(ridership_df[['R003', 'R005']])
    
# Pandas axis

df = pd.DataFrame({'A': [0, 1, 2], 'B': [3, 4, 5]})

print(df.sum())
print()
print(df.sum(axis=1))
print()
print(df.values.sum()) # here df.values returns a 2D numpy array
    
def mean_riders_for_max_station(ridership):
    '''
    Fill in this function to find the station with the maximum riders on the
    first day, then return the mean riders per day for that station. Also
    return the mean ridership overall for comparsion.
    
    This is the same as a previous exercise, but this time the
    input is a Pandas DataFrame rather than a 2D NumPy array.
    '''
    overall_mean = ridership.values.mean()
    station_id = ridership.iloc[0].idxmax()
    mean_for_max = ridership[station_id].mean()
    
    return (overall_mean, mean_for_max)

mean_riders_for_max_station(ridership_df)<jupyter_output><empty_output><jupyter_text>Loading data into a DataFrame<jupyter_code>filename = "../data/nyc_subway_weather.csv"
subway_df = pd.read_csv(filename)

subway_df.head()  # get the first 5 rows
# get the stats per column

subway_df.describe()<jupyter_output><empty_output><jupyter_text>Calculating correlation using Pearson's r coefficient<jupyter_code>def correlation(x, y):
    '''
    Fill in this function to compute the correlation between the two
    input variables. Each input is either a NumPy array or a Pandas
    Series.
    
    correlation = average of (x in standard units) times (y in standard units)
    
    Remember to pass the argument "ddof=0" to the Pandas std() function!
    '''
    x = x - x.mean()
    y = y - y.mean()
    zscores_x = x / x.std(ddof=0)
    zscores_y = y / y.std(ddof=0)
    return (zscores_x * zscores_y).mean()


entries = subway_df['ENTRIESn_hourly']
cum_entries = subway_df['ENTRIESn']
rain = subway_df['meanprecipi']
temp = subway_df['meantempi']

print(correlation(entries, rain))
print(correlation(entries, temp))
print(correlation(rain, temp))
print(correlation(entries, cum_entries))<jupyter_output>0.035648515772230396
-0.026693348321570484
-0.22903432340834323
0.5858954707662166<jupyter_text>Using numpy.corrcoef for Pearson's r coefficient<jupyter_code>def correlation_numpy(x, y):
    return np.corrcoef(x, y)[0, 1]


print(correlation_numpy(entries, rain))
print(correlation_numpy(entries, temp))
print(correlation_numpy(rain, temp))
print(correlation_numpy(entries, cum_entries))<jupyter_output>0.03564851577224401
-0.026693348321570824
-0.2290343234084345
0.585895470766208<jupyter_text>Pandas axis names<jupyter_code>subway_df.mean(axis='index')
subway_df.mean(axis='columns')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13471023427866832, 'fraction_numerical': 0.10033908754623921, 'mean_word_length': 3.86431784107946, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '232314', 'license': 'MIT', 'n_tokens_mistral': 2870, 'n_tokens_neox': 2437, 'n_words': 636, 'path': 'src/Lesson 3 - Part 1.ipynb', 'repo_name': 'SpyrosDellas/intro-to-data-analysis', 'size': '6488'}