<jupyter_start><jupyter_text>Gather dataExcel has ~1M row limit which is easy to exceed today. Pandas can read these files and process them efficiently.<jupyter_code>import pandas as pd
from pathlib import Path
# CSV file with 1.1M rows (96MB)
# Can be read directly from the internet
df = pd.read_csv('https://talk-python-course-videos.nyc3.digitaloceanspaces.com/large-files/2019_customer_transactions.csv')

# Read from a local file
# File is not stored in github
#csv_file = Path.cwd() / 'data' / 'raw' / '2019_customer_transactions.csv'
#df = pd.read_csv(csv_file)
df.head()
df.info()
# Total the invoice amounts and count how many invoices there were
agg_func = {'invoice_total': ['sum'], 'invoice_num': ['nunique']}
df.groupby(['category']).agg(agg_func).style.format('{0:,.0f}')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12867647058823528, 'fraction_numerical': 0.0196078431372549, 'mean_word_length': 5.69672131147541, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '666509', 'license': 'MIT', 'n_tokens_mistral': 293, 'n_tokens_neox': 275, 'n_words': 80, 'path': 'webcast_materials/01-Large-File-Read.ipynb', 'repo_name': 'ctdorris/excel-to-python-course', 'size': '816'}