<jupyter_start><jupyter_text>Load data<jupyter_code># Load some data
df = pd.read_csv('lending_club_1000.csv')
# Split into a test & training set
df_training = df.sample(int(len(df) * 0.8), replace=False, random_state=123)
df_test = df.drop(df_training.index)
df.head()<jupyter_output><empty_output><jupyter_text>Log dataset sketches<jupyter_code>from whylogs import get_or_create_session

session = get_or_create_session()<jupyter_output>2020-09-22 16:47:19,339 - whylogs.app.config - DEBUG - Attempting to load config file: None
2020-09-22 16:47:19,340 - whylogs.app.config - DEBUG - Attempting to load config file: .whylogs.yaml<jupyter_text>Log dataframe<jupyter_code>session.log_dataframe(df_training, 'training.data')
# Then you could do whatever training or calculations you'd like<jupyter_output><empty_output><jupyter_text>Inspect profiles/statistics<jupyter_code># You can also capture the logger response and interact with the generated
# profiles

# Log the test data
prof = session.log_dataframe(df_test, 'test.data')
summary = prof.flat_summary()
stats_df = summary['summary']
stats_df
# See one of the inspected histograms
hist_data = summary['hist']['fico_range_high']
bins = hist_data['bin_edges']
n = hist_data['counts']
bin_width = np.diff(bins)

plt.bar(bins[0:-1], n, bin_width, align='edge')<jupyter_output><empty_output><jupyter_text>Load logged data<jupyter_code>import glob<jupyter_output><empty_output><jupyter_text>Load flat table statistics<jupyter_code># Load the flat table statistics from the 'test.data' dataset
fnames = glob.glob('whylogs-output/test.data/dataset_summary/flat_table/dataset_summary*.csv')
fnames.sort()
# Load the most recent file
test_stats = pd.read_csv(fnames[-1])
test_stats<jupyter_output><empty_output><jupyter_text>Load the full dataset profile sketch<jupyter_code>from whylogs import DatasetProfile

# Load a dataset profile from the 'test.data' dataset
fnames = glob.glob('whylogs-output/test.data/dataset_profile/protobuf/*.bin')
fnames.sort()

test_prof = DatasetProfile.read_protobuf(fnames[-1], delimited_file=False)
test_prof<jupyter_output><empty_output><jupyter_text>---<jupyter_code># Not necessary, but you can reset the WhyLogs session if you want
from whylogs import reset_default_session
reset_default_session()<jupyter_output>2020-09-22 16:47:24,053 - whylogs.app.config - DEBUG - Attempting to load config file: None
2020-09-22 16:47:24,054 - whylogs.app.config - DEBUG - Attempting to load config file: .whylogs.yaml
{'fraction_non_alphanumeric': 0.11164658634538152, 'fraction_numerical': 0.03253012048192771, 'mean_word_length': 7.194078947368421, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '352428', 'license': 'Apache-2.0', 'n_tokens_mistral': 913, 'n_tokens_neox': 858, 'n_words': 188, 'path': 'python/logging_example.ipynb', 'repo_name': 'FelipeAdachi/whylogs-examples', 'size': '2490'}