<jupyter_start><jupyter_text>Dependencies and logging<jupyter_code>import logging
from pathlib import Path

import humanfriendly
import ipywidgets as widgets
import pandas as pd
from IPython.display import display
from tqdm.notebook import tqdm

from helpers.logging import OutputWidgetHandler
from libratom.lib.pff import PffArchive
logger = logging.getLogger(__name__)
handler = OutputWidgetHandler()
handler.setFormatter(logging.Formatter(logging.BASIC_FORMAT))
logger.addHandler(handler)
logger.setLevel(logging.INFO)<jupyter_output><empty_output><jupyter_text>Location of PST files<jupyter_code># Edit as appropriate
CACHED_ENRON_DATA_DIR = Path("data/RevisedEDRMv1_Complete")

# Generate the list of files to know how many there are
files = sorted(CACHED_ENRON_DATA_DIR.glob('**/*.pst'))<jupyter_output><empty_output><jupyter_text>Report widget<jupyter_code># Container for the report
output = widgets.Output()
def update(out, data):
    """Refreshes the report output widget
    """

    out.clear_output(wait=True)
    
    df_data = {key: [value] for key, value in data.items()}
    df_data['Size'] = [humanfriendly.format_size(data['Size'])]
    
    with out:
        display(pd.DataFrame(df_data, index=['Total']) )<jupyter_output><empty_output><jupyter_text>Iterate over the PST files and do work<jupyter_code>handler.clear_logs()

# Overall report
report = {'Files': 0, 'Messages': 0, 'Attachments': 0, 'Size': 0, 'Errors': 0}

# Start displaying results
display(output)


# Iterate over files
with tqdm(total=len(files), desc="Files read", unit="files", leave=True) as file_bar:
    for pst_file in files:
        try:
            # Iterate over messages
            with PffArchive(pst_file) as archive:
                for message in archive.messages():
                    try:
                        # Do something with the message...
                        _ = archive.format_message(message)

                        # Update report
                        report['Messages'] += 1
                        report['Attachments'] += message.number_of_attachments

                        # Refresh report widget every 100 messages
                        if not report['Messages'] % 100:
                            update(output, report)

                    except Exception as exc:
                        # Log error and move on to the next message
                        report['Errors'] += 1
                        logger.exception(exc)

        except Exception as exc:
            # Log error and move on to the next file
            report['Errors'] += 1
            logger.exception(exc)

        # Update progress bar
        file_bar.update()

        # Update report
        report['Files'] += 1    
        report['Size'] += pst_file.stat().st_size

        # Refresh report widget
        update(output, report)
# Print out errors, if any 
handler.show_logs()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09466848940533151, 'fraction_numerical': 0.005468215994531784, 'mean_word_length': 2.61358024691358, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '476713', 'license': 'MIT', 'n_tokens_mistral': 854, 'n_tokens_neox': 804, 'n_words': 222, 'path': 'pst_walk.ipynb', 'repo_name': 'libratom/ratom-notebooks', 'size': '2926'}