# Path: config/config.py
# CONFIG = Config(dictionary)
# 
# Path: bigbang/analysis/utils.py
# def get_index_of_msgs_with_subject(
#     df: pd.DataFrame,
#     return_boolmask: bool = False,
# ) -> np.array:
# def get_index_of_msgs_with_datetime(
#     df: pd.DataFrame,
#     return_boolmask: bool = False,
# ) -> np.array:
# def clean_addresses(df: pd.DataFrame) -> pd.DataFrame:
# def clean_subject(df: pd.DataFrame) -> pd.DataFrame:
# def clean_datetime(df: pd.DataFrame) -> pd.DataFrame:
# def extract_email(from_field):
# def extract_domain(from_field):
# def domain_entropy(domain, froms):
# 
# Path: bigbang/data_types.py



import datetime
import glob
import os
import re
import logging
import mailbox
import numpy as np
import pandas as pd
from typing import Dict, List, Optional, Tuple, Union
from mailbox import mboxMessage
from pathlib import Path
from config.config import CONFIG
from bigbang.analysis import utils
from bigbang.data_types import Message, MailList, MailListDomain
def mlist_from_mbox_to_pandas_dataframe(filepath: str) -> pd.DataFrame:
    """
    Reads `mailbox.mboxMessage` objects from .mbox file and transforms it to a
    `pandas.DataFrame`.
    For a clearer definition on what a mailing list is, see:
    bigbang.ingress.abstract.AbstractList
    """
    box = mailbox.mbox(filepath, create=False)
    mlist = [dict(msg) for msg in list(box.values())]
    # TODO: find out why some fields are missing in some messages
    nr_of_fields = np.array([len(msg.keys()) for msg in mlist])
    nr_of_fields = np.unique(nr_of_fields, return_counts=True)
    df = pd.DataFrame(mlist)
    df = utils.clean_addresses(df)
    df = utils.clean_subject(df)
    df = utils.clean_datetime(df)
    return df


def mlist_from_mbox(filepath: str) -> list:
    """
    Reads `mailbox.mboxMessage` objects from .mbox file.
    For a clearer definition on what a mailing list is, see:
    bigbang.ingress.abstract.AbstractList
    """
    box = mailbox.mbox(filepath, create=False)
    return list(box.values())


def mlist_to_dict(

# Path: config/config.py
# CONFIG = Config(dictionary)
# 
# Path: bigbang/analysis/utils.py
# def get_index_of_msgs_with_subject(
#     df: pd.DataFrame,
#     return_boolmask: bool = False,
# ) -> np.array:
# def get_index_of_msgs_with_datetime(
#     df: pd.DataFrame,
#     return_boolmask: bool = False,
# ) -> np.array:
# def clean_addresses(df: pd.DataFrame) -> pd.DataFrame:
# def clean_subject(df: pd.DataFrame) -> pd.DataFrame:
# def clean_datetime(df: pd.DataFrame) -> pd.DataFrame:
# def extract_email(from_field):
# def extract_domain(from_field):
# def domain_entropy(domain, froms):
# 
# Path: bigbang/data_types.py



# Path: bigbang/bigbang_io.py
import datetime
import glob
import os
import re
import logging
import mailbox
import numpy as np
import pandas as pd
from typing import Dict, List, Optional, Tuple, Union
from mailbox import mboxMessage
from pathlib import Path
from config.config import CONFIG
from bigbang.analysis import utils
from bigbang.data_types import Message, MailList, MailListDomain

def mlist_from_mbox_to_pandas_dataframe(filepath: str) -> pd.DataFrame:
    """
    Reads `mailbox.mboxMessage` objects from .mbox file and transforms it to a
    `pandas.DataFrame`.
    For a clearer definition on what a mailing list is, see:
    bigbang.ingress.abstract.AbstractList
    """
    box = mailbox.mbox(filepath, create=False)
    mlist = [dict(msg) for msg in list(box.values())]
    # TODO: find out why some fields are missing in some messages
    nr_of_fields = np.array([len(msg.keys()) for msg in mlist])
    nr_of_fields = np.unique(nr_of_fields, return_counts=True)
    df = pd.DataFrame(mlist)
    df = utils.clean_addresses(df)
    df = utils.clean_subject(df)
    df = utils.clean_datetime(df)
    return df


def mlist_from_mbox(filepath: str) -> list:
    """
    Reads `mailbox.mboxMessage` objects from .mbox file.
    For a clearer definition on what a mailing list is, see:
    bigbang.ingress.abstract.AbstractList
    """
    box = mailbox.mbox(filepath, create=False)
    return list(box.values())


def mlist_to_dict(

    msgs: MailList,