# Path: config/config.py
# CONFIG = Config(dictionary)
# 
# Path: bigbang/utils.py
# def get_paths_to_files_in_directory(
#     directory: str, file_dsc: str = "*"
# ) -> List[str]:
#     """Get paths of all files matching file_dsc in directory"""
#     template = f"{directory}{file_dsc}"
#     file_paths = glob.glob(template)
#     return file_paths
# 
# Path: bigbang/data_types.py

# 
# Path: bigbang/ingress/utils.py
# def get_website_content(
#     url: str,
#     session: Optional[requests.Session] = None,
# ) -> Union[str, BeautifulSoup]:
#     """
#     Get HTML code from website
# 
#     Note
#     ----
#     Servers don't like it when one is sending too many requests from same
#     ip address in short period of time. Therefore we need to:
#         a) catch 'requests.exceptions.RequestException' errors
#             (includes all possible errors to be on the safe side),
#         b) safe intermediate results,
#         c) continue where we left off at a later stage.
#     """
#     # TODO: include option to change BeautifulSoup args
#     try:
#         if session is None:
#             sauce = requests.get(url)
#             if sauce.status_code == 200:
#                 soup = BeautifulSoup(sauce.content, "html.parser")
#                 return soup
#             elif sauce.status_code == 404:
#                 logger.exception(f"HTTP 404 Error for {url}.")
#                 return "RequestException"
#         else:
#             sauce = session.get(url)
#             soup = BeautifulSoup(sauce.text, "html.parser")
#             return soup
#     except requests.exceptions.RequestException as e:
#         if "A2=" in url:
#             # if URL of mboxMessage
#             logger.exception(f"{e} for {url}.")
#             return "RequestException"
#         else:
#             SystemExit()
# 
# def get_auth_session(
#     url_login: str, username: str, password: str
# ) -> requests.Session:
#     """
#     Create AuthSession.
# 
#     There are three ways to create an AuthSession:
#         - parse username & password directly into method
#         - create a /bigbang/config/authentication.yaml file that contains keys
#         - type then into terminal when the method 'get_login_from_terminal'
#             is raised
#     """
#     if os.path.isfile(filepath_auth):
#         # read from /config/authentication.yaml
#         with open(filepath_auth, "r") as stream:
#             auth_key = yaml.safe_load(stream)
#         username = auth_key["username"]
#         password = auth_key["password"]
#     else:
#         # ask user for login keys
#         username, password = get_login_from_terminal(username, password)
# 
#     if username is None or password is None:
#         # continue without authentication
#         return None
#     else:
#         # Start the AuthSession
#         session = requests.Session()
#         # Create the payload
#         payload = {
#             "LOGIN1": "",
#             "Y": username,
#             "p": password,
#             "X": "",
#         }
#         # Post the payload to the site to log in
#         session.post(url_login, data=payload)
#         return session
# 
# def set_website_preference_for_header(
#     url_pref: str,
#     session: requests.Session,
# ) -> requests.Session:
#     """
#     Set the 'Email Headers' of the 'Archive Preferences' for the auth session
#     to 'Show All Headers'. Otherwise only a restricted list of header fields is
#     shown.
#     """
#     url_archpref = url_pref + "&TAB=2"
#     payload = {
#         "Email Headers": "b",
#     }
#     session.post(url_archpref, data=payload)
#     return session


from tqdm import tqdm
from abc import ABC, abstractmethod
from mailbox import mboxMessage
from pathlib import Path
from typing import Dict, List, Optional, Tuple, Union
from bs4 import BeautifulSoup
from config.config import CONFIG
from bigbang.utils import get_paths_to_files_in_directory
from bigbang.data_types import Message, MailList
from bigbang.ingress.utils import (
    get_website_content,
    get_auth_session,
    set_website_preference_for_header,
)
import time
import logging
import os
import email
import mailbox
import numpy as np
import pandas as pd
import requests
import yaml
import bigbang.bigbang_io as bio

class AbstractMailListWarning(BaseException):
    """Base class for AbstractMailList class specific exceptions"""

    pass


class AbstractMailListDomainWarning(BaseException):
    """Base class for AbstractMailListDomain class specific exceptions"""

    pass


class AbstractMessageParser(ABC):
    """
    This class handles the creation of an mailbox.mboxMessage object
    (using the from_*() methods) and its storage in various other file formats
    (using the to_*() methods) that can be saved on the local memory.
    """

    def __init__(
        self,
        website=False,
        url_login: Optional[str] = None,
        url_pref: Optional[str] = None,
        login: Optional[Dict[str, str]] = {"username": None, "password": None},
        session: Optional[requests.Session] = None,
    ):
        if website:
            if (session is None) and (url_login is not None):

# Path: config/config.py
# CONFIG = Config(dictionary)
# 
# Path: bigbang/utils.py
# def get_paths_to_files_in_directory(
#     directory: str, file_dsc: str = "*"
# ) -> List[str]:
#     """Get paths of all files matching file_dsc in directory"""
#     template = f"{directory}{file_dsc}"
#     file_paths = glob.glob(template)
#     return file_paths
# 
# Path: bigbang/data_types.py

# 
# Path: bigbang/ingress/utils.py
# def get_website_content(
#     url: str,
#     session: Optional[requests.Session] = None,
# ) -> Union[str, BeautifulSoup]:
#     """
#     Get HTML code from website
# 
#     Note
#     ----
#     Servers don't like it when one is sending too many requests from same
#     ip address in short period of time. Therefore we need to:
#         a) catch 'requests.exceptions.RequestException' errors
#             (includes all possible errors to be on the safe side),
#         b) safe intermediate results,
#         c) continue where we left off at a later stage.
#     """
#     # TODO: include option to change BeautifulSoup args
#     try:
#         if session is None:
#             sauce = requests.get(url)
#             if sauce.status_code == 200:
#                 soup = BeautifulSoup(sauce.content, "html.parser")
#                 return soup
#             elif sauce.status_code == 404:
#                 logger.exception(f"HTTP 404 Error for {url}.")
#                 return "RequestException"
#         else:
#             sauce = session.get(url)
#             soup = BeautifulSoup(sauce.text, "html.parser")
#             return soup
#     except requests.exceptions.RequestException as e:
#         if "A2=" in url:
#             # if URL of mboxMessage
#             logger.exception(f"{e} for {url}.")
#             return "RequestException"
#         else:
#             SystemExit()
# 
# def get_auth_session(
#     url_login: str, username: str, password: str
# ) -> requests.Session:
#     """
#     Create AuthSession.
# 
#     There are three ways to create an AuthSession:
#         - parse username & password directly into method
#         - create a /bigbang/config/authentication.yaml file that contains keys
#         - type then into terminal when the method 'get_login_from_terminal'
#             is raised
#     """
#     if os.path.isfile(filepath_auth):
#         # read from /config/authentication.yaml
#         with open(filepath_auth, "r") as stream:
#             auth_key = yaml.safe_load(stream)
#         username = auth_key["username"]
#         password = auth_key["password"]
#     else:
#         # ask user for login keys
#         username, password = get_login_from_terminal(username, password)
# 
#     if username is None or password is None:
#         # continue without authentication
#         return None
#     else:
#         # Start the AuthSession
#         session = requests.Session()
#         # Create the payload
#         payload = {
#             "LOGIN1": "",
#             "Y": username,
#             "p": password,
#             "X": "",
#         }
#         # Post the payload to the site to log in
#         session.post(url_login, data=payload)
#         return session
# 
# def set_website_preference_for_header(
#     url_pref: str,
#     session: requests.Session,
# ) -> requests.Session:
#     """
#     Set the 'Email Headers' of the 'Archive Preferences' for the auth session
#     to 'Show All Headers'. Otherwise only a restricted list of header fields is
#     shown.
#     """
#     url_archpref = url_pref + "&TAB=2"
#     payload = {
#         "Email Headers": "b",
#     }
#     session.post(url_archpref, data=payload)
#     return session


# Path: bigbang/ingress/abstract.py
from tqdm import tqdm
from abc import ABC, abstractmethod
from mailbox import mboxMessage
from pathlib import Path
from typing import Dict, List, Optional, Tuple, Union
from bs4 import BeautifulSoup
from config.config import CONFIG
from bigbang.utils import get_paths_to_files_in_directory
from bigbang.data_types import Message, MailList
from bigbang.ingress.utils import (
    get_website_content,
    get_auth_session,
    set_website_preference_for_header,
)
import time
import logging
import os
import email
import mailbox
import numpy as np
import pandas as pd
import requests
import yaml
import bigbang.bigbang_io as bio


class AbstractMailListWarning(BaseException):
    """Base class for AbstractMailList class specific exceptions"""

    pass


class AbstractMailListDomainWarning(BaseException):
    """Base class for AbstractMailListDomain class specific exceptions"""

    pass


class AbstractMessageParser(ABC):
    """
    This class handles the creation of an mailbox.mboxMessage object
    (using the from_*() methods) and its storage in various other file formats
    (using the to_*() methods) that can be saved on the local memory.
    """

    def __init__(
        self,
        website=False,
        url_login: Optional[str] = None,
        url_pref: Optional[str] = None,
        login: Optional[Dict[str, str]] = {"username": None, "password": None},
        session: Optional[requests.Session] = None,
    ):
        if website:
            if (session is None) and (url_login is not None):

                session = get_auth_session(url_login, **login)